You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ji...@apache.org on 2014/02/07 02:57:30 UTC
svn commit: r1565515 [3/3] - in
/hadoop/common/branches/HDFS-5698/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop...
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java Fri Feb 7 01:57:21 2014
@@ -110,6 +110,7 @@ public class AppBlock extends HtmlBlock
_("User:", app.getUser()).
_("Name:", app.getName()).
_("Application Type:", app.getApplicationType()).
+ _("Application Tags:", app.getApplicationTags()).
_("State:", app.getState()).
_("FinalStatus:", app.getFinalStatus()).
_("Started:", Times.format(app.getStartTime())).
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Fri Feb 7 01:57:21 2014
@@ -261,12 +261,14 @@ public class RMWebServices {
@QueryParam("startedTimeEnd") String startedEnd,
@QueryParam("finishedTimeBegin") String finishBegin,
@QueryParam("finishedTimeEnd") String finishEnd,
- @QueryParam("applicationTypes") Set<String> applicationTypes) {
+ @QueryParam("applicationTypes") Set<String> applicationTypes,
+ @QueryParam("applicationTags") Set<String> applicationTags) {
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
+ boolean checkAppTags = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
@@ -327,6 +329,11 @@ public class RMWebServices {
checkAppTypes = true;
}
+ Set<String> appTags = parseQueries(applicationTags, false);
+ if (!appTags.isEmpty()) {
+ checkAppTags = true;
+ }
+
// stateQuery is deprecated.
if (stateQuery != null && !stateQuery.isEmpty()) {
statesQuery.add(stateQuery);
@@ -354,6 +361,10 @@ public class RMWebServices {
request.setApplicationTypes(appTypes);
}
+ if (checkAppTags) {
+ request.setApplicationTags(appTags);
+ }
+
if (checkAppStates) {
request.setApplicationStates(appStates);
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java Fri Feb 7 01:57:21 2014
@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
+import com.google.common.base.Joiner;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
@@ -67,6 +67,7 @@ public class AppInfo {
protected String diagnostics;
protected long clusterId;
protected String applicationType;
+ protected String applicationTags = "";
// these are only allowed if acls allow
protected long startedTime;
@@ -117,6 +118,9 @@ public class AppInfo {
if (diagnostics == null || diagnostics.isEmpty()) {
this.diagnostics = "";
}
+ if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
+ this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
+ }
this.finalStatus = app.getFinalApplicationStatus();
this.clusterId = ResourceManager.getClusterTimeStamp();
if (hasAccess) {
@@ -239,6 +243,10 @@ public class AppInfo {
public String getApplicationType() {
return this.applicationType;
}
+
+ public String getApplicationTags() {
+ return this.applicationTags;
+ }
public int getRunningContainers() {
return this.runningContainers;
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Feb 7 01:57:21 2014
@@ -482,6 +482,13 @@ public class MockRM extends ResourceMana
RMAppAttempt attempt = app.getCurrentAppAttempt();
nm.nodeHeartbeat(true);
MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.LAUNCHED);
+ return am;
+ }
+
+ public static MockAM launchAndRegisterAM(RMApp app, MockRM rm, MockNM nm)
+ throws Exception {
+ MockAM am = launchAM(app, rm, nm);
am.registerAppAttempt();
rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
return am;
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Fri Feb 7 01:57:21 2014
@@ -41,8 +41,10 @@ import java.util.concurrent.BrokenBarrie
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
+import com.google.common.collect.Sets;
import junit.framework.Assert;
+import org.apache.commons.lang.math.LongRange;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -51,6 +53,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
@@ -58,6 +61,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -71,6 +75,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
@@ -232,6 +237,20 @@ public class TestClientRMService {
"application " + request.getApplicationId());
}
}
+
+ @Test (expected = ApplicationNotFoundException.class)
+ public void testMoveAbsentApplication() throws YarnException {
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getRMApps()).thenReturn(
+ new ConcurrentHashMap<ApplicationId, RMApp>());
+ ClientRMService rmService = new ClientRMService(rmContext, null, null,
+ null, null, null);
+ ApplicationId applicationId =
+ BuilderUtils.newApplicationId(System.currentTimeMillis(), 0);
+ MoveApplicationAcrossQueuesRequest request =
+ MoveApplicationAcrossQueuesRequest.newInstance(applicationId, "newqueue");
+ rmService.moveApplicationAcrossQueues(request);
+ }
@Test
public void testGetQueueInfo() throws Exception {
@@ -450,6 +469,7 @@ public class TestClientRMService {
{MockApps.newAppName(), MockApps.newAppName(), MockApps.newAppName()};
ApplicationId[] appIds =
{getApplicationId(101), getApplicationId(102), getApplicationId(103)};
+ List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
// Submit applications
for (int i = 0; i < appIds.length; i++) {
@@ -457,7 +477,8 @@ public class TestClientRMService {
when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
ApplicationAccessType.VIEW_APP, null, appId)).thenReturn(true);
SubmitApplicationRequest submitRequest = mockSubmitAppRequest(
- appId, appNames[i], queues[i % queues.length]);
+ appId, appNames[i], queues[i % queues.length],
+ new HashSet<String>(tags.subList(0, i + 1)));
rmService.submitApplication(submitRequest);
}
@@ -498,6 +519,41 @@ public class TestClientRMService {
userSet.add(UserGroupInformation.getCurrentUser().getShortUserName());
assertEquals("Incorrect number of applications for user", 3,
rmService.getApplications(request).getApplicationList().size());
+
+ // Check tags
+ request = GetApplicationsRequest.newInstance(
+ ApplicationsRequestScope.ALL, null, null, null, null, null, null,
+ null, null);
+ Set<String> tagSet = new HashSet<String>();
+ request.setApplicationTags(tagSet);
+ assertEquals("Incorrect number of matching tags", 6,
+ rmService.getApplications(request).getApplicationList().size());
+
+ tagSet = Sets.newHashSet(tags.get(0));
+ request.setApplicationTags(tagSet);
+ assertEquals("Incorrect number of matching tags", 3,
+ rmService.getApplications(request).getApplicationList().size());
+
+ tagSet = Sets.newHashSet(tags.get(1));
+ request.setApplicationTags(tagSet);
+ assertEquals("Incorrect number of matching tags", 2,
+ rmService.getApplications(request).getApplicationList().size());
+
+ tagSet = Sets.newHashSet(tags.get(2));
+ request.setApplicationTags(tagSet);
+ assertEquals("Incorrect number of matching tags", 1,
+ rmService.getApplications(request).getApplicationList().size());
+
+ // Check scope
+ request = GetApplicationsRequest.newInstance(
+ ApplicationsRequestScope.VIEWABLE);
+ assertEquals("Incorrect number of applications for the scope", 6,
+ rmService.getApplications(request).getApplicationList().size());
+
+ request = GetApplicationsRequest.newInstance(
+ ApplicationsRequestScope.OWN);
+ assertEquals("Incorrect number of applications for the scope", 3,
+ rmService.getApplications(request).getApplicationList().size());
}
@Test(timeout=4000)
@@ -568,6 +624,11 @@ public class TestClientRMService {
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
String name, String queue) {
+ return mockSubmitAppRequest(appId, name, queue, null);
+ }
+
+ private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
+ String name, String queue, Set<String> tags) {
ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
Resource resource = Resources.createResource(
@@ -581,6 +642,7 @@ public class TestClientRMService {
submissionContext.setApplicationId(appId);
submissionContext.setResource(resource);
submissionContext.setApplicationType(appType);
+ submissionContext.setApplicationTags(tags);
SubmitApplicationRequest submitRequest =
recordFactory.newRecordInstance(SubmitApplicationRequest.class);
@@ -649,7 +711,7 @@ public class TestClientRMService {
when(asContext.getMaxAppAttempts()).thenReturn(1);
RMAppImpl app = spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null , System
- .currentTimeMillis(), "YARN"));
+ .currentTimeMillis(), "YARN", null));
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(applicationId3, 1);
RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
rmContext, yarnScheduler, null, asContext, config, false);
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java Fri Feb 7 01:57:21 2014
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.spy;
+
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
@@ -33,6 +37,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
@@ -44,9 +49,17 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
@@ -54,7 +67,9 @@ import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRM {
private static final Log LOG = LogFactory.getLog(TestRM.class);
@@ -397,19 +412,19 @@ public class TestRM {
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
nm1.registerNode();
- MockAM am1 = MockRM.launchAM(app1, rm1, nm1);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
MockRM.finishApplicationMaster(app1, rm1, nm1, am1);
// a failed app
RMApp app2 = rm1.submitApp(200);
- MockAM am2 = MockRM.launchAM(app2, rm1, nm1);
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
am2.waitForState(RMAppAttemptState.FAILED);
rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED);
// a killed app
RMApp app3 = rm1.submitApp(200);
- MockAM am3 = MockRM.launchAM(app3, rm1, nm1);
+ MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
rm1.killApp(app3.getApplicationId());
rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED);
rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED);
@@ -449,7 +464,7 @@ public class TestRM {
// a failed app
RMApp app2 = rm1.submitApp(200);
- MockAM am2 = MockRM.launchAM(app2, rm1, nm1);
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
nm1
.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
am2.waitForState(RMAppAttemptState.FAILED);
@@ -466,10 +481,88 @@ public class TestRM {
Assert.assertEquals(-1, report1.getRpcPort());
}
+ /**
+ * Validate killing an application when it is at accepted state.
+ * @throws Exception exception
+ */
+ @Test (timeout = 60000)
+ public void testApplicationKillAtAcceptedState() throws Exception {
+
+ YarnConfiguration conf = new YarnConfiguration();
+ final Dispatcher dispatcher = new AsyncDispatcher() {
+ @Override
+ public EventHandler getEventHandler() {
+
+ class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
+ @Override
+ public boolean matches(Object argument) {
+ if (argument instanceof RMAppAttemptEvent) {
+ if (((RMAppAttemptEvent) argument).getType().equals(
+ RMAppAttemptEventType.KILL)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ EventHandler handler = spy(super.getEventHandler());
+ doNothing().when(handler).handle(argThat(new EventArgMatcher()));
+ return handler;
+ }
+ };
+
+ MockRM rm = new MockRM(conf) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+ };
+
+ rm.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ // a failed app
+ RMApp application = rm.submitApp(200);
+ MockAM am = MockRM.launchAM(application, rm, nm1);
+ am.waitForState(RMAppAttemptState.LAUNCHED);
+ nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
+ rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);
+
+ // Now kill the application before new attempt is launched, the app report
+ // returns the invalid AM host and port.
+ KillApplicationRequest request =
+ KillApplicationRequest.newInstance(application.getApplicationId());
+ rm.getClientRMService().forceKillApplication(request);
+
+ // Specific test for YARN-1689 follows
+ // Now let's say a race causes AM to register now. This should not crash RM.
+ am.registerAppAttempt(false);
+
+ // We explicitly intercepted the kill-event to RMAppAttempt, so app should
+ // still be in KILLING state.
+ rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
+ // AM should now be in running
+ rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
+
+ // Simulate that appAttempt is killed.
+ rm.getRMContext().getDispatcher().getEventHandler().handle(
+ new RMAppEvent(application.getApplicationId(),
+ RMAppEventType.ATTEMPT_KILLED));
+ rm.waitForState(application.getApplicationId(), RMAppState.KILLED);
+ }
+
public static void main(String[] args) throws Exception {
TestRM t = new TestRM();
t.testGetNewAppId();
t.testAppWithNoContainers();
t.testAppOnMultiNode();
+ t.testNMToken();
+ t.testActivatingApplicationAfterAddingNM();
+ t.testInvalidateAMHostPortWhenAMFailedOrKilled();
+ t.testInvalidatedAMHostPortOnAMRestart();
+ t.testApplicationKillAtAcceptedState();
}
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Fri Feb 7 01:57:21 2014
@@ -36,6 +36,8 @@ import org.junit.Test;
import java.io.IOException;
+import junit.framework.Assert;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -48,12 +50,15 @@ public class TestRMHA {
private static final String STATE_ERR =
"ResourceManager is in wrong HA state";
- private static final String RM1_ADDRESS = "0.0.0.0:0";
+ private static final String RM1_ADDRESS = "1.1.1.1:1";
private static final String RM1_NODE_ID = "rm1";
- private static final String RM2_ADDRESS = "1.1.1.1:1";
+ private static final String RM2_ADDRESS = "0.0.0.0:0";
private static final String RM2_NODE_ID = "rm2";
+ private static final String RM3_ADDRESS = "2.2.2.2:2";
+ private static final String RM3_NODE_ID = "rm3";
+
@Before
public void setUp() throws Exception {
configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
@@ -61,8 +66,8 @@ public class TestRMHA {
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
}
- configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
}
private void checkMonitorHealth() throws IOException {
@@ -278,6 +283,36 @@ public class TestRMHA {
rm.stop();
}
+ @Test
+ public void testHAIDLookup() {
+ //test implicitly lookup HA-ID
+ Configuration conf = new YarnConfiguration(configuration);
+ rm = new MockRM(conf);
+ rm.init(conf);
+
+ assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM2_NODE_ID);
+
+ //test explicitly lookup HA-ID
+ configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
+ conf = new YarnConfiguration(configuration);
+ rm = new MockRM(conf);
+ rm.init(conf);
+ assertEquals(conf.get(YarnConfiguration.RM_HA_ID), RM1_NODE_ID);
+
+ //test if RM_HA_ID can not be found
+ configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID+ "," + RM3_NODE_ID);
+ configuration.unset(YarnConfiguration.RM_HA_ID);
+ conf = new YarnConfiguration(configuration);
+ try {
+ rm = new MockRM(conf);
+ rm.init(conf);
+ fail("Should get an exception here.");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex.getMessage().contains(
+ "Invalid configuration! Can not find valid RM_HA_ID."));
+ }
+ }
+
@SuppressWarnings("rawtypes")
class MyCountingDispatcher extends AbstractService implements Dispatcher {
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Fri Feb 7 01:57:21 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.MockApps;
@@ -140,6 +141,11 @@ public abstract class MockAsm extends Mo
}
@Override
+ public Set<String> getApplicationTags() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
public void setQueue(String name) {
throw new UnsupportedOperationException("Not supported yet.");
}
@@ -235,7 +241,11 @@ public abstract class MockAsm extends Mo
public int getMaxAppAttempts() {
return maxAppAttempts;
}
-
+
+ @Override
+ public Set<String> getApplicationTags() {
+ return null;
+ }
};
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Fri Feb 7 01:57:21 2014
@@ -69,7 +69,7 @@ public class TestAMRestart {
new MockNM("127.0.0.1:2351", 4089, rm1.getResourceTrackerService());
nm2.registerNode();
- MockAM am1 = MockRM.launchAM(app1, rm1, nm1);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
int NUM_CONTAINERS = 3;
// allocate NUM_CONTAINERS containers
am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Fri Feb 7 01:57:21 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -218,6 +219,11 @@ public class MockRMApp implements RMApp
}
@Override
+ public Set<String> getApplicationTags() {
+ return null;
+ }
+
+ @Override
public boolean isAppSafeToTerminate() {
return true;
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Fri Feb 7 01:57:21 2014
@@ -230,7 +230,7 @@ public class TestRMAppTransitions {
RMApp application =
new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN");
+ System.currentTimeMillis(), "YARN", null);
testAppStartState(applicationId, user, name, queue, application);
this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
@@ -639,6 +639,13 @@ public class TestRMAppTransitions {
RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
+
+ assertAppState(RMAppState.KILLING, application);
+ RMAppEvent appAttemptKilled =
+ new RMAppEvent(application.getApplicationId(),
+ RMAppEventType.ATTEMPT_KILLED);
+ application.handle(appAttemptKilled);
+ assertAppState(RMAppState.FINAL_SAVING, application);
sendAppUpdateSavedEvent(application);
assertKilled(application);
assertAppFinalStateSaved(application);
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Fri Feb 7 01:57:21 2014
@@ -73,7 +73,7 @@ public class TestQueueMetrics {
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
- metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
+ metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
@@ -81,7 +81,7 @@ public class TestQueueMetrics {
metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
- metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
+ metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true);
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2));
@@ -171,7 +171,7 @@ public class TestQueueMetrics {
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
- metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
+ metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
@@ -181,7 +181,7 @@ public class TestQueueMetrics {
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
- metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
+ metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true);
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
checkResources(userSource, 6*GB, 6, 3, 3, 0, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
@@ -232,7 +232,7 @@ public class TestQueueMetrics {
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
- metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
+ metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3));
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
checkResources(parentQueueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
@@ -242,7 +242,7 @@ public class TestQueueMetrics {
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
- metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
+ metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true);
metrics.reserveResource(user, Resources.createResource(3*GB, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Feb 7 01:57:21 2014
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -56,10 +57,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -621,7 +624,7 @@ public class TestFairScheduler {
ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
null, null, null, ApplicationSubmissionContext.newInstance(null, null,
- null, null, null, false, false, 0, null, null), null, null, 0, null);
+ null, null, null, false, false, 0, null, null), null, null, 0, null, null);
appsMap.put(appAttemptId.getApplicationId(), rmApp);
AppAddedSchedulerEvent appAddedEvent =
@@ -647,7 +650,7 @@ public class TestFairScheduler {
ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
null, null, null, ApplicationSubmissionContext.newInstance(null, null,
- null, null, null, false, false, 0, null, null), null, null, 0, null);
+ null, null, null, false, false, 0, null, null), null, null, 0, null, null);
appsMap.put(appAttemptId.getApplicationId(), rmApp);
AppAddedSchedulerEvent appAddedEvent =
@@ -1765,7 +1768,7 @@ public class TestFairScheduler {
RMApp application =
new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user,
queue, submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN");
+ System.currentTimeMillis(), "YARN", null);
resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application);
application.handle(new RMAppEvent(applicationId, RMAppEventType.START));
@@ -2547,4 +2550,138 @@ public class TestFairScheduler {
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
scheduler.getSchedulerApplications(), scheduler, "default");
}
+
+ @Test
+ public void testMoveRunnableApp() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+ ApplicationId appId = appAttId.getApplicationId();
+ RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(updateEvent);
+
+ assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
+ scheduler.update();
+ assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
+
+ scheduler.moveApplication(appId, "queue2");
+ FSSchedulerApp app = scheduler.getSchedulerApp(appAttId);
+ assertSame(targetQueue, app.getQueue());
+ assertFalse(oldQueue.getRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertTrue(targetQueue.getRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
+ assertEquals(0, oldQueue.getNumRunnableApps());
+ assertEquals(1, targetQueue.getNumRunnableApps());
+ assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
+
+ scheduler.update();
+ assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand());
+ assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand());
+ }
+
+ @Test
+ public void testMoveNonRunnableApp() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+
+ assertEquals(0, oldQueue.getNumRunnableApps());
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ assertEquals(0, oldQueue.getNumRunnableApps());
+ assertEquals(0, targetQueue.getNumRunnableApps());
+ assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps());
+ }
+
+ @Test
+ public void testMoveMakesAppRunnable() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+
+ FSSchedulerApp app = scheduler.getSchedulerApp(appAttId);
+ assertTrue(oldQueue.getNonRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ assertFalse(oldQueue.getNonRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertFalse(targetQueue.getNonRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertTrue(targetQueue.getRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertEquals(1, targetQueue.getNumRunnableApps());
+ assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
+ }
+
+ @Test (expected = YarnException.class)
+ public void testMoveWouldViolateMaxAppsConstraints() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ }
+
+ @Test (expected = YarnException.class)
+ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2",
+ Resource.newInstance(1024, 1));
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+ RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+
+ assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ }
+
+ @Test (expected = YarnException.class)
+ public void testMoveToNonexistentQueue() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ scheduler.getQueueManager().getLeafQueue("queue1", true);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Fri Feb 7 01:57:21 2014
@@ -77,7 +77,8 @@ public class TestMaxRunningAppsEnforcer
private void removeApp(FSSchedulerApp app) {
app.getQueue().removeApp(app);
- maxAppsEnforcer.updateRunnabilityOnAppRemoval(app);
+ maxAppsEnforcer.untrackRunnableApp(app);
+ maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue());
}
@Test
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Fri Feb 7 01:57:21 2014
@@ -1317,8 +1317,8 @@ public class TestRMWebServicesApps exten
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- // 15 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 19, info.length());
+ // 20 because trackingUrl not assigned yet
+ assertEquals("incorrect number of elements", 20, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"), info.getString("queue"),
@@ -1389,7 +1389,7 @@ public class TestRMWebServicesApps exten
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- MockAM am = MockRM.launchAM(app1, rm, amNodeManager);
+ MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
int maxAppAttempts = rm.getConfig().getInt(
YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
@@ -1405,7 +1405,7 @@ public class TestRMWebServicesApps exten
}
// wait for app to start a new attempt.
rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
- am = MockRM.launchAM(app1, rm, amNodeManager);
+ am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
numAttempt++;
}
assertEquals("incorrect number of attempts", maxAppAttempts,
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Feb 7 01:57:21 2014
@@ -349,16 +349,20 @@ Queue Access Control Lists (ACLs)
* {Administration}
- The fair scheduler provides support for administration at runtime through two mechanisms:
+ The fair scheduler provides support for administration at runtime through a few mechanisms:
- * It is possible to modify minimum shares, limits, weights, preemption timeouts
- and queue scheduling policies at runtime by editing the allocation file. The
- scheduler will reload this file 10-15 seconds after it sees that it was
- modified.
-
- * Current applications, queues, and fair shares can be examined through the
- ResourceManager's web interface, at
- http://<ResourceManager URL>/cluster/scheduler.
+Modifying configuration at runtime
+
+ It is possible to modify minimum shares, limits, weights, preemption timeouts
+ and queue scheduling policies at runtime by editing the allocation file. The
+ scheduler will reload this file 10-15 seconds after it sees that it was
+ modified.
+
+Monitoring through web UI
+
+ Current applications, queues, and fair shares can be examined through the
+ ResourceManager's web interface, at
+ http://<ResourceManager URL>/cluster/scheduler.
The following fields can be seen for each queue on the web interface:
@@ -382,3 +386,17 @@ Queue Access Control Lists (ACLs)
In addition to the information that the ResourceManager normally displays
about each application, the web interface includes the application's fair share.
+Moving applications between queues
+
+ The Fair Scheduler supports moving a running application to a different queue.
+ This can be useful for moving an important application to a higher priority
+ queue, or for moving an unimportant application to a lower priority queue.
+ Apps can be moved by running "yarn application -movetoqueue appID -queue
+ targetQueueName".
+
+ When an application is moved to a queue, its existing allocations become
+ counted with the new queue's allocations instead of the old for purposes
+ of determining fairness. An attempt to move an application to a queue will
+ fail if the addition of the app's resources to that queue would violate the
+ its maxRunningApps or maxResources constraints.
+
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1565515&r1=1565514&r2=1565515&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Fri Feb 7 01:57:21 2014
@@ -1123,6 +1123,7 @@ ResourceManager REST API's.
* finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch
* finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
* applicationTypes - applications matching the given application types, specified as a comma-separated list.
+ * applicationTags - applications matching any of the given application tags, specified as a comma-separated list.
------
** Elements of the <apps> (Applications) object