You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2014/02/07 03:43:10 UTC
svn commit: r1565519 [3/3] - in
/hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop...
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1565519&r1=1565518&r2=1565519&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Fri Feb 7 02:43:04 2014
@@ -230,7 +230,7 @@ public class TestRMAppTransitions {
RMApp application =
new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN");
+ System.currentTimeMillis(), "YARN", null);
testAppStartState(applicationId, user, name, queue, application);
this.rmContext.getRMApps().putIfAbsent(application.getApplicationId(),
@@ -639,6 +639,13 @@ public class TestRMAppTransitions {
RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
+
+ assertAppState(RMAppState.KILLING, application);
+ RMAppEvent appAttemptKilled =
+ new RMAppEvent(application.getApplicationId(),
+ RMAppEventType.ATTEMPT_KILLED);
+ application.handle(appAttemptKilled);
+ assertAppState(RMAppState.FINAL_SAVING, application);
sendAppUpdateSavedEvent(application);
assertKilled(application);
assertAppFinalStateSaved(application);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1565519&r1=1565518&r2=1565519&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Fri Feb 7 02:43:04 2014
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -56,10 +57,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -621,7 +624,7 @@ public class TestFairScheduler {
ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
null, null, null, ApplicationSubmissionContext.newInstance(null, null,
- null, null, null, false, false, 0, null, null), null, null, 0, null);
+ null, null, null, false, false, 0, null, null), null, null, 0, null, null);
appsMap.put(appAttemptId.getApplicationId(), rmApp);
AppAddedSchedulerEvent appAddedEvent =
@@ -647,7 +650,7 @@ public class TestFairScheduler {
ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
null, null, null, ApplicationSubmissionContext.newInstance(null, null,
- null, null, null, false, false, 0, null, null), null, null, 0, null);
+ null, null, null, false, false, 0, null, null), null, null, 0, null, null);
appsMap.put(appAttemptId.getApplicationId(), rmApp);
AppAddedSchedulerEvent appAddedEvent =
@@ -1765,7 +1768,7 @@ public class TestFairScheduler {
RMApp application =
new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user,
queue, submissionContext, scheduler, masterService,
- System.currentTimeMillis(), "YARN");
+ System.currentTimeMillis(), "YARN", null);
resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application);
application.handle(new RMAppEvent(applicationId, RMAppEventType.START));
@@ -2547,4 +2550,138 @@ public class TestFairScheduler {
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
scheduler.getSchedulerApplications(), scheduler, "default");
}
+
+ @Test
+ public void testMoveRunnableApp() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+ ApplicationId appId = appAttId.getApplicationId();
+ RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(updateEvent);
+
+ assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
+ scheduler.update();
+ assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
+
+ scheduler.moveApplication(appId, "queue2");
+ FSSchedulerApp app = scheduler.getSchedulerApp(appAttId);
+ assertSame(targetQueue, app.getQueue());
+ assertFalse(oldQueue.getRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertTrue(targetQueue.getRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
+ assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
+ assertEquals(0, oldQueue.getNumRunnableApps());
+ assertEquals(1, targetQueue.getNumRunnableApps());
+ assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
+
+ scheduler.update();
+ assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand());
+ assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand());
+ }
+
+ @Test
+ public void testMoveNonRunnableApp() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+
+ assertEquals(0, oldQueue.getNumRunnableApps());
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ assertEquals(0, oldQueue.getNumRunnableApps());
+ assertEquals(0, targetQueue.getNumRunnableApps());
+ assertEquals(0, queueMgr.getRootQueue().getNumRunnableApps());
+ }
+
+ @Test
+ public void testMoveMakesAppRunnable() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+
+ FSSchedulerApp app = scheduler.getSchedulerApp(appAttId);
+ assertTrue(oldQueue.getNonRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ assertFalse(oldQueue.getNonRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertFalse(targetQueue.getNonRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertTrue(targetQueue.getRunnableAppSchedulables()
+ .contains(app.getAppSchedulable()));
+ assertEquals(1, targetQueue.getNumRunnableApps());
+ assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
+ }
+
+ @Test (expected = YarnException.class)
+ public void testMoveWouldViolateMaxAppsConstraints() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ }
+
+ @Test (expected = YarnException.class)
+ public void testMoveWouldViolateMaxResourcesConstraints() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
+ queueMgr.getLeafQueue("queue2", true);
+ scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2",
+ Resource.newInstance(1024, 1));
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+ RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.handle(updateEvent);
+ scheduler.handle(updateEvent);
+
+ assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ }
+
+ @Test (expected = YarnException.class)
+ public void testMoveToNonexistentQueue() throws Exception {
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ scheduler.getQueueManager().getLeafQueue("queue1", true);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1024, 1, "queue1", "user1", 3);
+ scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1565519&r1=1565518&r2=1565519&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Fri Feb 7 02:43:04 2014
@@ -77,7 +77,8 @@ public class TestMaxRunningAppsEnforcer
private void removeApp(FSSchedulerApp app) {
app.getQueue().removeApp(app);
- maxAppsEnforcer.updateRunnabilityOnAppRemoval(app);
+ maxAppsEnforcer.untrackRunnableApp(app);
+ maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue());
}
@Test
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1565519&r1=1565518&r2=1565519&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Fri Feb 7 02:43:04 2014
@@ -1317,8 +1317,8 @@ public class TestRMWebServicesApps exten
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- // 15 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 19, info.length());
+ // 20 because trackingUrl not assigned yet
+ assertEquals("incorrect number of elements", 20, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
info.getString("name"), info.getString("applicationType"), info.getString("queue"),
@@ -1389,7 +1389,7 @@ public class TestRMWebServicesApps exten
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- MockAM am = MockRM.launchAM(app1, rm, amNodeManager);
+ MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
int maxAppAttempts = rm.getConfig().getInt(
YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
@@ -1405,7 +1405,7 @@ public class TestRMWebServicesApps exten
}
// wait for app to start a new attempt.
rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
- am = MockRM.launchAM(app1, rm, amNodeManager);
+ am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
numAttempt++;
}
assertEquals("incorrect number of attempts", maxAppAttempts,
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1565519&r1=1565518&r2=1565519&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Feb 7 02:43:04 2014
@@ -349,16 +349,20 @@ Queue Access Control Lists (ACLs)
* {Administration}
- The fair scheduler provides support for administration at runtime through two mechanisms:
+ The fair scheduler provides support for administration at runtime through a few mechanisms:
- * It is possible to modify minimum shares, limits, weights, preemption timeouts
- and queue scheduling policies at runtime by editing the allocation file. The
- scheduler will reload this file 10-15 seconds after it sees that it was
- modified.
-
- * Current applications, queues, and fair shares can be examined through the
- ResourceManager's web interface, at
- http://<ResourceManager URL>/cluster/scheduler.
+Modifying configuration at runtime
+
+ It is possible to modify minimum shares, limits, weights, preemption timeouts
+ and queue scheduling policies at runtime by editing the allocation file. The
+ scheduler will reload this file 10-15 seconds after it sees that it was
+ modified.
+
+Monitoring through web UI
+
+ Current applications, queues, and fair shares can be examined through the
+ ResourceManager's web interface, at
+ http://<ResourceManager URL>/cluster/scheduler.
The following fields can be seen for each queue on the web interface:
@@ -382,3 +386,17 @@ Queue Access Control Lists (ACLs)
In addition to the information that the ResourceManager normally displays
about each application, the web interface includes the application's fair share.
+Moving applications between queues
+
+ The Fair Scheduler supports moving a running application to a different queue.
+ This can be useful for moving an important application to a higher priority
+ queue, or for moving an unimportant application to a lower priority queue.
+ Apps can be moved by running "yarn application -movetoqueue appID -queue
+ targetQueueName".
+
+ When an application is moved to a queue, its existing allocations become
+ counted with the new queue's allocations instead of the old for purposes
+ of determining fairness. An attempt to move an application to a queue will
+ fail if the addition of the app's resources to that queue would violate the
+ its maxRunningApps or maxResources constraints.
+
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1565519&r1=1565518&r2=1565519&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Fri Feb 7 02:43:04 2014
@@ -1123,6 +1123,7 @@ ResourceManager REST API's.
* finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch
* finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
* applicationTypes - applications matching the given application types, specified as a comma-separated list.
+ * applicationTags - applications matching any of the given application tags, specified as a comma-separated list.
------
** Elements of the <apps> (Applications) object