You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2012/02/02 03:31:06 UTC
svn commit: r1239443 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server...
Author: acmurthy
Date: Thu Feb 2 02:31:05 2012
New Revision: 1239443
URL: http://svn.apache.org/viewvc?rev=1239443&view=rev
Log:
MAPREDUCE-3780. Fixed a bug where applications killed before getting activated were not getting cleaned up properly. Contributed by Hitesh Shah.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1239443&r1=1239442&r2=1239443&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Feb 2 02:31:05 2012
@@ -657,6 +657,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3744. Fix the yarn logs command line. Improve error messages for
mapred job -logs. (Jason Lowe via sseth)
+ MAPREDUCE-3780. Fixed a bug where applications killed before getting
+ activated were not getting cleaned up properly. (Hitesh Shah via acmurthy)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1239443&r1=1239442&r2=1239443&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Feb 2 02:31:05 2012
@@ -635,7 +635,7 @@ public class LeafQueue implements CSQueu
user.activateApplication();
activeApplications.add(application);
i.remove();
- LOG.info("Application " + application.getApplicationId().getId() +
+ LOG.info("Application " + application.getApplicationId() +
" from user: " + application.getUser() +
" activated in queue: " + getQueueName());
}
@@ -673,10 +673,13 @@ public class LeafQueue implements CSQueu
}
public synchronized void removeApplication(SchedulerApp application, User user) {
- activeApplications.remove(application);
+ boolean wasActive = activeApplications.remove(application);
+ if (!wasActive) {
+ pendingApplications.remove(application);
+ }
applicationsMap.remove(application.getApplicationAttemptId());
- user.finishApplication();
+ user.finishApplication(wasActive);
if (user.getTotalApplications() == 0) {
users.remove(application.getUser());
}
@@ -1415,8 +1418,13 @@ public class LeafQueue implements CSQueu
++activeApplications;
}
- public synchronized void finishApplication() {
- --activeApplications;
+ public synchronized void finishApplication(boolean wasActive) {
+ if (wasActive) {
+ --activeApplications;
+ }
+ else {
+ --pendingApplications;
+ }
}
public synchronized void assignContainer(Resource resource) {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1239443&r1=1239442&r2=1239443&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java Thu Feb 2 02:31:05 2012
@@ -257,6 +257,87 @@ public class TestApplicationLimits {
}
@Test
+ public void testActiveLimitsWithKilledApps() throws Exception {
+ final String user_0 = "user_0";
+
+ int APPLICATION_ID = 0;
+
+ // set max active to 2
+ doReturn(2).when(queue).getMaximumActiveApplications();
+
+ // Submit first application
+ SchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0);
+ queue.submitApplication(app_0, user_0, A);
+ assertEquals(1, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(1, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+ assertTrue(queue.activeApplications.contains(app_0));
+
+ // Submit second application
+ SchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0);
+ queue.submitApplication(app_1, user_0, A);
+ assertEquals(2, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(2, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+ assertTrue(queue.activeApplications.contains(app_1));
+
+ // Submit third application, should remain pending
+ SchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0);
+ queue.submitApplication(app_2, user_0, A);
+ assertEquals(2, queue.getNumActiveApplications());
+ assertEquals(1, queue.getNumPendingApplications());
+ assertEquals(2, queue.getNumActiveApplications(user_0));
+ assertEquals(1, queue.getNumPendingApplications(user_0));
+ assertTrue(queue.pendingApplications.contains(app_2));
+
+ // Submit fourth application, should remain pending
+ SchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0);
+ queue.submitApplication(app_3, user_0, A);
+ assertEquals(2, queue.getNumActiveApplications());
+ assertEquals(2, queue.getNumPendingApplications());
+ assertEquals(2, queue.getNumActiveApplications(user_0));
+ assertEquals(2, queue.getNumPendingApplications(user_0));
+ assertTrue(queue.pendingApplications.contains(app_3));
+
+ // Kill 3rd pending application
+ queue.finishApplication(app_2, A);
+ assertEquals(2, queue.getNumActiveApplications());
+ assertEquals(1, queue.getNumPendingApplications());
+ assertEquals(2, queue.getNumActiveApplications(user_0));
+ assertEquals(1, queue.getNumPendingApplications(user_0));
+ assertFalse(queue.pendingApplications.contains(app_2));
+ assertFalse(queue.activeApplications.contains(app_2));
+
+ // Finish 1st application, app_3 should become active
+ queue.finishApplication(app_0, A);
+ assertEquals(2, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(2, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+ assertTrue(queue.activeApplications.contains(app_3));
+ assertFalse(queue.pendingApplications.contains(app_3));
+ assertFalse(queue.activeApplications.contains(app_0));
+
+ // Finish 2nd application
+ queue.finishApplication(app_1, A);
+ assertEquals(1, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(1, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+ assertFalse(queue.activeApplications.contains(app_1));
+
+ // Finish 4th application
+ queue.finishApplication(app_3, A);
+ assertEquals(0, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(0, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+ assertFalse(queue.activeApplications.contains(app_3));
+ }
+
+ @Test
public void testHeadroom() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();