You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2012/02/26 07:53:44 UTC
svn commit: r1293753 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/had...
Author: vinodkv
Date: Sun Feb 26 06:53:44 2012
New Revision: 1293753
URL: http://svn.apache.org/viewvc?rev=1293753&view=rev
Log:
MAPREDUCE-3910. Fixed a bug in CapacityScheduler LeafQueue which was causing app-submission to fail. Contributed by John George.
svn merge --ignore-ancestry -c 1293750 ../../trunk
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1293753&r1=1293752&r2=1293753&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Sun Feb 26 06:53:44 2012
@@ -77,6 +77,9 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-3904 Job history produced with mapreduce.cluster.acls.enabled
false can not be viewed with mapreduce.cluster.acls.enabled true
(Jonathon Eagles via tgraves)
+
+ MAPREDUCE-3910. Fixed a bug in CapacityScheduler LeafQueue which was causing
+ app-submission to fail. (John George via vinodkv)
Release 0.23.1 - 2012-02-17
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1293753&r1=1293752&r2=1293753&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Sun Feb 26 06:53:44 2012
@@ -572,12 +572,7 @@ public class LeafQueue implements CSQueu
// Careful! Locking order is important!
// Check queue ACLs
- UserGroupInformation userUgi;
- try {
- userUgi = UserGroupInformation.getCurrentUser();
- } catch (IOException ioe) {
- throw new AccessControlException(ioe);
- }
+ UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName);
if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) {
throw new AccessControlException("User " + userName + " cannot submit" +
" applications to queue " + getQueuePath());
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1293753&r1=1293752&r2=1293753&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Sun Feb 26 06:53:44 2012
@@ -119,10 +119,11 @@ public class TestLeafQueue {
private static final String B = "b";
private static final String C = "c";
private static final String C1 = "c1";
+ private static final String D = "d";
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C});
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
@@ -133,7 +134,7 @@ public class TestLeafQueue {
conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
- conf.setCapacity(Q_B, 90);
+ conf.setCapacity(Q_B, 80);
conf.setMaximumCapacity(Q_B, 99);
conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
@@ -146,6 +147,11 @@ public class TestLeafQueue {
final String Q_C1 = Q_C + "." + C1;
conf.setCapacity(Q_C1, 100);
+
+ final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
+ conf.setCapacity(Q_D, 10);
+ conf.setMaximumCapacity(Q_D, 11);
+ conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d");
}
@@ -202,8 +208,8 @@ public class TestLeafQueue {
assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
- assertEquals(0.9, b.getCapacity(), epsilon);
- assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.80, b.getCapacity(), epsilon);
+ assertEquals(0.80, b.getAbsoluteCapacity(), epsilon);
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
@@ -257,10 +263,35 @@ public class TestLeafQueue {
// Only 1 container
a.assignContainers(clusterResource, node_0);
- assertEquals(7*GB, a.getMetrics().getAvailableMB());
+ assertEquals(6*GB, a.getMetrics().getAvailableMB());
}
@Test
+ public void testUserQueueAcl() throws Exception {
+
+ // Manipulate queue 'a'
+ LeafQueue d = stubLeafQueue((LeafQueue) queues.get(D));
+
+ // Users
+ final String user_d = "user_d";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId_0 = TestUtils
+ .getMockApplicationAttemptId(0, 1);
+ SchedulerApp app_0 = new SchedulerApp(appAttemptId_0, user_d, d, null,
+ rmContext, null);
+ d.submitApplication(app_0, user_d, D);
+
+ // Attempt the same application again
+ final ApplicationAttemptId appAttemptId_1 = TestUtils
+ .getMockApplicationAttemptId(0, 2);
+ SchedulerApp app_1 = new SchedulerApp(appAttemptId_1, user_d, d, null,
+ rmContext, null);
+ d.submitApplication(app_1, user_d, D); // same user
+ }
+
+
+ @Test
public void testAppAttemptMetrics() throws Exception {
// Manipulate queue 'a'