You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/12/13 01:20:02 UTC
svn commit: r1213511 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server...
Author: mahadev
Date: Tue Dec 13 00:20:01 2011
New Revision: 1213511
URL: http://svn.apache.org/viewvc?rev=1213511&view=rev
Log:
MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue -showacls (Jonathan Eagles via mahadev)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1213511&r1=1213510&r2=1213511&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Dec 13 00:20:01 2011
@@ -270,6 +270,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3328. mapred queue -list output inconsistent and missing child
queues. (Ravi Prakash via mahadev)
+ MAPREDUCE-3510. Capacity Scheduler inherited ACLs not displayed by mapred queue
+ -showacls (Jonathan Eagles via mahadev)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1213511&r1=1213510&r2=1213511&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Tue Dec 13 00:20:01 2011
@@ -198,19 +198,33 @@ public class CapacitySchedulerConfigurat
private static String getAclKey(QueueACL acl) {
return "acl_" + acl.toString().toLowerCase();
}
-
+
+ public AccessControlList getAcl(String queue, QueueACL acl) {
+ String queuePrefix = getQueuePrefix(queue);
+ String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL);
+ return new AccessControlList(aclString);
+ }
+
+ public void setAcl(String queue, QueueACL acl, String aclString) {
+ String queuePrefix = getQueuePrefix(queue);
+ set(queuePrefix + getAclKey(acl), aclString);
+ }
+
public Map<QueueACL, AccessControlList> getAcls(String queue) {
- Map<QueueACL, AccessControlList> acls =
+ Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
- String queuePrefix = getQueuePrefix(queue);
for (QueueACL acl : QueueACL.values()) {
- acls.put(acl,
- new AccessControlList(get(queuePrefix + getAclKey(acl),
- DEFAULT_ACL)));
+ acls.put(acl, getAcl(queue, acl));
}
return acls;
}
+ public void setAcls(String queue, Map<QueueACL, AccessControlList> acls) {
+ for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
+ setAcl(queue, e.getKey(), e.getValue().getAclString());
+ }
+ }
+
public String[] getQueues(String queue) {
LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1213511&r1=1213510&r2=1213511&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Dec 13 00:20:01 2011
@@ -492,11 +492,8 @@ public class LeafQueue implements CSQueu
QueueUserACLInfo userAclInfo =
recordFactory.newRecordInstance(QueueUserACLInfo.class);
List<QueueACL> operations = new ArrayList<QueueACL>();
- for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
- QueueACL operation = e.getKey();
- AccessControlList acl = e.getValue();
-
- if (acl.isUserAllowed(user)) {
+ for (QueueACL operation : QueueACL.values()) {
+ if (hasAccess(operation, user)) {
operations.add(operation);
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1213511&r1=1213510&r2=1213511&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Dec 13 00:20:01 2011
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -30,11 +31,14 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -102,20 +106,29 @@ public class TestLeafQueue {
private static final String A = "a";
private static final String B = "b";
+ private static final String C = "c";
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
+ conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C});
conf.setCapacity(CapacityScheduler.ROOT, 100);
conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
+ conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
final String Q_A = CapacityScheduler.ROOT + "." + A;
- conf.setCapacity(Q_A, 10);
+ conf.setCapacity(Q_A, 9);
conf.setMaximumCapacity(Q_A, 20);
+ conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
final String Q_B = CapacityScheduler.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
conf.setMaximumCapacity(Q_B, 99);
+ conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
+
+ final String Q_C = CapacityScheduler.ROOT + "." + C;
+ conf.setCapacity(Q_C, 1);
+ conf.setMaximumCapacity(Q_C, 10);
+ conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
LOG.info("Setup top-level queues a and b");
}
@@ -167,8 +180,8 @@ public class TestLeafQueue {
//can add more sturdy test with 3-layer queues
//once MAPREDUCE:3410 is resolved
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
- assertEquals(0.1, a.getCapacity(), epsilon);
- assertEquals(0.1, a.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.09, a.getCapacity(), epsilon);
+ assertEquals(0.09, a.getAbsoluteCapacity(), epsilon);
assertEquals(0.2, a.getMaximumCapacity(), epsilon);
assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
@@ -177,6 +190,12 @@ public class TestLeafQueue {
assertEquals(0.9, b.getAbsoluteCapacity(), epsilon);
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+
+ LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+ assertEquals(0.01, c.getCapacity(), epsilon);
+ assertEquals(0.01, c.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.1, c.getMaximumCapacity(), epsilon);
+ assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
}
@Test
@@ -1082,6 +1101,37 @@ public class TestLeafQueue {
assertEquals(0, app_0.getTotalRequiredResources(priority));
}
+
+ public boolean hasQueueACL(List<QueueUserACLInfo> aclInfos, QueueACL acl) {
+ for (QueueUserACLInfo aclInfo : aclInfos) {
+ if (aclInfo.getUserAcls().contains(acl)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void testInheritedQueueAcls() throws IOException {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+ LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+ LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+
+ assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+
+ assertTrue(hasQueueACL(
+ a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+ assertTrue(hasQueueACL(
+ b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+ assertFalse(hasQueueACL(
+ c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+
+ }
@After
public void tearDown() throws Exception {