You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/12/29 20:06:35 UTC
svn commit: r1225618 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server...
Author: acmurthy
Date: Thu Dec 29 19:06:34 2011
New Revision: 1225618
URL: http://svn.apache.org/viewvc?rev=1225618&view=rev
Log:
MAPREDUCE-3522. Ensure queues inherit ACLs from parent if they aren't explicitly specified. Contributed by Jonathan Eagles.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Dec 29 19:06:34 2011
@@ -378,6 +378,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3521. Fixed streaming to ensure it doesn't silently ignore
unknown arguments. (Robert Evans via acmurthy)
+ MAPREDUCE-3522. Ensure queues inherit ACLs from parent if they aren't
+ explicitly specified. (Jonathan Eagles via acmurthy)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Thu Dec 29 19:06:34 2011
@@ -201,11 +201,8 @@ implements ResourceScheduler, CapacitySc
}
@Private
- public static final String ROOT = "root";
-
- @Private
public static final String ROOT_QUEUE =
- CapacitySchedulerConfiguration.PREFIX + ROOT;
+ CapacitySchedulerConfiguration.PREFIX + CapacitySchedulerConfiguration.ROOT;
static class QueueHook {
public CSQueue hook(CSQueue queue) {
@@ -218,7 +215,7 @@ implements ResourceScheduler, CapacitySc
private void initializeQueues(CapacitySchedulerConfiguration conf)
throws IOException {
root =
- parseQueue(this, conf, null, ROOT, queues, queues,
+ parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, queues, queues,
queueComparator, applicationComparator, noop);
LOG.info("Initialized root queue " + root);
}
@@ -229,7 +226,7 @@ implements ResourceScheduler, CapacitySc
// Parse new queues
Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
CSQueue newRoot =
- parseQueue(this, conf, null, ROOT, newQueues, queues,
+ parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
queueComparator, applicationComparator, noop);
// Ensure all existing queues are still present
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Thu Dec 29 19:06:34 2011
@@ -104,14 +104,20 @@ public class CapacitySchedulerConfigurat
@Private
public static final float DEFAULT_USER_LIMIT_FACTOR = 1.0f;
-
+
+ @Private
+ public static final String ALL_ACL = "*";
+
@Private
- public static final String DEFAULT_ACL = "*";
+ public static final String NONE_ACL = " ";
@Private public static final String ENABLE_USER_METRICS =
PREFIX +"user-metrics.enable";
@Private public static final boolean DEFAULT_ENABLE_USER_METRICS = false;
+ @Private
+ public static final String ROOT = "root";
+
public CapacitySchedulerConfiguration() {
this(new Configuration());
}
@@ -143,14 +149,14 @@ public class CapacitySchedulerConfigurat
throw new IllegalArgumentException("Illegal " +
"capacity of " + capacity + " for queue " + queue);
}
- LOG.info("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
+ LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
", capacity=" + capacity);
return capacity;
}
public void setCapacity(String queue, int capacity) {
setInt(getQueuePrefix(queue) + CAPACITY, capacity);
- LOG.info("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
+ LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
", capacity=" + capacity);
}
@@ -162,7 +168,7 @@ public class CapacitySchedulerConfigurat
public void setMaximumCapacity(String queue, int maxCapacity) {
setInt(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
- LOG.info("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
+ LOG.debug("CSConf - setMaxCapacity: queuePrefix=" + getQueuePrefix(queue) +
", maxCapacity=" + maxCapacity);
}
@@ -174,7 +180,7 @@ public class CapacitySchedulerConfigurat
public void setUserLimit(String queue, int userLimit) {
setInt(getQueuePrefix(queue) + USER_LIMIT, userLimit);
- LOG.info("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) +
+ LOG.debug("here setUserLimit: queuePrefix=" + getQueuePrefix(queue) +
", userLimit=" + getUserLimit(queue));
}
@@ -201,7 +207,10 @@ public class CapacitySchedulerConfigurat
public AccessControlList getAcl(String queue, QueueACL acl) {
String queuePrefix = getQueuePrefix(queue);
- String aclString = get(queuePrefix + getAclKey(acl), DEFAULT_ACL);
+ // The root queue defaults to all access if not defined
+ // Sub queues inherit access if not defined
+ String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL;
+ String aclString = get(queuePrefix + getAclKey(acl), defaultAcl);
return new AccessControlList(aclString);
}
@@ -226,16 +235,16 @@ public class CapacitySchedulerConfigurat
}
public String[] getQueues(String queue) {
- LOG.info("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
+ LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
- LOG.info("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) +
+ LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) +
", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues)));
return queues;
}
public void setQueues(String queue, String[] subQueues) {
set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues));
- LOG.info("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) +
+ LOG.debug("CSConf - setQueues: qPrefix=" + getQueuePrefix(queue) +
", queues=" + StringUtils.arrayToString(subQueues));
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java Thu Dec 29 19:06:34 2011
@@ -88,13 +88,13 @@ public class TestApplicationLimits {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String Q_A = CapacityScheduler.ROOT + "." + A;
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 10);
- final String Q_B = CapacityScheduler.ROOT + "." + B;
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
LOG.info("Setup top-level queues a and b");
@@ -252,7 +252,7 @@ public class TestApplicationLimits {
public void testHeadroom() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setUserLimit(CapacityScheduler.ROOT + "." + A, 25);
+ csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25);
setupQueueConfiguration(csConf);
CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Thu Dec 29 19:06:34 2011
@@ -197,13 +197,13 @@ public class TestCapacityScheduler {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b"});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String A = CapacityScheduler.ROOT + ".a";
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
- final String B = CapacityScheduler.ROOT + ".b";
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 90);
// Define 2nd-level queues
@@ -250,9 +250,9 @@ public class TestCapacityScheduler {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
- conf.setQueues(CapacityScheduler.ROOT + ".a.a1", new String[] {"b1"} );
- conf.setCapacity(CapacityScheduler.ROOT + ".a.a1.b1", 100);
- conf.setUserLimitFactor(CapacityScheduler.ROOT + ".a.a1.b1", 100.0f);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100);
+ conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
cs.reinitialize(conf, null, null);
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Thu Dec 29 19:06:34 2011
@@ -107,29 +107,35 @@ public class TestLeafQueue {
private static final String A = "a";
private static final String B = "b";
private static final String C = "c";
+ private static final String C1 = "c1";
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
- conf.setMaximumCapacity(CapacityScheduler.ROOT, 100);
- conf.setAcl(CapacityScheduler.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
+ conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
+ conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
- final String Q_A = CapacityScheduler.ROOT + "." + A;
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 9);
conf.setMaximumCapacity(Q_A, 20);
conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
- final String Q_B = CapacityScheduler.ROOT + "." + B;
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
conf.setMaximumCapacity(Q_B, 99);
conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
- final String Q_C = CapacityScheduler.ROOT + "." + C;
+ final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
conf.setCapacity(Q_C, 1);
conf.setMaximumCapacity(Q_C, 10);
conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
+ conf.setQueues(Q_C, new String[] {C1});
+
+ final String Q_C1 = Q_C + "." + C1;
+ conf.setCapacity(Q_C1, 100);
+
LOG.info("Setup top-level queues a and b");
}
@@ -191,7 +197,7 @@ public class TestLeafQueue {
assertEquals(0.99, b.getMaximumCapacity(), epsilon);
assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
- LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+ ParentQueue c = (ParentQueue)queues.get(C);
assertEquals(0.01, c.getCapacity(), epsilon);
assertEquals(0.01, c.getAbsoluteCapacity(), epsilon);
assertEquals(0.1, c.getMaximumCapacity(), epsilon);
@@ -1117,12 +1123,14 @@ public class TestLeafQueue {
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
- LeafQueue c = stubLeafQueue((LeafQueue)queues.get(C));
+ ParentQueue c = (ParentQueue)queues.get(C);
+ LeafQueue c1 = stubLeafQueue((LeafQueue)queues.get(C1));
assertFalse(root.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertTrue(a.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertTrue(b.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertFalse(c.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
+ assertFalse(c1.hasAccess(QueueACL.SUBMIT_APPLICATIONS, user));
assertTrue(hasQueueACL(
a.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
@@ -1130,6 +1138,8 @@ public class TestLeafQueue {
b.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
assertFalse(hasQueueACL(
c.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
+ assertFalse(hasQueueACL(
+ c1.getQueueUserAclInfo(user), QueueACL.SUBMIT_APPLICATIONS));
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Thu Dec 29 19:06:34 2011
@@ -70,13 +70,13 @@ public class TestParentQueue {
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String Q_A = CapacityScheduler.ROOT + "." + A;
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
- final String Q_B = CapacityScheduler.ROOT + "." + B;
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -144,7 +144,7 @@ public class TestParentQueue {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
- CapacityScheduler.ROOT, queues, queues,
+ CapacitySchedulerConfiguration.ROOT, queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
@@ -246,19 +246,19 @@ public class TestParentQueue {
private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {A, B, C, D});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String Q_A = CapacityScheduler.ROOT + "." + A;
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 10);
- final String Q_B = CapacityScheduler.ROOT + "." + B;
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 50);
- final String Q_C = CapacityScheduler.ROOT + "." + C;
+ final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
conf.setCapacity(Q_C, 20);
- final String Q_D = CapacityScheduler.ROOT + "." + D;
+ final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
conf.setCapacity(Q_D, 20);
// Define 2-nd level queues
@@ -282,7 +282,7 @@ public class TestParentQueue {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue root =
CapacityScheduler.parseQueue(csContext, csConf, null,
- CapacityScheduler.ROOT, queues, queues,
+ CapacitySchedulerConfiguration.ROOT, queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
TestUtils.spyHook);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java Thu Dec 29 19:06:34 2011
@@ -42,16 +42,16 @@ public class TestQueueParsing {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String A = CapacityScheduler.ROOT + ".a";
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
- final String B = CapacityScheduler.ROOT + ".b";
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
- final String C = CapacityScheduler.ROOT + ".c";
+ final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
LOG.info("Setup top-level queues");
@@ -100,7 +100,7 @@ public class TestQueueParsing {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
// non-100 percent value will throw IllegalArgumentException
- conf.setCapacity(CapacityScheduler.ROOT, 90);
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.reinitialize(conf, null, null);
@@ -109,14 +109,14 @@ public class TestQueueParsing {
public void testMaxCapacity() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String A = CapacityScheduler.ROOT + ".a";
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 50);
conf.setMaximumCapacity(A, 60);
- final String B = CapacityScheduler.ROOT + ".b";
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 50);
conf.setMaximumCapacity(B, 45); // Should throw an exception
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Thu Dec 29 19:06:34 2011
@@ -155,16 +155,16 @@ public class TestRMWebApp {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacityScheduler.ROOT, new String[] {"a", "b", "c"});
- conf.setCapacity(CapacityScheduler.ROOT, 100);
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
- final String A = CapacityScheduler.ROOT + ".a";
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
- final String B = CapacityScheduler.ROOT + ".b";
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
- final String C = CapacityScheduler.ROOT + ".c";
+ final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
// Define 2nd-level queues
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1225618&r1=1225617&r2=1225618&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Thu Dec 29 19:06:34 2011
@@ -161,7 +161,7 @@ Hadoop MapReduce Next Generation - Capac
A given queue's children can be defined with the configuration knob:
<<<yarn.scheduler.capacity.<queue-path>.queues>>>. Children do not
- inherit properties directly from the parent.
+ inherit properties directly from the parent unless otherwise noted.
Here is an example with three top-level child-queues <<<a>>>, <<<b>>> and
<<<c>>> and some sub-queues for <<<a>>> and <<<b>>>:
@@ -274,17 +274,21 @@ Hadoop MapReduce Next Generation - Capac
| <<<yarn.scheduler.capacity.root.<queue-path>.acl_submit_jobs>>> | |
| | The <ACL> which controls who can <submit> jobs to the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
-| | <one of the parent queues in the hierarchy> they can submit jobs. |
+| | <one of the parent queues in the hierarchy> they can submit jobs. |
+| | <ACLs> for this property <are> inherited from the parent queue |
+| | if not specified. |
*--------------------------------------+--------------------------------------+
| <<<yarn.scheduler.capacity.root.<queue-path>.acl_administer_jobs>>> | |
| | The <ACL> which controls who can <administer> jobs on the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
-| | <one of the parent queues in the hierarchy> they can administer jobs. |
+| | <one of the parent queues in the hierarchy> they can administer jobs. |
+| | <ACLs> for this property <are> inherited from the parent queue |
+| | if not specified. |
*--------------------------------------+--------------------------------------+
<Note:> An <ACL> is of the form <user1>, <user2><space><group1>, <group2>.
The special value of <<*>> implies <anyone>. The special value of <space>
- implies <no one>. The default is <<*>> if not specified.
+ implies <no one>. The default is <<*>> for the root queue if not specified.
* Reviewing the configuration of the CapacityScheduler