You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2014/12/09 04:31:23 UTC
[29/41] hadoop git commit: YARN-2869. CapacityScheduler should trim
sub queue names when parse configuration. Contributed by Wangda Tan
YARN-2869. CapacityScheduler should trim sub queue names when parse configuration. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e69af836
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e69af836
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e69af836
Branch: refs/heads/YARN-2139
Commit: e69af836f34f16fba565ab112c9bf0d367675b16
Parents: 475c6b4
Author: Jian He <ji...@apache.org>
Authored: Fri Dec 5 17:33:39 2014 -0800
Committer: Jian He <ji...@apache.org>
Committed: Fri Dec 5 17:33:39 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../CapacitySchedulerConfiguration.java | 10 +-
.../scheduler/capacity/TestQueueParsing.java | 110 +++++++++++++++++++
3 files changed, 122 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0b88959..0d7a843 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -194,6 +194,9 @@ Release 2.7.0 - UNRELEASED
YARN-2461. Fix PROCFS_USE_SMAPS_BASED_RSS_ENABLED property in
YarnConfiguration. (rchiang via rkanter)
+ YARN-2869. CapacityScheduler should trim sub queue names when parse
+ configuration. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 0a49224..5bbb436 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -260,7 +260,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
- private String getQueuePrefix(String queue) {
+ static String getQueuePrefix(String queue) {
String queueName = PREFIX + queue + DOT;
return queueName;
}
@@ -538,6 +538,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public String[] getQueues(String queue) {
LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
+ List<String> trimmedQueueNames = new ArrayList<String>();
+ if (null != queues) {
+ for (String s : queues) {
+ trimmedQueueNames.add(s.trim());
+ }
+ queues = trimmedQueueNames.toArray(new String[0]);
+ }
+
LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) +
", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues)));
return queues;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index cf2e5ce..5a9fbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -82,6 +82,56 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(capacityScheduler);
}
+ private void setupQueueConfigurationWithSpacesShouldBeTrimmed(
+ CapacitySchedulerConfiguration conf) {
+ // Define top-level queues
+ conf.set(
+ CapacitySchedulerConfiguration
+ .getQueuePrefix(CapacitySchedulerConfiguration.ROOT)
+ + CapacitySchedulerConfiguration.QUEUES, " a ,b, c");
+
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ conf.setCapacity(A, 10);
+ conf.setMaximumCapacity(A, 15);
+
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ conf.setCapacity(B, 20);
+
+ final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+ conf.setCapacity(C, 70);
+ conf.setMaximumCapacity(C, 70);
+ }
+
+ private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed(
+ CapacitySchedulerConfiguration conf) {
+ // Define top-level queues
+ conf.set(
+ CapacitySchedulerConfiguration
+ .getQueuePrefix(CapacitySchedulerConfiguration.ROOT)
+ + CapacitySchedulerConfiguration.QUEUES, " a ,b, c");
+
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ conf.setCapacity(A, 10);
+ conf.setMaximumCapacity(A, 15);
+
+ final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ conf.setCapacity(B, 20);
+
+ final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+ conf.setCapacity(C, 70);
+ conf.setMaximumCapacity(C, 70);
+
+ // sub queues for A
+ conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A)
+ + CapacitySchedulerConfiguration.QUEUES, "a1, a2 ");
+
+ final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1";
+ conf.setCapacity(A1, 60);
+
+ final String A2 = CapacitySchedulerConfiguration.ROOT + ".a.a2";
+ conf.setCapacity(A2, 40);
+ }
+
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
@@ -659,4 +709,64 @@ public class TestQueueParsing {
DELTA);
capacityScheduler.stop();
}
+
+ @Test
+ public void testQueueParsingShouldTrimSpaces() throws Exception {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ setupQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ capacityScheduler.setConf(conf);
+ capacityScheduler.setRMContext(TestUtils.getMockRMContext());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
+ capacityScheduler.reinitialize(conf, TestUtils.getMockRMContext());
+
+ CSQueue a = capacityScheduler.getQueue("a");
+ Assert.assertNotNull(a);
+ Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA);
+
+ CSQueue c = capacityScheduler.getQueue("c");
+ Assert.assertNotNull(c);
+ Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA);
+ }
+
+ @Test
+ public void testNestedQueueParsingShouldTrimSpaces() throws Exception {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ setupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ capacityScheduler.setConf(conf);
+ capacityScheduler.setRMContext(TestUtils.getMockRMContext());
+ capacityScheduler.init(conf);
+ capacityScheduler.start();
+ capacityScheduler.reinitialize(conf, TestUtils.getMockRMContext());
+
+ CSQueue a = capacityScheduler.getQueue("a");
+ Assert.assertNotNull(a);
+ Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA);
+
+ CSQueue c = capacityScheduler.getQueue("c");
+ Assert.assertNotNull(c);
+ Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA);
+
+ CSQueue a1 = capacityScheduler.getQueue("a1");
+ Assert.assertNotNull(a1);
+ Assert.assertEquals(0.10 * 0.6, a1.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals(0.15, a1.getAbsoluteMaximumCapacity(), DELTA);
+
+ CSQueue a2 = capacityScheduler.getQueue("a2");
+ Assert.assertNotNull(a2);
+ Assert.assertEquals(0.10 * 0.4, a2.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals(0.15, a2.getAbsoluteMaximumCapacity(), DELTA);
+ }
}