You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2021/03/17 12:38:39 UTC
[hadoop] branch trunk updated: YARN-10497. Fix an issue in
CapacityScheduler which fails to delete queues. Contributed by Wangda Tan
and Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3e58d56 YARN-10497. Fix an issue in CapacityScheduler which fails to delete queues. Contributed by Wangda Tan and Qi Zhu.
3e58d56 is described below
commit 3e58d5611d2426f576e667ceccc5c8f64c9699e2
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Wed Mar 17 13:38:20 2021 +0100
YARN-10497. Fix an issue in CapacityScheduler which fails to delete queues. Contributed by Wangda Tan and Qi Zhu.
---
.../conf/MutableCSConfigurationProvider.java | 12 ++++++--
.../conf/TestMutableCSConfigurationProvider.java | 33 ++++++++++++++++++++++
2 files changed, 43 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 5f262be..b74414b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -68,10 +68,18 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
this.rmContext = rmContext;
}
+ // Unit test can overwrite this method
+ protected Configuration getInitSchedulerConfig() {
+ Configuration initialSchedConf = new Configuration(false);
+ initialSchedConf.
+ addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
+ return initialSchedConf;
+ }
+
@Override
public void init(Configuration config) throws IOException {
this.confStore = YarnConfigurationStoreFactory.getStore(config);
- Configuration initialSchedConf = new Configuration(false);
+ Configuration initialSchedConf = getInitSchedulerConfig();
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
// We need to explicitly set the key-values in schedConf, otherwise
@@ -231,7 +239,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
parentQueue + CapacitySchedulerConfiguration.DOT +
CapacitySchedulerConfiguration.QUEUES;
- return new ArrayList<>(conf.getStringCollection(childQueuesKey));
+ return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey));
}
private Map<String, String> constructKeyValueConfUpdate(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 0c9a312..aec2c21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -217,6 +217,39 @@ public class TestMutableCSConfigurationProvider {
}
+ @Test
+ public void testAddRemoveQueueWithSpacesInConfig() throws Exception {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] {" a , b, c" });
+
+ final String a = CapacitySchedulerConfiguration.ROOT + ".a";
+ final String b = CapacitySchedulerConfiguration.ROOT + ".b";
+ final String c = CapacitySchedulerConfiguration.ROOT + ".c";
+ csConf.setCapacity(a, 0);
+ csConf.setCapacity(b, 50);
+ csConf.setCapacity(c, 50);
+
+ confProvider = new MutableCSConfigurationProvider(rmContext) {
+ @Override
+ protected Configuration getInitSchedulerConfig() {
+ return csConf;
+ }
+ };
+
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+ confProvider.init(conf);
+
+ SchedConfUpdateInfo update = new SchedConfUpdateInfo();
+ update.getRemoveQueueInfo().add("root.a");
+
+ confProvider.logAndApplyMutation(UserGroupInformation
+ .getCurrentUser(), update);
+ }
+
private void writeConf(Configuration conf, String storePath)
throws IOException {
FileSystem fileSystem = FileSystem.get(new Configuration(conf));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org