You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2020/02/26 09:44:42 UTC
[hadoop] branch trunk updated: YARN-9593. Support Comma in the
value of Scheduler Configuration Mutation API.
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b5df1da YARN-9593. Support Comma in the value of Scheduler Configuration Mutation API.
b5df1da is described below
commit b5df1da3a12dc70367ecee503e7fb90b4bac3ac3
Author: tanu.ajmera <ta...@cloudera.com>
AuthorDate: Thu Feb 20 15:24:07 2020 +0530
YARN-9593. Support Comma in the value of Scheduler Configuration Mutation API.
---
.../hadoop/yarn/client/cli/SchedConfCLI.java | 16 +++++++--
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 39 ++++++++++++++++++++++
2 files changed, 53 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index 2f2b601..6460de9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -79,6 +79,7 @@ public class SchedConfCLI extends Configured implements Tool {
private static final String GET_SCHEDULER_CONF = "getConf";
private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
+ private static final String SPLIT_BY_SLASH_COMMA = "(?<!\\\\)\\,";
private static final String CONF_ERR_MSG = "Specify configuration key " +
"value as confKey=confVal.";
@@ -410,7 +411,8 @@ public class SchedConfCLI extends Configured implements Tool {
return;
}
HashMap<String, String> globalUpdates = new HashMap<>();
- for (String globalUpdate : args.split(",")) {
+ for (String globalUpdate : args.split(SPLIT_BY_SLASH_COMMA)) {
+ globalUpdate = globalUpdate.replace("\\", "");
putKeyValuePair(globalUpdates, globalUpdate);
}
updateInfo.setGlobalParams(globalUpdates);
@@ -421,8 +423,9 @@ public class SchedConfCLI extends Configured implements Tool {
String queuePath = args[0];
Map<String, String> queueConfigs = new HashMap<>();
if (args.length > 1) {
- String[] queueArgs = args[1].split(",");
+ String[] queueArgs = args[1].split(SPLIT_BY_SLASH_COMMA);
for (int i = 0; i < queueArgs.length; ++i) {
+ queueArgs[i] = queueArgs[i].replace("\\", "");
putKeyValuePair(queueConfigs, queueArgs[i]);
}
}
@@ -461,13 +464,22 @@ public class SchedConfCLI extends Configured implements Tool {
+ "Example (adding queues): yarn schedulerconf -add "
+ "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ "maximum-capacity=0\"\n"
+ + "Example (adding queues with comma in value): yarn schedulerconf "
+ + "-add \"root.default:acl_administer_queue=user1\\,user2 group1\\,"
+ + "group2,maximum-capacity=100;root.a.a2:capacity=0\"\n"
+ "Example (removing queues): yarn schedulerconf -remove \"root.a.a1;"
+ "root.a.a2\"\n"
+ "Example (updating queues): yarn schedulerconf -update \"root.a.a1"
+ ":capacity=25,maximum-capacity=25;root.a.a2:capacity=75,"
+ "maximum-capacity=75\"\n"
+ + "Example (updating queues with comma in value): yarn schedulerconf "
+ + "-update \"root.default:acl_administer_queue=user1\\,user2 group1\\,"
+ + "group2,maximum-capacity=25;root.a.a2:capacity=75\"\n"
+ "Example (global scheduler update): yarn schedulerconf "
+ "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ + "Example (global scheduler update with comma in value): yarn "
+ + "schedulerconf "
+ + "-global \"acl_administer_queue=user1\\,user2 group1\\,group2\"\n"
+ "Example (format scheduler configuration): yarn schedulerconf "
+ "-format\n"
+ "Example (get scheduler configuration): yarn schedulerconf "
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index 3b961df..201b6d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -316,6 +316,19 @@ public class TestSchedConfCLI extends JerseyTestBase {
}
@Test(timeout = 10000)
+ public void testAddQueuesWithCommaInValue() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.addQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2",
+ schedUpdateInfo);
+ QueueConfigInfo addInfo = schedUpdateInfo.getAddQueueInfo().get(0);
+ assertEquals("root.a", addInfo.getQueue());
+ Map<String, String> params = addInfo.getParams();
+ assertEquals(2, params.size());
+ assertEquals("a1Val1,a1Val2 a1Val3", params.get("a1"));
+ assertEquals("a2Val1,a2Val2", params.get("a2"));
+ }
+
+ @Test(timeout = 10000)
public void testRemoveQueues() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.removeQueues("root.a;root.b;root.c.c1", schedUpdateInfo);
@@ -354,6 +367,19 @@ public class TestSchedConfCLI extends JerseyTestBase {
}
@Test(timeout = 10000)
+ public void testUpdateQueuesWithCommaInValue() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.updateQueues("root.a:a1=a1Val1\\,a1Val2 a1Val3,a2=a2Val1\\,a2Val2",
+ schedUpdateInfo);
+ QueueConfigInfo updateInfo = schedUpdateInfo.getUpdateQueueInfo().get(0);
+ assertEquals("root.a", updateInfo.getQueue());
+ Map<String, String> params = updateInfo.getParams();
+ assertEquals(2, params.size());
+ assertEquals("a1Val1,a1Val2 a1Val3", params.get("a1"));
+ assertEquals("a2Val1,a2Val2", params.get("a2"));
+ }
+
+ @Test(timeout = 10000)
public void testGlobalUpdate() {
SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
cli.globalUpdates("schedKey1=schedVal1,schedKey2=schedVal2",
@@ -363,4 +389,17 @@ public class TestSchedConfCLI extends JerseyTestBase {
assertEquals("schedVal1", globalInfo.get("schedKey1"));
assertEquals("schedVal2", globalInfo.get("schedKey2"));
}
+
+ @Test(timeout = 10000)
+ public void testGlobalUpdateWithCommaInValue() {
+ SchedConfUpdateInfo schedUpdateInfo = new SchedConfUpdateInfo();
+ cli.globalUpdates(
+ "schedKey1=schedVal1.1\\,schedVal1.2 schedVal1.3,schedKey2=schedVal2",
+ schedUpdateInfo);
+ Map<String, String> globalInfo = schedUpdateInfo.getGlobalParams();
+ assertEquals(2, globalInfo.size());
+ assertEquals("schedVal1.1,schedVal1.2 schedVal1.3",
+ globalInfo.get("schedKey1"));
+ assertEquals("schedVal2", globalInfo.get("schedKey2"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org