You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2019/11/04 17:58:48 UTC
[hadoop] branch branch-3.2 updated: YARN-9950. Unset Ordering
Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively.
Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 597b315 YARN-9950. Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
597b315 is described below
commit 597b315811188597d66e7fb264fd8b46628d1404
Author: Sunil G <su...@apache.org>
AuthorDate: Mon Nov 4 23:27:33 2019 +0530
YARN-9950. Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
(cherry picked from commit 51e7d1b37eb06165657befdae82c015055c58334)
---
.../conf/MutableCSConfigurationProvider.java | 17 ++++
.../TestRMWebServicesConfigurationMutation.java | 100 +++++++++++++++++++++
2 files changed, 117 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 992eea7..7eab65f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -42,6 +42,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
+
/**
* CS configuration provider which implements
* {@link MutableConfigurationProvider} for modifying capacity scheduler
@@ -278,6 +280,13 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
+ CapacitySchedulerConfiguration.QUEUES;
if (siblingQueues.size() == 0) {
confUpdate.put(queuesConfig, null);
+ // Unset Ordering Policy of Leaf Queue converted from
+ // Parent Queue after removeQueue
+ String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + ORDERING_POLICY;
+ proposedConf.unset(queueOrderingPolicy);
+ confUpdate.put(queueOrderingPolicy, null);
} else {
confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
}
@@ -326,6 +335,14 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
}
+ // Unset Ordering Policy of Parent Queue converted from
+ // Leaf Queue after addQueue
+ String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ if (siblingQueues.size() == 1) {
+ proposedConf.unset(queueOrderingPolicy);
+ confUpdate.put(queueOrderingPolicy, null);
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 773b4bf..24a71fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -60,6 +60,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
/**
* Test scheduler configuration mutation via REST API.
@@ -334,6 +335,105 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
}
@Test
+ public void testUnsetParentQueueOrderingPolicy() throws Exception {
+ WebResource r = resource();
+ ClientResponse response;
+
+ // Update ordering policy of Leaf Queue root.b to fair
+ SchedConfUpdateInfo updateInfo1 = new SchedConfUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.ORDERING_POLICY,
+ "fair");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.b", updateParam);
+ updateInfo1.getUpdateQueueInfo().add(aUpdateInfo);
+ response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo1,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ String bOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + "root.b" + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ assertEquals("fair", newCSConf.get(bOrderingPolicy));
+
+ stopQueue("root.b");
+
+ // Add root.b.b1 which makes root.b a Parent Queue
+ SchedConfUpdateInfo updateInfo2 = new SchedConfUpdateInfo();
+ Map<String, String> capacity = new HashMap<>();
+ capacity.put(CapacitySchedulerConfiguration.CAPACITY, "100");
+ QueueConfigInfo b1 = new QueueConfigInfo("root.b.b1", capacity);
+ updateInfo2.getAddQueueInfo().add(b1);
+ response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo2,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+
+ // Validate unset ordering policy of root.b after converted to
+ // Parent Queue
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ newCSConf = ((CapacityScheduler) rm.getResourceScheduler())
+ .getConfiguration();
+ bOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + "root.b" + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ assertNull("Failed to unset Parent Queue OrderingPolicy",
+ newCSConf.get(bOrderingPolicy));
+ }
+
+ @Test
+ public void testUnsetLeafQueueOrderingPolicy() throws Exception {
+ WebResource r = resource();
+ ClientResponse response;
+
+ // Update ordering policy of Parent Queue root.c to priority-utilization
+ SchedConfUpdateInfo updateInfo1 = new SchedConfUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.ORDERING_POLICY,
+ "priority-utilization");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.c", updateParam);
+ updateInfo1.getUpdateQueueInfo().add(aUpdateInfo);
+ response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo1,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ CapacitySchedulerConfiguration newCSConf =
+ ((CapacityScheduler) rm.getResourceScheduler()).getConfiguration();
+ String cOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + "root.c" + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ assertEquals("priority-utilization", newCSConf.get(cOrderingPolicy));
+
+ stopQueue("root.c.c1");
+
+ // Remove root.c.c1 which makes root.c a Leaf Queue
+ SchedConfUpdateInfo updateInfo2 = new SchedConfUpdateInfo();
+ updateInfo2.getRemoveQueueInfo().add("root.c.c1");
+ response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo2,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+ // Validate unset ordering policy of root.c after converted to
+ // Leaf Queue
+ newCSConf = ((CapacityScheduler) rm.getResourceScheduler())
+ .getConfiguration();
+ cOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + "root.c" + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ assertNull("Failed to unset Leaf Queue OrderingPolicy",
+ newCSConf.get(cOrderingPolicy));
+ }
+
+ @Test
public void testRemoveQueue() throws Exception {
WebResource r = resource();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org