[hadoop] branch trunk updated: YARN-10160. Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph

[sn...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 0debe55  YARN-10160. Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph
0debe55 is described below

commit 0debe55d6cf36b358c86c27d43991aa44baef4f2
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue May 5 13:11:28 2020 +0200

    YARN-10160. Add auto queue creation related configs to RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph
---
 .../hadoop/yarn/webapp/WebServicesTestUtils.java   | 18 +++++
 .../webapp/dao/CapacitySchedulerQueueInfo.java     | 14 ++++
 .../webapp/dao/LeafQueueTemplateInfo.java          | 91 ++++++++++++++++++++++
 .../webapp/TestRMWebServicesCapacitySched.java     | 36 +++++++--
 4 files changed, 154 insertions(+), 5 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
index 6d9e70f..b4832ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/WebServicesTestUtils.java
@@ -94,6 +94,24 @@ public class WebServicesTestUtils {
     return val;
   }
 
+  public static String getPropertyValue(Element element, String elementName,
+      String propertyName) {
+    NodeList id = element.getElementsByTagName(elementName);
+    Element line = (Element) id.item(0);
+    if (line == null) {
+      return null;
+    }
+    NodeList properties = line.getChildNodes();
+    for (int i = 0; i < properties.getLength(); i++) {
+      Element property = (Element) properties.item(i);
+      if (getXmlString(property, "name").equals(propertyName)) {
+        return getXmlString(property, "value");
+      }
+    }
+    return null;
+  }
+
+
   public static String getXmlAttrString(Element element, String name) {
     Attr at = element.getAttributeNode(name);
     if (at != null) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 2231e51..897f940 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -84,6 +84,8 @@ public class CapacitySchedulerQueueInfo {
   protected QueueAclsInfo queueAcls;
   protected int queuePriority;
   protected String orderingPolicyInfo;
+  protected boolean autoCreateChildQueueEnabled;
+  protected LeafQueueTemplateInfo leafQueueTemplate;
 
   CapacitySchedulerQueueInfo() {
   };
@@ -163,6 +165,10 @@ public class CapacitySchedulerQueueInfo {
         CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
     isAbsoluteResource = (configuredCapacity != null)
         && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+
+    autoCreateChildQueueEnabled = conf.
+        isAutoCreateChildQueueEnabled(queuePath);
+    leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath);
   }
 
   protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -292,4 +298,12 @@ public class CapacitySchedulerQueueInfo {
   public boolean isLeafQueue() {
     return getQueues() == null;
   }
+
+  public boolean isAutoCreateChildQueueEnabled() {
+    return autoCreateChildQueueEnabled;
+  }
+
+  public LeafQueueTemplateInfo getLeafQueueTemplate() {
+    return leafQueueTemplate;
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java
new file mode 100644
index 0000000..f528a47
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+
+/**
+ * This class stores the LeafQueue Template configuration.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LeafQueueTemplateInfo {
+
+  private ArrayList<ConfItem> property = new ArrayList<>();
+
+  public LeafQueueTemplateInfo() {
+  } // JAXB needs this
+
+  public LeafQueueTemplateInfo(Configuration conf, String queuePath) {
+    String configPrefix = CapacitySchedulerConfiguration.
+        getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+        + DOT;
+    conf.forEach(entry -> {
+      if (entry.getKey().startsWith(configPrefix)) {
+        String name = entry.getKey();
+        int start = name.lastIndexOf(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+            + DOT);
+        add(new ConfItem(name.substring(start), entry.getValue()));
+      }
+    });
+  }
+
+  public void add(ConfItem confItem) {
+    property.add(confItem);
+  }
+
+  public ArrayList<ConfItem> getItems() {
+    return property;
+  }
+
+  /**
+   * This class stores the Configuration Property.
+   */
+  @XmlAccessorType(XmlAccessType.FIELD)
+  public static class ConfItem {
+
+    private String name;
+    private String value;
+
+    public ConfItem() {
+      // JAXB needs this
+    }
+
+    public ConfItem(String name, String value){
+      this.name = name;
+      this.value = value;
+    }
+
+    public String getKey() {
+      return name;
+    }
+
+    public String getValue() {
+      return value;
+    }
+  }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 8098622..1e856bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -24,6 +24,11 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+
 import java.io.StringReader;
 
 import javax.ws.rs.core.MediaType;
@@ -81,6 +86,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     String queueName;
     String state;
     boolean isAbsoluteResource;
+    boolean autoCreateChildQueueEnabled;
   }
 
   private class LeafQueueInfo extends QueueInfo {
@@ -160,11 +166,17 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
     config.setCapacity(B3, 0.5f);
     config.setUserLimitFactor(B3, 100.0f);
 
-    config.setQueues(A1, new String[] {"a1a", "a1b"});
+    config.setQueues(A1, new String[] {"a1a", "a1b", "a1c"});
     final String A1A = A1 + ".a1a";
-    config.setCapacity(A1A, 85);
+    config.setCapacity(A1A, 65);
     final String A1B = A1 + ".a1b";
     config.setCapacity(A1B, 15);
+    final String A1C = A1 + ".a1c";
+    config.setCapacity(A1C, 20);
+
+    config.setAutoCreateChildQueueEnabled(A1C, true);
+    config.setInt(PREFIX + A1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+        + DOT + CAPACITY, 50);
   }
 
   @Before
@@ -295,6 +307,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
         WebServicesTestUtils.getXmlInt(qElem, "numApplications");
     qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
     qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
+    qi.autoCreateChildQueueEnabled = WebServicesTestUtils.getXmlBoolean(qElem,
+        "autoCreateChildQueueEnabled");
     qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem,
         "isAbsoluteResource");
     verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
@@ -311,6 +325,14 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
           }
         }
       }
+    } else if (qi.autoCreateChildQueueEnabled) {
+      assertEquals("queueName doesn't match", "a1c", qi.queueName);
+      String capacityStr = WebServicesTestUtils.getPropertyValue(qElem,
+          "leafQueueTemplate", AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+          + DOT + CAPACITY);
+      int capacity = Integer.parseInt(capacityStr);
+      assertEquals(AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX + DOT
+          + CAPACITY + " doesn't match", 50, capacity);
     } else {
       LeafQueueInfo lqi = (LeafQueueInfo) qi;
       lqi.numActiveApplications =
@@ -391,10 +413,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
   private void verifySubQueue(JSONObject info, String q,
       float parentAbsCapacity, float parentAbsMaxCapacity)
       throws JSONException, Exception {
-    int numExpectedElements = 25;
+    int numExpectedElements = 27;
     boolean isParentQueue = true;
     if (!info.has("queues")) {
-      numExpectedElements = 43;
+      numExpectedElements = 45;
       isParentQueue = false;
     }
     assertEquals("incorrect number of elements", numExpectedElements, info.length());
@@ -412,7 +434,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
 
     verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
 
-    if (isParentQueue) {
+    // Separate Condition for Managed Parent Queue
+    if (qi.queueName.equals("a1c")) {
+      assertTrue(info.getBoolean("autoCreateChildQueueEnabled"));
+    } else if (isParentQueue) {
       JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
       // test subqueues
       for (int i = 0; i < arr.length(); i++) {
@@ -430,6 +455,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
 
       assertEquals("0", info.getString("queuePriority"));
       assertEquals("utilization", info.getString("orderingPolicyInfo"));
+      assertFalse(info.getBoolean("autoCreateChildQueueEnabled"));
     } else {
       Assert.assertEquals("\"type\" field is incorrect",
           "capacitySchedulerLeafQueueInfo", info.getString("type"));


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org