You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/04/23 14:12:32 UTC
[hadoop] branch trunk updated: YARN-10746. RmWebApp add
default-node-label-expression to the queue info. Contributed by Gergely
Pollak
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 919daec YARN-10746. RmWebApp add default-node-label-expression to the queue info. Contributed by Gergely Pollak
919daec is described below
commit 919daec36b23457c107e819855791e7e6d144594
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Fri Apr 23 16:12:12 2021 +0200
YARN-10746. RmWebApp add default-node-label-expression to the queue info. Contributed by Gergely Pollak
---
.../webapp/dao/CapacitySchedulerInfo.java | 3 ++
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 12 ++---
.../webapp/dao/CapacitySchedulerQueueInfo.java | 8 +++-
.../webapp/TestRMWebServicesCapacitySched.java | 52 ++++++++++++++++++++++
4 files changed, 65 insertions(+), 10 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index fc18d4d..78e2133 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -61,6 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected String queueType;
protected String creationMethod;
protected String autoCreationEligibility;
+ protected String defaultNodeLabelExpression;
@XmlTransient
static final float EPSILON = 1e-8f;
@@ -114,6 +115,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
creationMethod = CapacitySchedulerInfoHelper.getCreationMethod(parent);
autoCreationEligibility = CapacitySchedulerInfoHelper
.getAutoCreationEligibility(parent);
+
+ defaultNodeLabelExpression = parent.getDefaultNodeLabelExpression();
}
public float getCapacity() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 6aef01b..16abb99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -52,7 +52,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected ResourceInfo userAMResourceLimit;
protected boolean preemptionDisabled;
protected boolean intraQueuePreemptionDisabled;
- protected String defaultNodeLabelExpression;
protected int defaultPriority;
protected boolean isAutoCreatedLeafQueue;
protected long maxApplicationLifetime;
@@ -81,7 +80,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
intraQueuePreemptionDisabled = q.getIntraQueuePreemptionDisabled();
orderingPolicyDisplayName = q.getOrderingPolicy().getInfo();
orderingPolicyInfo = q.getOrderingPolicy().getConfigName();
- defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
defaultPriority = q.getDefaultApplicationPriority().getPriority();
ArrayList<UserInfo> usersList = users.getUsersList();
if (usersList.isEmpty()) {
@@ -148,17 +146,17 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public float getConfiguredMaxAMResourceLimit() {
return configuredMaxAMResourceLimit;
}
-
+
public ResourceInfo getAMResourceLimit() {
return AMResourceLimit;
}
-
+
public ResourceInfo getUsedAMResource() {
return usedAMResource;
}
public ResourceInfo getUserAMResourceLimit() {
- return userAMResourceLimit;
+ return userAMResourceLimit;
}
public boolean getPreemptionDisabled() {
@@ -172,10 +170,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public String getOrderingPolicyDisplayName() {
return orderingPolicyDisplayName;
}
-
- public String getDefaultNodeLabelExpression() {
- return defaultNodeLabelExpression;
- }
public int getDefaultApplicationPriority() {
return defaultPriority;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 3865cce..31857bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -91,6 +91,7 @@ public class CapacitySchedulerQueueInfo {
protected String queueType;
protected String creationMethod;
protected String autoCreationEligibility;
+ protected String defaultNodeLabelExpression;
CapacitySchedulerQueueInfo() {
};
@@ -120,6 +121,7 @@ public class CapacitySchedulerQueueInfo {
reservedContainers = q.getMetrics().getReservedContainers();
queueName = q.getQueueName();
state = q.getState();
+ defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
resourcesUsed = new ResourceInfo(q.getUsedResources());
if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) {
hideReservationQueues = true;
@@ -271,7 +273,7 @@ public class CapacitySchedulerQueueInfo {
static float cap(float val, float low, float hi) {
return Math.min(Math.max(val, low), hi);
}
-
+
public ArrayList<String> getNodeLabels() {
return this.nodeLabels;
}
@@ -335,4 +337,8 @@ public class CapacitySchedulerQueueInfo {
public float getNormalizedWeight() {
return normalizedWeight;
}
+
+ public String getDefaultNodeLabelExpression() {
+ return defaultNodeLabelExpression;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index a0a0884..ab32941 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -661,6 +661,58 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
}
}
+ @Test
+ public void testNodeLabelDefaultAPI() throws Exception {
+ CapacitySchedulerConfiguration config =
+ ((CapacityScheduler)rm.getResourceScheduler()).getConfiguration();
+
+ config.setDefaultNodeLabelExpression("root", "ROOT-INHERITED");
+ config.setDefaultNodeLabelExpression("root.a", "root-a-default-label");
+ rm.getResourceScheduler().reinitialize(config, rm.getRMContext());
+
+ //Start RM so that it accepts app submissions
+ rm.start();
+ try {
+ //Get the XML from ws/v1/cluster/scheduler
+ WebResource r = resource();
+ ClientResponse response = r.path("ws/v1/cluster/scheduler")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ String xml = response.getEntity(String.class);
+ DocumentBuilder db = DocumentBuilderFactory.newInstance()
+ .newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ //Parse the XML we got
+ Document dom = db.parse(is);
+
+ NodeList allQueues = dom.getElementsByTagName("queue");
+ for (int i = 0; i < allQueues.getLength(); ++i) {
+ Node queueNode = allQueues.item(i);
+ Node queuePathNode = getChildNodeByName(queueNode, "queuePath");
+ if (queuePathNode == null) {
+ continue;
+ }
+
+ String queuePath = queuePathNode.getTextContent();
+ if (queuePath != null) {
+ if (queuePath.startsWith("root.a")) {
+ assertEquals("root-a-default-label",
+ getChildNodeByName(queueNode, "defaultNodeLabelExpression")
+ .getTextContent());
+ } else {
+ assertEquals("ROOT-INHERITED",
+ getChildNodeByName(queueNode, "defaultNodeLabelExpression")
+ .getTextContent());
+ }
+ }
+ }
+ } finally {
+ rm.stop();
+ }
+ }
+
private void checkResourcesUsed(JSONObject queue) throws JSONException {
queue.getJSONObject("resourcesUsed").getInt("memory");
queue.getJSONObject("resourcesUsed").getInt("vCores");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org