You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/04/29 07:36:49 UTC
[hadoop] branch branch-3.3 updated: YARN-10237. Add
isAbsoluteResource config for queue in scheduler response. Contributed by
Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 75433a6 YARN-10237. Add isAbsoluteResource config for queue in scheduler response. Contributed by Prabhu Joseph
75433a6 is described below
commit 75433a658d1363b8d4a7fa9c969b6d1ccda07c0a
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Apr 29 09:36:37 2020 +0200
YARN-10237. Add isAbsoluteResource config for queue in scheduler response. Contributed by Prabhu Joseph
---
.../capacity/CapacitySchedulerConfiguration.java | 3 ++-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 15 +++++++++++++++
.../webapp/TestRMWebServicesCapacitySched.java | 19 +++++++++++++++++--
3 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 3ae66ca..7edf2da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -377,7 +377,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$";
- private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
+ public static final Pattern RESOURCE_PATTERN =
+ Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
/**
* Different resource types supported.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index f4d4070..947c44c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -42,6 +42,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
+ CapacitySchedulerConfiguration.RESOURCE_PATTERN;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
+ CapacitySchedulerConfiguration.CAPACITY;
+
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@@ -61,6 +66,7 @@ public class CapacitySchedulerQueueInfo {
protected float absoluteUsedCapacity;
protected int numApplications;
protected String queueName;
+ protected boolean isAbsoluteResource;
protected QueueState state;
protected CapacitySchedulerQueueInfoList queues;
protected ResourceInfo resourcesUsed;
@@ -151,6 +157,11 @@ public class CapacitySchedulerQueueInfo {
orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
.getConfigName();
}
+
+ String configuredCapacity = conf.get(
+ CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
+ isAbsoluteResource = (configuredCapacity != null)
+ && RESOURCE_PATTERN.matcher(configuredCapacity).find();
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -203,6 +214,10 @@ public class CapacitySchedulerQueueInfo {
return pendingContainers;
}
+ public boolean isAbsoluteResource() {
+ return isAbsoluteResource;
+ }
+
public String getQueueName() {
return this.queueName;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 324a392..8098622 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -79,6 +80,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
int numApplications;
String queueName;
String state;
+ boolean isAbsoluteResource;
}
private class LeafQueueInfo extends QueueInfo {
@@ -131,6 +133,9 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
final String B = CapacitySchedulerConfiguration.ROOT + ".b";
config.setCapacity(B, 89.5f);
+ final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+ config.setCapacity(C, "[memory=1024]");
+
// Define 2nd-level queues
final String A1 = A + ".a1";
final String A2 = A + ".a2";
@@ -290,6 +295,8 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
WebServicesTestUtils.getXmlInt(qElem, "numApplications");
qi.queueName = WebServicesTestUtils.getXmlString(qElem, "queueName");
qi.state = WebServicesTestUtils.getXmlString(qElem, "state");
+ qi.isAbsoluteResource = WebServicesTestUtils.getXmlBoolean(qElem,
+ "isAbsoluteResource");
verifySubQueueGeneric(q, qi, parentAbsCapacity, parentAbsMaxCapacity);
if (hasSubQueues) {
for (int j = 0; j < children.getLength(); j++) {
@@ -384,10 +391,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
- int numExpectedElements = 24;
+ int numExpectedElements = 25;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 42;
+ numExpectedElements = 43;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
@@ -471,6 +478,14 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
+ " expected: " + q, qshortName.matches(info.queueName));
assertTrue("state doesn't match",
(csConf.getState(q).toString()).matches(info.state));
+ if (q.equals("c")) {
+ assertTrue("c queue is not configured in Absolute resource",
+ info.isAbsoluteResource);
+ } else {
+ assertFalse(info.queueName
+ + " queue is not configured in Absolute resource",
+ info.isAbsoluteResource);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org