You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2022/03/02 13:45:48 UTC
[hadoop] branch branch-3.3 updated: YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 3ef3c5a YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
3ef3c5a is described below
commit 3ef3c5a05be2df7ec3dcb6e546387a8b9caf057a
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Mar 2 14:45:31 2022 +0100
YARN-11033. isAbsoluteResource is not correct for dynamically created queues. Contributed by Tamas Domok
---
.../resourcemanager/scheduler/capacity/AbstractCSQueue.java | 2 +-
.../resourcemanager/webapp/dao/CapacitySchedulerInfo.java | 5 +++++
.../webapp/dao/CapacitySchedulerQueueInfo.java | 12 +++---------
.../webapp/TestRMWebServicesCapacitySched.java | 4 +++-
.../webapp/TestRMWebServicesForCSWithPartitions.java | 2 +-
5 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e7d9fa5..3870d42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -128,7 +128,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// either at this level or anywhere in the queue's hierarchy.
private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false;
- protected enum CapacityConfigType {
+ public enum CapacityConfigType {
NONE, PERCENTAGE, ABSOLUTE_RESOURCE
};
protected CapacityConfigType capacityConfigType =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 8de7529..9b8773e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -46,6 +47,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected float usedCapacity;
protected float maxCapacity;
protected String queueName;
+ private boolean isAbsoluteResource;
protected CapacitySchedulerQueueInfoList queues;
protected QueueCapacitiesInfo capacities;
protected CapacitySchedulerHealthInfo health;
@@ -75,6 +77,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
health = new CapacitySchedulerHealthInfo(cs);
maximumAllocation = new ResourceInfo(parent.getMaximumAllocation());
+ isAbsoluteResource = parent.getCapacityConfigType() ==
+ AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
+
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
for (Map.Entry<AccessType, AccessControlList> e : conf
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index e38ab0d..56ac29b25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -42,11 +43,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
- CapacitySchedulerConfiguration.RESOURCE_PATTERN;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.
- CapacitySchedulerConfiguration.CAPACITY;
-
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
@@ -160,10 +156,8 @@ public class CapacitySchedulerQueueInfo {
.getConfigName();
}
- String configuredCapacity = conf.get(
- CapacitySchedulerConfiguration.getQueuePrefix(queuePath) + CAPACITY);
- isAbsoluteResource = (configuredCapacity != null)
- && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+ isAbsoluteResource = q.getCapacityConfigType() ==
+ AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
autoCreateChildQueueEnabled = conf.
isAutoCreateChildQueueEnabled(queuePath);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1e856bd..b24b7c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -362,11 +362,13 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements in: " + info, 12, info.length());
+ assertEquals("incorrect number of elements in: " + info, 13, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), info.getString("queueName"));
+ assertFalse("isAbsoluteResource should be null for root queue",
+ info.getBoolean("isAbsoluteResource"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
assertEquals("incorrect number of elements in: " + health, 3,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
index f587498..93351f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesForCSWithPartitions.java
@@ -576,7 +576,7 @@ public class TestRMWebServicesForCSWithPartitions extends JerseyTestBase {
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements", 12, info.length());
+ assertEquals("incorrect number of elements", 13, info.length());
JSONObject capacitiesJsonObject = info.getJSONObject(CAPACITIES);
JSONArray partitionsCapsArray =
capacitiesJsonObject.getJSONArray(QUEUE_CAPACITIES_BY_PARTITION);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org