You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2019/10/30 19:04:52 UTC
[hadoop] branch trunk updated: YARN-9937. Add missing queue configs
in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9a2e43e YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.
9a2e43e is described below
commit 9a2e43e29e14905d98edd8512d24363dec407634
Author: Sunil G <su...@apache.org>
AuthorDate: Thu Oct 31 00:34:27 2019 +0530
YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.
---
.../webapp/JAXBContextResolver.java | 3 +-
.../webapp/dao/CapacitySchedulerInfo.java | 11 +++--
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 32 +++++++++----
.../webapp/dao/CapacitySchedulerQueueInfo.java | 55 +++++++++++++++++++++-
.../resourcemanager/webapp/dao/QueueAclInfo.java | 48 +++++++++++++++++++
.../resourcemanager/webapp/dao/QueueAclsInfo.java | 47 ++++++++++++++++++
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
7 files changed, 181 insertions(+), 19 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index e0c6647..f6eb2ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -56,7 +56,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class, ResourceInformationsInfo.class,
- ActivitiesInfo.class, AppActivitiesInfo.class};
+ ActivitiesInfo.class, AppActivitiesInfo.class,
+ QueueAclsInfo.class, QueueAclInfo.class};
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index ca42041..b8a5426 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -61,7 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
- queues = getQueues(parent);
+ queues = getQueues(cs, parent);
health = new CapacitySchedulerHealthInfo(cs);
}
@@ -89,7 +89,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
return this.queues;
}
- protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
+ protected CapacitySchedulerQueueInfoList getQueues(
+ CapacityScheduler cs, CSQueue parent) {
CapacitySchedulerQueueInfoList queuesInfo =
new CapacitySchedulerQueueInfoList();
// JAXB marashalling leads to situation where the "type" field injected
@@ -112,10 +113,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
for (CSQueue queue : childQueues) {
CapacitySchedulerQueueInfo info;
if (queue instanceof LeafQueue) {
- info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
+ info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue);
} else {
- info = new CapacitySchedulerQueueInfo(queue);
- info.queues = getQueues(queue);
+ info = new CapacitySchedulerQueueInfo(cs, queue);
+ info.queues = getQueues(cs, queue);
}
queuesInfo.addToQueueInfoList(info);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index a53e921..4920f76 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -22,13 +22,13 @@ import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.AutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
@@ -45,6 +45,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
+ protected float configuredMaxAMResourceLimit;
protected ResourceInfo AMResourceLimit;
protected ResourceInfo usedAMResource;
protected ResourceInfo userAMResourceLimit;
@@ -53,15 +54,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected String defaultNodeLabelExpression;
protected int defaultPriority;
protected boolean isAutoCreatedLeafQueue;
-
- @XmlTransient
- protected String orderingPolicyInfo;
+ protected long maxApplicationLifetime;
+ protected long defaultApplicationLifetime;
CapacitySchedulerLeafQueueInfo() {
};
- CapacitySchedulerLeafQueueInfo(LeafQueue q) {
- super(q);
+ CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) {
+ super(cs, q);
numActiveApplications = q.getNumActiveApplications();
numPendingApplications = q.getNumPendingApplications();
numContainers = q.getNumContainers();
@@ -70,6 +70,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsersManager().getUsersInfo());
userLimitFactor = q.getUserLimitFactor();
+ configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent();
AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
preemptionDisabled = q.getPreemptionDisabled();
@@ -91,6 +92,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
if ( q instanceof AutoCreatedLeafQueue) {
isAutoCreatedLeafQueue = true;
}
+ defaultApplicationLifetime = q.getDefaultApplicationLifetime();
+ maxApplicationLifetime = q.getMaximumApplicationLifetime();
}
@Override
@@ -136,6 +139,10 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public float getUserLimitFactor() {
return userLimitFactor;
}
+
+ public float getConfiguredMaxAMResourceLimit() {
+ return configuredMaxAMResourceLimit;
+ }
public ResourceInfo getAMResourceLimit() {
return AMResourceLimit;
@@ -157,10 +164,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
return intraQueuePreemptionDisabled;
}
- public String getOrderingPolicyInfo() {
- return orderingPolicyInfo;
- }
-
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
}
@@ -172,4 +175,13 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public boolean isAutoCreatedLeafQueue() {
return isAutoCreatedLeafQueue;
}
+
+ public long getDefaultApplicationLifetime() {
+ return defaultApplicationLifetime;
+ }
+
+ public long getMaxApplicationLifetime() {
+ return maxApplicationLifetime;
+ }
+
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index fd256b1..bde1bb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
@@ -27,10 +29,16 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -65,11 +73,15 @@ public class CapacitySchedulerQueueInfo {
protected ResourcesInfo resources;
protected ResourceInfo minEffectiveCapacity;
protected ResourceInfo maxEffectiveCapacity;
+ protected ResourceInfo maximumAllocation;
+ protected QueueAclsInfo queueAcls;
+ protected int queuePriority;
+ protected String orderingPolicyInfo;
CapacitySchedulerQueueInfo() {
};
- CapacitySchedulerQueueInfo(CSQueue q) {
+ CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) {
queuePath = q.getQueuePath();
capacity = q.getCapacity() * 100;
@@ -114,6 +126,31 @@ public class CapacitySchedulerQueueInfo {
q.getQueueResourceQuotas().getEffectiveMinResource());
maxEffectiveCapacity = new ResourceInfo(
q.getQueueResourceQuotas().getEffectiveMaxResource());
+ maximumAllocation = new ResourceInfo(q.getMaximumAllocation());
+
+ CapacitySchedulerConfiguration conf = cs.getConfiguration();
+ queueAcls = new QueueAclsInfo();
+ for (Map.Entry<AccessType, AccessControlList> e : conf
+ .getAcls(queueName).entrySet()) {
+ QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
+ e.getValue().getAclString());
+ queueAcls.add(queueAcl);
+ }
+
+ String aclApplicationMaxPriority = "acl_" +
+ StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
+ String priorityAcls = conf.get(queuePath + aclApplicationMaxPriority,
+ conf.ALL_ACL);
+
+ QueueAclInfo queueAcl = new QueueAclInfo(
+ AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
+ queueAcls.add(queueAcl);
+
+ queuePriority = q.getPriority().getPriority();
+ if (q instanceof ParentQueue) {
+ orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
+ .getConfigName();
+ }
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -220,6 +257,22 @@ public class CapacitySchedulerQueueInfo {
return maxEffectiveCapacity;
}
+ public ResourceInfo getMaximumAllocation() {
+ return maximumAllocation;
+ }
+
+ public QueueAclsInfo getQueueAcls() {
+ return queueAcls;
+ }
+
+ public int getPriority() {
+ return queuePriority;
+ }
+
+ public String getOrderingPolicyInfo() {
+ return orderingPolicyInfo;
+ }
+
public boolean isLeafQueue() {
return getQueues() == null;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java
new file mode 100644
index 0000000..6e623ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "queueAcl")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueAclInfo {
+
+ protected String accessType;
+ protected String accessControlList;
+
+ public QueueAclInfo() {
+ // JAXB needs this
+ }
+
+ public QueueAclInfo(String accessType, String accessControlList) {
+ this.accessType = accessType;
+ this.accessControlList = accessControlList;
+ }
+
+ public String getAccessType() {
+ return accessType;
+ }
+
+ public String getAccessControlList() {
+ return accessControlList;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java
new file mode 100644
index 0000000..ea07dbf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "queueAcls")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueAclsInfo {
+
+ protected ArrayList<QueueAclInfo> queueAcl = new ArrayList<QueueAclInfo>();
+
+ public QueueAclsInfo() {
+ } // JAXB needs this
+
+ public void add(QueueAclInfo queueAclInfo) {
+ queueAcl.add(queueAclInfo);
+ }
+
+ public ArrayList<QueueAclInfo> getQueueAcls() {
+ return queueAcl;
+ }
+
+ public void addAll(ArrayList<QueueAclInfo> queueAclsInfo) {
+ queueAcl.addAll(queueAclsInfo);
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index d66130f..25be1eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -364,10 +364,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
- int numExpectedElements = 20;
+ int numExpectedElements = 24;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 35;
+ numExpectedElements = 42;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org