You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2019/11/27 16:54:28 UTC
[hadoop] branch branch-3.2 updated: YARN-9937. Add missing queue
configs in RMWebService#CapacitySchedulerQueueInfo.
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 8eda9fc YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo.
8eda9fc is described below
commit 8eda9fcab8a931117a2b53f2a5353b6219e2b433
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Oct 29 12:24:37 2019 +0530
YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo.
Contributed by Prabhu Joseph.
---
.../scheduler/capacity/CSQueue.java | 4 ++
.../capacity/CapacitySchedulerConfiguration.java | 2 +-
.../webapp/JAXBContextResolver.java | 3 +-
.../webapp/dao/CapacitySchedulerInfo.java | 11 +++--
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 30 ++++++++----
.../webapp/dao/CapacitySchedulerQueueInfo.java | 54 +++++++++++++++++++++-
.../resourcemanager/webapp/dao/QueueAclInfo.java | 48 +++++++++++++++++++
.../resourcemanager/webapp/dao/QueueAclsInfo.java | 47 +++++++++++++++++++
8 files changed, 182 insertions(+), 17 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index c0c280e..1af3250 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -86,6 +86,10 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
public PrivilegedEntity getPrivilegedEntity();
+ Resource getMaximumAllocation();
+
+ Resource getMinimumAllocation();
+
/**
* Get the configured <em>capacity</em> of the queue.
* @return configured queue capacity
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 760636e..ada554c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -395,7 +395,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
- static String getQueuePrefix(String queue) {
+ public static String getQueuePrefix(String queue) {
String queueName = PREFIX + queue + DOT;
return queueName;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index 2e4204e..8c9c056 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -55,7 +55,8 @@ public class JAXBContextResolver implements ContextResolver<JAXBContext> {
UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class,
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
- AppTimeoutInfo.class, ResourceInformationsInfo.class };
+ AppTimeoutInfo.class, ResourceInformationsInfo.class,
+ QueueAclsInfo.class, QueueAclInfo.class};
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index ca42041..b8a5426 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -61,7 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
- queues = getQueues(parent);
+ queues = getQueues(cs, parent);
health = new CapacitySchedulerHealthInfo(cs);
}
@@ -89,7 +89,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
return this.queues;
}
- protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
+ protected CapacitySchedulerQueueInfoList getQueues(
+ CapacityScheduler cs, CSQueue parent) {
CapacitySchedulerQueueInfoList queuesInfo =
new CapacitySchedulerQueueInfoList();
// JAXB marashalling leads to situation where the "type" field injected
@@ -112,10 +113,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
for (CSQueue queue : childQueues) {
CapacitySchedulerQueueInfo info;
if (queue instanceof LeafQueue) {
- info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
+ info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue);
} else {
- info = new CapacitySchedulerQueueInfo(queue);
- info.queues = getQueues(queue);
+ info = new CapacitySchedulerQueueInfo(cs, queue);
+ info.queues = getQueues(cs, queue);
}
queuesInfo.addToQueueInfoList(info);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index a53e921..64cc04a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuot
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.AutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
@@ -45,6 +46,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
+ protected float configuredMaxAMResourceLimit;
protected ResourceInfo AMResourceLimit;
protected ResourceInfo usedAMResource;
protected ResourceInfo userAMResourceLimit;
@@ -53,15 +55,14 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected String defaultNodeLabelExpression;
protected int defaultPriority;
protected boolean isAutoCreatedLeafQueue;
-
- @XmlTransient
- protected String orderingPolicyInfo;
+ protected long maxApplicationLifetime;
+ protected long defaultApplicationLifetime;
CapacitySchedulerLeafQueueInfo() {
};
- CapacitySchedulerLeafQueueInfo(LeafQueue q) {
- super(q);
+ CapacitySchedulerLeafQueueInfo(CapacityScheduler cs, LeafQueue q) {
+ super(cs, q);
numActiveApplications = q.getNumActiveApplications();
numPendingApplications = q.getNumPendingApplications();
numContainers = q.getNumContainers();
@@ -70,6 +71,7 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsersManager().getUsersInfo());
userLimitFactor = q.getUserLimitFactor();
+ configuredMaxAMResourceLimit = q.getMaxAMResourcePerQueuePercent();
AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
preemptionDisabled = q.getPreemptionDisabled();
@@ -91,6 +93,8 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
if ( q instanceof AutoCreatedLeafQueue) {
isAutoCreatedLeafQueue = true;
}
+ defaultApplicationLifetime = q.getDefaultApplicationLifetime();
+ maxApplicationLifetime = q.getMaximumApplicationLifetime();
}
@Override
@@ -133,6 +137,10 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
return users;
}
+ public float getConfiguredMaxAMResourceLimit() {
+ return configuredMaxAMResourceLimit;
+ }
+
public float getUserLimitFactor() {
return userLimitFactor;
}
@@ -156,10 +164,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public boolean getIntraQueuePreemptionDisabled() {
return intraQueuePreemptionDisabled;
}
-
- public String getOrderingPolicyInfo() {
- return orderingPolicyInfo;
- }
public String getDefaultNodeLabelExpression() {
return defaultNodeLabelExpression;
@@ -172,4 +176,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public boolean isAutoCreatedLeafQueue() {
return isAutoCreatedLeafQueue;
}
+
+ public long getDefaultApplicationLifetime() {
+ return defaultApplicationLifetime;
+ }
+
+ public long getMaxApplicationLifetime() {
+ return maxApplicationLifetime;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index fd256b1..c066e0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAccessType;
@@ -27,10 +28,16 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -65,11 +72,15 @@ public class CapacitySchedulerQueueInfo {
protected ResourcesInfo resources;
protected ResourceInfo minEffectiveCapacity;
protected ResourceInfo maxEffectiveCapacity;
+ protected ResourceInfo maximumAllocation;
+ protected QueueAclsInfo queueAcls;
+ protected int queuePriority;
+ protected String orderingPolicyInfo;
CapacitySchedulerQueueInfo() {
};
- CapacitySchedulerQueueInfo(CSQueue q) {
+ CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) {
queuePath = q.getQueuePath();
capacity = q.getCapacity() * 100;
@@ -114,6 +125,31 @@ public class CapacitySchedulerQueueInfo {
q.getQueueResourceQuotas().getEffectiveMinResource());
maxEffectiveCapacity = new ResourceInfo(
q.getQueueResourceQuotas().getEffectiveMaxResource());
+ maximumAllocation = new ResourceInfo(q.getMaximumAllocation());
+
+ CapacitySchedulerConfiguration conf = cs.getConfiguration();
+ queueAcls = new QueueAclsInfo();
+ for (Map.Entry<AccessType, AccessControlList> e : conf
+ .getAcls(queuePath).entrySet()) {
+ QueueAclInfo queueAcl = new QueueAclInfo(e.getKey().toString(),
+ e.getValue().getAclString());
+ queueAcls.add(queueAcl);
+ }
+
+ String aclApplicationMaxPriority = "acl_" +
+ StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
+ String priorityAcls = conf.get(CapacitySchedulerConfiguration
+ .getQueuePrefix(queuePath) + aclApplicationMaxPriority, conf.ALL_ACL);
+
+ QueueAclInfo queueAcl = new QueueAclInfo(
+ AccessType.APPLICATION_MAX_PRIORITY.toString(), priorityAcls);
+ queueAcls.add(queueAcl);
+
+ queuePriority = q.getPriority().getPriority();
+ if (q instanceof ParentQueue) {
+ orderingPolicyInfo = ((ParentQueue) q).getQueueOrderingPolicy()
+ .getConfigName();
+ }
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -220,6 +256,22 @@ public class CapacitySchedulerQueueInfo {
return maxEffectiveCapacity;
}
+ public ResourceInfo getMaximumAllocation() {
+ return maximumAllocation;
+ }
+
+ public QueueAclsInfo getQueueAcls() {
+ return queueAcls;
+ }
+
+ public int getPriority() {
+ return queuePriority;
+ }
+
+ public String getOrderingPolicyInfo() {
+ return orderingPolicyInfo;
+ }
+
public boolean isLeafQueue() {
return getQueues() == null;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java
new file mode 100644
index 0000000..8d78ed5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclInfo.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "queueAcl")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueAclInfo {
+
+ protected String accessType;
+ protected String accessControlList;
+
+ public QueueAclInfo() {
+ // JAXB needs this
+ }
+
+ public QueueAclInfo(String accessType, String accessControlList) {
+ this.accessType = accessType;
+ this.accessControlList = accessControlList;
+ }
+
+ public String getAccessType() {
+ return accessType;
+ }
+
+ public String getAccessControlList() {
+ return accessControlList;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java
new file mode 100644
index 0000000..38e836e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueAclsInfo.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "queueAcls")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueueAclsInfo {
+
+ protected ArrayList<QueueAclInfo> queueAcl = new ArrayList<QueueAclInfo>();
+
+ public QueueAclsInfo() {
+ } // JAXB needs this
+
+ public void add(QueueAclInfo queueAclInfo) {
+ queueAcl.add(queueAclInfo);
+ }
+
+ public ArrayList<QueueAclInfo> getQueueAcls() {
+ return queueAcl;
+ }
+
+ public void addAll(ArrayList<QueueAclInfo> queueAclsInfo) {
+ queueAcl.addAll(queueAclsInfo);
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org