You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2021/03/18 12:42:25 UTC
[hadoop] branch trunk updated: YARN-10641. Refactor the max app
related update,
and fix maxApllications update error when add new queues. Contributed by Qi
Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ce6bfd5 YARN-10641. Refactor the max app related update, and fix maxApllications update error when add new queues. Contributed by Qi Zhu.
ce6bfd5 is described below
commit ce6bfd5718a3adb124a45ce09d2ef16cfed2c1a4
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Thu Mar 18 13:40:16 2021 +0100
YARN-10641. Refactor the max app related update, and fix maxApllications update error when add new queues. Contributed by Qi Zhu.
---
.../scheduler/capacity/AbstractCSQueue.java | 81 ++++++++++++----------
.../capacity/CapacitySchedulerConfiguration.java | 7 ++
.../scheduler/capacity/LeafQueue.java | 32 +++------
.../TestCapacitySchedulerNewQueueAutoCreation.java | 57 +++++++++++++++
4 files changed, 118 insertions(+), 59 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 5c0f99a..06575be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -1489,6 +1489,44 @@ public abstract class AbstractCSQueue implements CSQueue {
return ret;
}
+ void updateMaxAppRelatedField(CapacitySchedulerConfiguration conf,
+ LeafQueue leafQueue, String label) {
+ int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
+ if (maxApplications < 0) {
+ int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue();
+ if (maxGlobalPerQueueApps > 0) {
+ // In absolute mode, should
+ // shrink when change to corresponding label capacity.
+ maxApplications = this.capacityConfigType
+ != CapacityConfigType.ABSOLUTE_RESOURCE ?
+ maxGlobalPerQueueApps :
+ (int) (maxGlobalPerQueueApps * queueCapacities
+ .getAbsoluteCapacity(label));
+ } else{
+ maxApplications = (int) (conf.getMaximumSystemApplications()
+ * queueCapacities.getAbsoluteCapacity(label));
+ }
+ }
+ leafQueue.setMaxApplications(maxApplications);
+
+ int maxApplicationsPerUser = Math.min(maxApplications,
+ (int) (maxApplications
+ * (leafQueue.getUsersManager().getUserLimit() / 100.0f)
+ * leafQueue.getUsersManager().getUserLimitFactor()));
+ if (leafQueue.getUsersManager().getUserLimitFactor() == -1) {
+ maxApplicationsPerUser = maxApplications;
+ }
+
+ leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
+ LOG.info("LeafQueue:" + leafQueue.getQueuePath() +
+ "update max app related, maxApplications="
+ + maxApplications + ", maxApplicationsPerUser="
+ + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities
+ .getAbsoluteCapacity(label) + ", Cap: " + queueCapacities
+ .getCapacity(label) + ", MaxCap : " + queueCapacities
+ .getMaximumCapacity(label));
+ }
+
private void deriveCapacityFromAbsoluteConfigurations(String label,
Resource clusterResource, ResourceCalculator rc) {
@@ -1522,42 +1560,6 @@ public abstract class AbstractCSQueue implements CSQueue {
queueCapacities.setAbsoluteMaximumCapacity(label,
queueCapacities.getMaximumCapacity(label) * parent.getQueueCapacities()
.getAbsoluteMaximumCapacity(label));
-
- // Re-visit max applications for a queue based on absolute capacity if
- // needed.
- if (this instanceof LeafQueue) {
- LeafQueue leafQueue = (LeafQueue) this;
- CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications = conf.getMaximumApplicationsPerQueue(queuePath);
- if (maxApplications < 0) {
- int maxGlobalPerQueueApps = conf.getGlobalMaximumApplicationsPerQueue();
- if (maxGlobalPerQueueApps > 0) {
- maxApplications = (int) (maxGlobalPerQueueApps * queueCapacities
- .getAbsoluteCapacity(label));
- } else{
- maxApplications =
- (int) (conf.getMaximumSystemApplications() * queueCapacities
- .getAbsoluteCapacity(label));
- }
- }
- leafQueue.setMaxApplications(maxApplications);
-
- int maxApplicationsPerUser = Math.min(maxApplications,
- (int) (maxApplications
- * (leafQueue.getUsersManager().getUserLimit() / 100.0f)
- * leafQueue.getUsersManager().getUserLimitFactor()));
- if (leafQueue.getUsersManager().getUserLimitFactor() == -1) {
- maxApplicationsPerUser = maxApplications;
- }
-
- leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
- LOG.info("LeafQueue:" + leafQueue.getQueuePath() + ", maxApplications="
- + maxApplications + ", maxApplicationsPerUser="
- + maxApplicationsPerUser + ", Abs Cap:" + queueCapacities
- .getAbsoluteCapacity(label) + ", Cap: " + queueCapacities
- .getCapacity(label) + ", MaxCap : " + queueCapacities
- .getMaximumCapacity(label));
- }
}
void updateEffectiveResources(Resource clusterResource) {
@@ -1603,6 +1605,13 @@ public abstract class AbstractCSQueue implements CSQueue {
// percentage, we have to calculate percentage and update.
ResourceCalculator rc = this.csContext.getResourceCalculator();
deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc);
+ // Re-visit max applications for a queue based on absolute capacity if
+ // needed.
+ if (this instanceof LeafQueue) {
+ LeafQueue leafQueue = (LeafQueue) this;
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ updateMaxAppRelatedField(conf, leafQueue, label);
+ }
} else{
queueResourceQuotas.setEffectiveMinResource(label, Resources
.multiply(resourceByLabel,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index efd56a2..5bae2b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -477,6 +477,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return maxApplicationsPerQueue;
}
+ @VisibleForTesting
+ public void setMaximumApplicationsPerQueue(String queue,
+ int numMaxApps) {
+ setInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
+ numMaxApps);
+ }
+
/**
* Get the maximum am resource percent per queue setting.
* @param queue name of the queue
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 71e65cb..dce3115 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -87,6 +87,7 @@ public class LeafQueue extends AbstractCSQueue {
private float absoluteUsedCapacity = 0.0f;
+ // TODO the max applications should consider label
protected int maxApplications;
protected volatile int maxApplicationsPerUser;
@@ -1915,28 +1916,6 @@ public class LeafQueue extends AbstractCSQueue {
currentResourceLimits.getLimit()));
}
- private void updateAbsoluteCapacitiesAndRelatedFields() {
- updateAbsoluteCapacities();
- CapacitySchedulerConfiguration schedulerConf = csContext.getConfiguration();
-
- // If maxApplications not set, use the system total max app, apply newly
- // calculated abs capacity of the queue.
- if (maxApplications <= 0) {
- int maxSystemApps = schedulerConf.
- getMaximumSystemApplications();
- maxApplications =
- (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity());
- }
- maxApplicationsPerUser =
- Math.min(maxApplications,
- (int) (maxApplications * (usersManager.getUserLimit() / 100.0f)
- * usersManager.getUserLimitFactor()));
-
- if (getUserLimitFactor() == -1) {
- maxApplicationsPerUser = maxApplications;
- }
- }
-
@Override
public void updateClusterResource(Resource clusterResource,
ResourceLimits currentResourceLimits) {
@@ -1944,7 +1923,14 @@ public class LeafQueue extends AbstractCSQueue {
try {
lastClusterResource = clusterResource;
- updateAbsoluteCapacitiesAndRelatedFields();
+ updateAbsoluteCapacities();
+
+ // If maxApplications not set, use the system total max app, apply newly
+ // calculated abs capacity of the queue.
+ // When add new queue, the parent queue's other children should also
+ // update the max app.
+ super.updateMaxAppRelatedField(csContext.getConfiguration(),
+ this, CommonNodeLabelsManager.NO_LABEL);
super.updateEffectiveResources(clusterResource);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index f493fe6..c403d23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -550,6 +550,63 @@ public class TestCapacitySchedulerNewQueueAutoCreation
}
@Test
+ public void testAutoQueueCreationMaxAppUpdate() throws Exception {
+ startScheduler();
+
+ // When no conf for max apps
+ LeafQueue a1 = (LeafQueue)cs.
+ getQueue("root.a.a1");
+ Assert.assertNotNull(a1);
+ Assert.assertEquals(csConf.getMaximumSystemApplications()
+ * a1.getAbsoluteCapacity(), a1.getMaxApplications(), 1);
+
+ LeafQueue b = (LeafQueue)cs.
+ getQueue("root.b");
+ Assert.assertNotNull(b);
+ Assert.assertEquals(csConf.getMaximumSystemApplications()
+ * b.getAbsoluteCapacity(), b.getMaxApplications(), 1);
+
+ createQueue("root.e");
+
+ // Make sure other children queues
+ // max app correct.
+ LeafQueue e = (LeafQueue)cs.
+ getQueue("root.e");
+ Assert.assertNotNull(e);
+ Assert.assertEquals(csConf.getMaximumSystemApplications()
+ * e.getAbsoluteCapacity(), e.getMaxApplications(), 1);
+
+ a1 = (LeafQueue)cs.
+ getQueue("root.a.a1");
+ Assert.assertNotNull(a1);
+ Assert.assertEquals(csConf.getMaximumSystemApplications()
+ * a1.getAbsoluteCapacity(), a1.getMaxApplications(), 1);
+
+ b = (LeafQueue)cs.
+ getQueue("root.b");
+ Assert.assertNotNull(b);
+ Assert.assertEquals(csConf.getMaximumSystemApplications()
+ * b.getAbsoluteCapacity(), b.getMaxApplications(), 1);
+
+ // When update global max app per queue
+ csConf.setGlobalMaximumApplicationsPerQueue(1000);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ Assert.assertEquals(1000, b.getMaxApplications());
+ Assert.assertEquals(1000, a1.getMaxApplications());
+ Assert.assertEquals(1000, e.getMaxApplications());
+
+ // when set some queue for max apps
+ csConf.setMaximumApplicationsPerQueue("root.e1", 50);
+ createQueue("root.e1");
+ LeafQueue e1 = (LeafQueue)cs.
+ getQueue("root.e1");
+ Assert.assertNotNull(e1);
+
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ Assert.assertEquals(50, e1.getMaxApplications());
+ }
+
+ @Test
public void testAutoCreateQueueIfAmbiguousQueueNames() throws Exception {
startScheduler();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org