You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/10/12 16:06:06 UTC
[hadoop] branch branch-3.3 updated: YARN-10869. CS considers only
the default maximum-allocation-mb/vcore property as a maximum when it
creates dynamic queues (#3504)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 7000458 YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504)
7000458 is described below
commit 700045896c6175913d9071c132b533871dcbc20e
Author: Benjamin Teke <br...@users.noreply.github.com>
AuthorDate: Tue Oct 12 18:05:50 2021 +0200
YARN-10869. CS considers only the default maximum-allocation-mb/vcore property as a maximum when it creates dynamic queues (#3504)
Co-authored-by: Benjamin Teke <bt...@cloudera.com>
---
.../scheduler/capacity/AbstractCSQueue.java | 50 ++++++++++++----------
.../capacity/CapacitySchedulerConfiguration.java | 14 ++++++
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 32 ++++++++++++--
.../TestCapacitySchedulerAutoQueueCreation.java | 33 +++++++++++++-
4 files changed, 102 insertions(+), 27 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fda5e16..e7d9fa5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -100,7 +100,7 @@ public abstract class AbstractCSQueue implements CSQueue {
String defaultLabelExpression;
private String multiNodeSortingPolicyName = null;
- Map<AccessType, AccessControlList> acls =
+ Map<AccessType, AccessControlList> acls =
new HashMap<AccessType, AccessControlList>();
volatile boolean reservationsContinueLooking;
private volatile boolean preemptionDisabled;
@@ -112,7 +112,7 @@ public abstract class AbstractCSQueue implements CSQueue {
volatile ResourceUsage queueUsage;
private final boolean fullPathQueueNamingPolicy = false;
-
+
// Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
// etc.
QueueCapacities queueCapacities;
@@ -134,7 +134,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected CapacityConfigType capacityConfigType =
CapacityConfigType.NONE;
- private final RecordFactory recordFactory =
+ private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
protected YarnAuthorizationProvider authorizer = null;
@@ -250,12 +250,12 @@ public abstract class AbstractCSQueue implements CSQueue {
public QueueState getState() {
return state;
}
-
+
@Override
public CSQueueMetrics getMetrics() {
return metrics;
}
-
+
@Override
public String getQueueShortName() {
return queueName;
@@ -283,7 +283,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public void setParent(CSQueue newParentQueue) {
this.parent = newParentQueue;
}
-
+
public Set<String> getAccessibleNodeLabels() {
return accessibleLabels;
}
@@ -344,7 +344,7 @@ public abstract class AbstractCSQueue implements CSQueue {
public String getDefaultNodeLabelExpression() {
return defaultLabelExpression;
}
-
+
void setupQueueConfigs(Resource clusterResource)
throws IOException {
setupQueueConfigs(clusterResource, csContext.getConfiguration());
@@ -471,8 +471,14 @@ public abstract class AbstractCSQueue implements CSQueue {
private void setupMaximumAllocation(CapacitySchedulerConfiguration csConf) {
String myQueuePath = getQueuePath();
+ /* YARN-10869: When using AutoCreatedLeafQueues, the passed configuration
+ * object is a cloned one containing only the template configs
+ * (see ManagedParentQueue#getLeafQueueConfigs). To ensure that the actual
+ * cluster maximum allocation is fetched the original config object should
+ * be used.
+ */
Resource clusterMax = ResourceUtils
- .fetchMaximumAllocationFromConfig(csConf);
+ .fetchMaximumAllocationFromConfig(this.csContext.getConfiguration());
Resource queueMax = csConf.getQueueMaximumAllocation(myQueuePath);
maximumAllocation = Resources.clone(
@@ -740,7 +746,7 @@ public abstract class AbstractCSQueue implements CSQueue {
stats.setReservedContainers(getMetrics().getReservedContainers());
return stats;
}
-
+
public Map<String, QueueConfigurations> getQueueConfigurations() {
Map<String, QueueConfigurations> queueConfigurations = new HashMap<>();
Set<String> nodeLabels = getNodeLabelsForQueue();
@@ -776,12 +782,12 @@ public abstract class AbstractCSQueue implements CSQueue {
public Resource getMaximumAllocation() {
return maximumAllocation;
}
-
+
@Private
public Resource getMinimumAllocation() {
return minimumAllocation;
}
-
+
void allocateResource(Resource clusterResource,
Resource resource, String nodePartition) {
writeLock.lock();
@@ -796,7 +802,7 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.unlock();
}
}
-
+
protected void releaseResource(Resource clusterResource,
Resource resource, String nodePartition) {
writeLock.lock();
@@ -811,12 +817,12 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.unlock();
}
}
-
+
@Private
public boolean getReservationContinueLooking() {
return reservationsContinueLooking;
}
-
+
@Private
public Map<AccessType, AccessControlList> getACLs() {
readLock.lock();
@@ -841,12 +847,12 @@ public abstract class AbstractCSQueue implements CSQueue {
public boolean getIntraQueuePreemptionDisabledInHierarchy() {
return intraQueuePreemptionDisabledInHierarchy;
}
-
+
@Private
public QueueCapacities getQueueCapacities() {
return queueCapacities;
}
-
+
@Private
public ResourceUsage getQueueResourceUsage() {
return queueUsage;
@@ -1018,7 +1024,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// all queues on this label equals to total resource with the label.
return labelManager.getResourceByLabel(nodePartition, clusterResource);
}
-
+
return Resources.none();
}
@@ -1159,7 +1165,7 @@ public abstract class AbstractCSQueue implements CSQueue {
parent.incPendingResource(nodeLabel, resourceToInc);
}
}
-
+
@Override
public void decPendingResource(String nodeLabel, Resource resourceToDec) {
if (nodeLabel == null) {
@@ -1171,7 +1177,7 @@ public abstract class AbstractCSQueue implements CSQueue {
parent.decPendingResource(nodeLabel, resourceToDec);
}
}
-
+
@Override
public void incUsedResource(String nodeLabel, Resource resourceToInc,
SchedulerApplicationAttempt application) {
@@ -1206,14 +1212,14 @@ public abstract class AbstractCSQueue implements CSQueue {
/**
* Return if the queue has pending resource on given nodePartition and
- * schedulingMode.
+ * schedulingMode.
*/
- boolean hasPendingResourceRequest(String nodePartition,
+ boolean hasPendingResourceRequest(String nodePartition,
Resource cluster, SchedulingMode schedulingMode) {
return SchedulerUtils.hasPendingResourceRequest(resourceCalculator,
queueUsage, nodePartition, cluster, schedulingMode);
}
-
+
public boolean accessibleToPartition(String nodePartition) {
// if queue's label is *, it can access any node
if (accessibleLabels != null
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 9d4b6e2..b973406 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -953,6 +953,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
+ public void setQueueMaximumAllocation(String queue, String maximumAllocation) {
+ String queuePrefix = getQueuePrefix(queue);
+ set(queuePrefix + MAXIMUM_ALLOCATION, maximumAllocation);
+ }
+
public long getQueueMaximumAllocationMb(String queue) {
String queuePrefix = getQueuePrefix(queue);
return getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED);
@@ -2061,6 +2066,15 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
setDefaultNodeLabelExpression(leafQueueConfPrefix, expression);
}
+ @Private
+ @VisibleForTesting
+ public void setAutoCreatedLeafQueueConfigMaximumAllocation(String
+ queuePath, String expression) {
+ String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ queuePath);
+ setQueueMaximumAllocation(leafQueueConfPrefix, expression);
+ }
+
public static String getUnits(String resourceValue) {
String units;
for (int i = 0; i < resourceValue.length(); i++) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index cc6a71b..b06429a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -385,6 +385,9 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
conf.setAutoCreatedLeafQueueConfigMaxCapacity(C, 100.0f);
conf.setAutoCreatedLeafQueueConfigUserLimit(C, 100);
conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f);
+ conf.setAutoCreatedLeafQueueConfigUserLimitFactor(C, 3.0f);
+ conf.setAutoCreatedLeafQueueConfigMaximumAllocation(C,
+ "memory-mb=10240,vcores=6");
conf.setAutoCreatedLeafQueueTemplateCapacityByLabel(C, NODEL_LABEL_GPU,
NODE_LABEL_GPU_TEMPLATE_CAPACITY);
@@ -540,9 +543,31 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ ".memory-mb.maximum-allocation", 16384);
+
return new CapacitySchedulerConfiguration(schedConf);
}
+ protected void setSchedulerMinMaxAllocation(CapacitySchedulerConfiguration conf) {
+ unsetMinMaxAllocation(conf);
+
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, 1);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 8);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 18384);
+
+ }
+
+ private void unsetMinMaxAllocation(CapacitySchedulerConfiguration conf) {
+ conf.unset(YarnConfiguration.RESOURCE_TYPES
+ + ".vcores.minimum-allocation");
+ conf.unset(YarnConfiguration.RESOURCE_TYPES
+ + ".vcores.maximum-allocation");
+ conf.unset(YarnConfiguration.RESOURCE_TYPES
+ + ".memory-mb.minimum-allocation");
+ conf.unset(YarnConfiguration.RESOURCE_TYPES
+ + ".memory-mb.maximum-allocation");
+ }
+
protected MockRM setupSchedulerInstance() throws Exception {
if (mockRM != null) {
@@ -640,10 +665,11 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
}
protected void validateContainerLimits(
- AutoCreatedLeafQueue autoCreatedLeafQueue) {
- assertEquals(8,
+ AutoCreatedLeafQueue autoCreatedLeafQueue, int vCoreLimit,
+ long memorySize) {
+ assertEquals(vCoreLimit,
autoCreatedLeafQueue.getMaximumAllocation().getVirtualCores());
- assertEquals(16384,
+ assertEquals(memorySize,
autoCreatedLeafQueue.getMaximumAllocation().getMemorySize());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index 596cca1..a0ad435 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -137,7 +137,7 @@ public class TestCapacitySchedulerAutoQueueCreation
expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC);
validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000);
- validateContainerLimits(autoCreatedLeafQueue);
+ validateContainerLimits(autoCreatedLeafQueue, 6, 10240);
assertTrue(autoCreatedLeafQueue
.getOrderingPolicy() instanceof FairOrderingPolicy);
@@ -167,6 +167,35 @@ public class TestCapacitySchedulerAutoQueueCreation
}
@Test(timeout = 20000)
+ public void testAutoCreateLeafQueueCreationSchedulerMaximumAllocation()
+ throws Exception {
+ try {
+ // Check the minimum/maximum allocation settings via the
+ // yarn.scheduler.minimum/maximum-allocation-mb/vcore property
+ setSchedulerMinMaxAllocation(cs.getConfiguration());
+ cs.getConfiguration().setAutoCreatedLeafQueueConfigMaximumAllocation(C,
+ "memory-mb=18384,vcores=8");
+ cs.reinitialize(cs.getConfiguration(), mockRM.getRMContext());
+
+ // submit an app
+ submitApp(mockRM, cs.getQueue(PARENT_QUEUE), USER0, USER0, 1, 1);
+
+ // check preconditions
+ List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
+ assertEquals(1, appsInC.size());
+ assertNotNull(cs.getQueue(USER0));
+
+ AutoCreatedLeafQueue autoCreatedLeafQueue =
+ (AutoCreatedLeafQueue) cs.getQueue(USER0);
+
+ validateContainerLimits(autoCreatedLeafQueue, 8, 18384);
+ } finally {
+ cleanupQueue(USER0);
+ cleanupQueue(TEST_GROUPUSER);
+ }
+ }
+
+ @Test(timeout = 20000)
public void testAutoCreateLeafQueueCreationUsingFullParentPath()
throws Exception {
@@ -825,7 +854,7 @@ public class TestCapacitySchedulerAutoQueueCreation
validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f);
validateUserAndAppLimits(user3Queue, 900, 900);
- validateContainerLimits(user3Queue);
+ validateContainerLimits(user3Queue, 6, 10240);
GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
(GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org