You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2019/06/10 21:34:17 UTC
[hadoop] branch trunk updated: YARN-9569. Auto-created leaf queues
do not honor cluster-wide min/max memory/vcores. Contributed by Craig
Condit.
This is an automated email from the ASF dual-hosted git repository.
sumasai pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9191e08 YARN-9569. Auto-created leaf queues do not honor cluster-wide min/max memory/vcores. Contributed by Craig Condit.
9191e08 is described below
commit 9191e08f0ad4ebc2a3b776c4cc71d0fc5c053beb
Author: Suma Shivaprasad <su...@apache.org>
AuthorDate: Mon Jun 10 14:33:24 2019 -0700
YARN-9569. Auto-created leaf queues do not honor cluster-wide min/max memory/vcores. Contributed by Craig Condit.
---
.../capacity/AbstractManagedParentQueue.java | 8 +++++++
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 27 ++++++++++++++++++++--
.../TestCapacitySchedulerAutoQueueCreation.java | 11 ++++-----
3 files changed, 38 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
index 3e90863..7626c66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
@@ -200,6 +201,13 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
CapacitySchedulerConfiguration leafQueueConfigs = new
CapacitySchedulerConfiguration(new Configuration(false), false);
+ String prefix = YarnConfiguration.RESOURCE_TYPES + ".";
+ Map<String, String> rtProps = csContext
+ .getConfiguration().getPropsWithPrefix(prefix);
+ for (Map.Entry<String, String> entry : rtProps.entrySet()) {
+ leafQueueConfigs.set(prefix + entry.getKey(), entry.getValue());
+ }
+
SortedMap<String, String> sortedConfigs = sortCSConfigurations();
SortedMap<String, String> templateConfigs = getConfigurationsWithPrefix
(sortedConfigs, configPrefix);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index c64582e..d0cacde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.RandomUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
@@ -189,7 +190,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
@Before
public void setUp() throws Exception {
- CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@@ -494,8 +495,22 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
return queueMappings;
}
+ protected CapacitySchedulerConfiguration setupSchedulerConfiguration() {
+ Configuration schedConf = new Configuration();
+ schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ + ".vcores.minimum-allocation", 1);
+ schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ + ".vcores.maximum-allocation", 8);
+ schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ + ".memory-mb.minimum-allocation", 1024);
+ schedConf.setInt(YarnConfiguration.RESOURCE_TYPES
+ + ".memory-mb.maximum-allocation", 16384);
+
+ return new CapacitySchedulerConfiguration(schedConf);
+ }
+
protected MockRM setupSchedulerInstance() throws Exception {
- CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@@ -579,6 +594,14 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
autoCreatedLeafQueue.getMaxApplicationsPerUser());
}
+ protected void validateContainerLimits(
+ AutoCreatedLeafQueue autoCreatedLeafQueue) {
+ assertEquals(8,
+ autoCreatedLeafQueue.getMaximumAllocation().getVirtualCores());
+ assertEquals(16384,
+ autoCreatedLeafQueue.getMaximumAllocation().getMemorySize());
+ }
+
protected void validateInitialQueueEntitlement(CSQueue parentQueue, String
leafQueueName, Map<String, Float>
expectedTotalChildQueueAbsCapacityByLabel,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index f859ca7..5f32e0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -132,6 +132,7 @@ public class TestCapacitySchedulerAutoQueueCreation
expectedChildQueueAbsCapacity, accessibleNodeLabelsOnC);
validateUserAndAppLimits(autoCreatedLeafQueue, 1000, 1000);
+ validateContainerLimits(autoCreatedLeafQueue);
assertTrue(autoCreatedLeafQueue
.getOrderingPolicy() instanceof FairOrderingPolicy);
@@ -256,8 +257,7 @@ public class TestCapacitySchedulerAutoQueueCreation
throws Exception {
CapacityScheduler newCS = new CapacityScheduler();
try {
- CapacitySchedulerConfiguration newConf =
- new CapacitySchedulerConfiguration();
+ CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration();
setupQueueConfiguration(newConf);
newConf.setAutoCreateChildQueueEnabled(C, false);
@@ -285,8 +285,7 @@ public class TestCapacitySchedulerAutoQueueCreation
throws Exception {
CapacityScheduler newCS = new CapacityScheduler();
try {
- CapacitySchedulerConfiguration newConf =
- new CapacitySchedulerConfiguration();
+ CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration();
setupQueueConfiguration(newConf);
newConf.setAutoCreatedLeafQueueConfigCapacity(A1, A1_CAPACITY / 10);
newConf.setAutoCreateChildQueueEnabled(A1, true);
@@ -315,8 +314,7 @@ public class TestCapacitySchedulerAutoQueueCreation
throws Exception {
CapacityScheduler newCS = new CapacityScheduler();
try {
- CapacitySchedulerConfiguration newConf =
- new CapacitySchedulerConfiguration();
+ CapacitySchedulerConfiguration newConf = setupSchedulerConfiguration();
setupQueueConfiguration(newConf);
newConf.setAutoCreatedLeafQueueConfigCapacity(A, A_CAPACITY / 10);
newConf.setAutoCreateChildQueueEnabled(A, true);
@@ -773,6 +771,7 @@ public class TestCapacitySchedulerAutoQueueCreation
validateCapacities(user3Queue, 0.3f, 0.09f, 0.4f,0.2f);
validateUserAndAppLimits(user3Queue, 900, 900);
+ validateContainerLimits(user3Queue);
GuaranteedOrZeroCapacityOverTimePolicy autoCreatedQueueManagementPolicy =
(GuaranteedOrZeroCapacityOverTimePolicy) ((ManagedParentQueue)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org