You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bt...@apache.org on 2024/01/26 13:43:15 UTC
(hadoop) branch trunk updated: YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup (#6499)
This is an automated email from the ASF dual-hosted git repository.
bteke pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 40467519399c YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup (#6499)
40467519399c is described below
commit 40467519399c0e51beb25d6e557c55859382e8cf
Author: Peter Szucs <11...@users.noreply.github.com>
AuthorDate: Fri Jan 26 14:42:59 2024 +0100
YARN-11041. Replace all occurences of queuePath with the new QueuePath class - followup (#6499)
---
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 5 +-
.../policygenerator/TestPolicyGenerator.java | 17 +-
.../reservation/AbstractReservationSystem.java | 9 +-
.../reservation/CapacityOverTimePolicy.java | 4 +-
.../ReservationSchedulerConfiguration.java | 23 +-
.../planning/SimpleCapacityReplanner.java | 3 +-
.../capacity/AbstractAutoCreatedLeafQueue.java | 18 +-
.../scheduler/capacity/AbstractCSQueue.java | 28 +-
.../scheduler/capacity/AbstractLeafQueue.java | 26 +-
.../scheduler/capacity/AbstractParentQueue.java | 10 +-
.../capacity/AutoCreatedQueueTemplate.java | 84 ++---
.../capacity/CSQueuePreemptionSettings.java | 8 +-
.../scheduler/capacity/CSQueueUtils.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../capacity/CapacitySchedulerConfigValidator.java | 4 +-
.../capacity/CapacitySchedulerConfiguration.java | 366 ++++++++++-----------
.../capacity/CapacitySchedulerQueueManager.java | 43 ++-
.../scheduler/capacity/ManagedParentQueue.java | 47 ++-
.../scheduler/capacity/ParentQueue.java | 2 +-
.../scheduler/capacity/PlanQueue.java | 2 +-
.../capacity/QueueAllocationSettings.java | 2 +-
.../capacity/QueueAppLifetimeAndLimitSettings.java | 6 +-
.../capacity/QueueNodeLabelsSettings.java | 6 +-
.../scheduler/capacity/QueuePath.java | 69 +++-
.../scheduler/capacity/QueuePrefixes.java | 57 ++++
.../scheduler/capacity/QueueStateHelper.java | 2 +-
.../scheduler/capacity/UserWeights.java | 4 +-
.../conf/ConfigurationUpdateAssembler.java | 80 +++--
.../capacity/conf/QueueCapacityConfigParser.java | 6 +-
.../scheduler/fair/AllocationConfiguration.java | 27 +-
.../scheduler/fair/FairScheduler.java | 13 +-
.../converter/FSConfigToCSConfigConverter.java | 5 +-
.../scheduler/fair/converter/FSQueueConverter.java | 28 +-
.../fair/converter/QueuePlacementConverter.java | 21 +-
.../weightconversion/WeightToPercentConverter.java | 5 +-
.../weightconversion/WeightToWeightConverter.java | 8 +-
.../security/CapacityReservationsACLsManager.java | 2 +-
.../security/FairReservationsACLsManager.java | 3 +-
.../webapp/dao/CapacitySchedulerInfo.java | 3 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 14 +-
.../webapp/dao/FairSchedulerQueueInfo.java | 7 +-
.../webapp/dao/LeafQueueTemplateInfo.java | 7 +-
.../yarn/server/resourcemanager/ACLsTestBase.java | 9 +
.../resourcemanager/ReservationACLsTestBase.java | 22 +-
.../server/resourcemanager/TestAppManager.java | 138 ++++----
.../TestApplicationMasterServiceCapacity.java | 38 ++-
.../TestWorkPreservingRMRestart.java | 92 +++---
.../TestProportionalCapacityPreemptionPolicy.java | 31 +-
...lCapacityPreemptionPolicyForNodePartitions.java | 3 +-
...ortionalCapacityPreemptionPolicyIntraQueue.java | 14 +-
.../capacity/mockframework/MockQueueHierarchy.java | 3 +-
.../nodelabels/TestRMNodeLabelsManager.java | 16 +-
.../reservation/ReservationSystemTestUtil.java | 70 ++--
.../TestCapacitySchedulerPlanFollower.java | 9 +-
.../planning/TestSimpleCapacityReplanner.java | 3 +-
.../rmapp/TestApplicationLifetimeMonitor.java | 59 ++--
.../CapacitySchedulerConfigGeneratorForTest.java | 16 +-
.../CapacitySchedulerQueueCalculationTestBase.java | 33 +-
.../capacity/CapacitySchedulerQueueHelpers.java | 88 +++--
.../scheduler/capacity/QueueAssertionBuilder.java | 18 +-
.../TestAbsoluteResourceConfiguration.java | 74 +++--
.../TestAbsoluteResourceWithAutoQueue.java | 45 +--
.../scheduler/capacity/TestApplicationLimits.java | 113 ++++---
.../capacity/TestApplicationLimitsByPartition.java | 23 +-
.../TestApplicationPriorityACLConfiguration.java | 40 ++-
.../capacity/TestApplicationPriorityACLs.java | 12 +-
.../TestAutoCreatedQueueDeletionPolicy.java | 23 +-
.../capacity/TestAutoCreatedQueueTemplate.java | 156 +++++----
.../capacity/TestCSAllocateCustomResource.java | 11 +-
.../capacity/TestCSMaxRunningAppsEnforcer.java | 14 +-
.../scheduler/capacity/TestCapacityScheduler.java | 69 ++--
.../TestCapacitySchedulerAmbiguousLeafs.java | 38 ++-
.../capacity/TestCapacitySchedulerApps.java | 9 +-
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 78 +++--
...apacitySchedulerAutoCreatedQueuePreemption.java | 10 +-
.../TestCapacitySchedulerAutoQueueCreation.java | 19 +-
.../TestCapacitySchedulerConfigValidator.java | 6 +-
.../TestCapacitySchedulerConfiguration.java | 26 +-
.../TestCapacitySchedulerDynamicBehavior.java | 4 +-
.../TestCapacitySchedulerLazyPreemption.java | 3 +-
.../capacity/TestCapacitySchedulerMultiNodes.java | 9 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 157 +++++----
...lerNewQueueAutoCreationWithCapacityVectors.java | 38 +--
.../TestCapacitySchedulerNodeLabelUpdate.java | 52 +--
.../capacity/TestCapacitySchedulerNodes.java | 4 +-
.../capacity/TestCapacitySchedulerOvercommit.java | 15 +-
.../capacity/TestCapacitySchedulerPerf.java | 13 +-
.../capacity/TestCapacitySchedulerQueueACLs.java | 44 ++-
.../capacity/TestCapacitySchedulerQueues.java | 67 ++--
.../TestCapacitySchedulerSurgicalPreemption.java | 57 ++--
.../capacity/TestCapacitySchedulerWeightMode.java | 66 ++--
...estCapacitySchedulerWithMultiResourceTypes.java | 49 +--
...stCapacitySchedulerWorkflowPriorityMapping.java | 16 +-
.../scheduler/capacity/TestChildQueueOrder.java | 11 +-
.../capacity/TestContainerAllocation.java | 49 +--
.../scheduler/capacity/TestLeafQueue.java | 186 ++++++-----
.../TestMixedQueueResourceCalculation.java | 6 +-
.../capacity/TestNodeLabelContainerAllocation.java | 244 +++++++-------
.../scheduler/capacity/TestParentQueue.java | 112 +++----
.../scheduler/capacity/TestQueueMappings.java | 9 +-
.../scheduler/capacity/TestQueueParsing.java | 237 ++++++-------
.../scheduler/capacity/TestQueuePath.java | 152 +++++++--
.../scheduler/capacity/TestQueueState.java | 21 +-
.../scheduler/capacity/TestQueueStateManager.java | 15 +-
.../scheduler/capacity/TestReservations.java | 37 +--
.../TestUniformQueueResourceCalculation.java | 12 +-
.../scheduler/capacity/TestUtils.java | 56 ++--
.../conf/TestConfigurationUpdateAssembler.java | 3 +-
.../conf/TestMutableCSConfigurationProvider.java | 12 +-
.../conf/TestQueueCapacityConfigParser.java | 37 ++-
.../constraint/TestPlacementProcessor.java | 13 +-
.../fair/TestAllocationFileLoaderService.java | 25 +-
.../converter/TestFSConfigToCSConfigConverter.java | 53 +--
.../fair/converter/TestFSQueueConverter.java | 148 +++++----
.../converter/TestQueuePlacementConverter.java | 3 +-
.../TestWeightToPercentageConverter.java | 34 +-
.../TestWeightToWeightConverter.java | 24 +-
.../scheduler/fifo/TestFifoScheduler.java | 6 +-
.../scheduler/policy/TestFairOrderingPolicy.java | 4 +-
.../resourcemanager/webapp/TestRMWebApp.java | 80 ++---
.../webapp/TestRMWebServiceAppsNodelabel.java | 15 +-
.../webapp/TestRMWebServicesAppsModification.java | 40 ++-
.../webapp/TestRMWebServicesCapacitySched.java | 7 +-
.../TestRMWebServicesConfigurationMutation.java | 113 ++++---
.../TestRMWebServicesForCSWithPartitions.java | 76 +++--
.../webapp/TestRMWebServicesReservation.java | 10 +-
...esSchedulerActivitiesWithMultiNodesEnabled.java | 17 +-
.../router/webapp/BaseRouterWebServicesTest.java | 5 +
.../webapp/MockDefaultRequestInterceptorREST.java | 12 +-
.../webapp/TestableFederationInterceptorREST.java | 13 +-
130 files changed, 2718 insertions(+), 2279 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index 4df20da68ccc..830fb8d5669f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfigurationProvider;
@@ -180,9 +181,9 @@ public class TestSchedConfCLI extends JerseyTestBase {
private static void setupQueueConfiguration(
CapacitySchedulerConfiguration config) {
- config.setQueues(CapacitySchedulerConfiguration.ROOT,
+ config.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT),
new String[]{"testqueue"});
- String a = CapacitySchedulerConfiguration.ROOT + ".testqueue";
+ QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".testqueue");
config.setCapacity(a, 100f);
config.setMaximumCapacity(a, 100f);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java
index 446eeee2cd92..86523d839276 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/policygenerator/TestPolicyGenerator.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
@@ -250,13 +251,13 @@ public class TestPolicyGenerator {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- final String a = CapacitySchedulerConfiguration.ROOT + ".a";
- final String b = CapacitySchedulerConfiguration.ROOT + ".b";
- final String a1 = a + ".a1";
- final String a2 = a + ".a2";
- final String b1 = b + ".b1";
- final String b2 = b + ".b2";
- final String b3 = b + ".b3";
+ final QueuePath a = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".a");
+ final QueuePath b = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".b");
+ final QueuePath a1 = new QueuePath(a + ".a1");
+ final QueuePath a2 = new QueuePath(a + ".a2");
+ final QueuePath b1 = new QueuePath(b + ".b1");
+ final QueuePath b2 = new QueuePath(b + ".b2");
+ final QueuePath b3 = new QueuePath(b + ".b3");
float aCapacity = 10.5f;
float bCapacity = 89.5f;
float a1Capacity = 30;
@@ -266,7 +267,7 @@ public class TestPolicyGenerator {
float b3Capacity = 20;
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT),
new String[] {"a", "b"});
csConf.setCapacity(a, aCapacity);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index d9e4be9e5225..b0b41f9697db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.Reserv
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.CapacityReservationsACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.FairReservationsACLsManager;
@@ -430,7 +431,7 @@ public abstract class AbstractReservationSystem extends AbstractService
Plan plan = new InMemoryPlan(getRootQueueMetrics(), adPolicy,
getAgent(planQueuePath), totCap, planStepSize, rescCalc, minAllocation,
maxAllocation, planQueueName, getReplanner(planQueuePath),
- getReservationSchedulerConfiguration().getMoveOnExpiry(planQueuePath),
+ getReservationSchedulerConfiguration().getMoveOnExpiry(new QueuePath(planQueuePath)),
maxPeriodicity, rmContext);
LOG.info("Initialized plan {} based on reservable queue {}",
plan.toString(), planQueueName);
@@ -440,7 +441,7 @@ public abstract class AbstractReservationSystem extends AbstractService
protected Planner getReplanner(String planQueueName) {
ReservationSchedulerConfiguration reservationConfig =
getReservationSchedulerConfiguration();
- String plannerClassName = reservationConfig.getReplanner(planQueueName);
+ String plannerClassName = reservationConfig.getReplanner(new QueuePath(planQueueName));
LOG.info("Using Replanner: " + plannerClassName + " for queue: "
+ planQueueName);
try {
@@ -463,7 +464,7 @@ public abstract class AbstractReservationSystem extends AbstractService
protected ReservationAgent getAgent(String queueName) {
ReservationSchedulerConfiguration reservationConfig =
getReservationSchedulerConfiguration();
- String agentClassName = reservationConfig.getReservationAgent(queueName);
+ String agentClassName = reservationConfig.getReservationAgent(new QueuePath(queueName));
LOG.info("Using Agent: " + agentClassName + " for queue: " + queueName);
try {
Class<?> agentClazz = conf.getClassByName(agentClassName);
@@ -487,7 +488,7 @@ public abstract class AbstractReservationSystem extends AbstractService
ReservationSchedulerConfiguration reservationConfig =
getReservationSchedulerConfiguration();
String admissionPolicyClassName =
- reservationConfig.getReservationAdmissionPolicy(queueName);
+ reservationConfig.getReservationAdmissionPolicy(new QueuePath(queueName));
LOG.info("Using AdmissionPolicy: " + admissionPolicyClassName
+ " for queue: " + queueName);
try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
index 1f3f9bc4ba6b..db9794d6f688 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation.RLEOperator;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningQuotaException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.Map;
@@ -57,9 +58,10 @@ public class CapacityOverTimePolicy extends NoOverCommitPolicy {
private float maxAvg;
@Override
- public void init(String reservationQueuePath,
+ public void init(String reservationQueue,
ReservationSchedulerConfiguration conf) {
this.conf = conf;
+ QueuePath reservationQueuePath = new QueuePath(reservationQueue);
validWindow = this.conf.getReservationWindow(reservationQueuePath);
maxInst = this.conf.getInstantaneousMaxCapacity(reservationQueuePath) / 100;
maxAvg = this.conf.getAverageCapacity(reservationQueuePath) / 100;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java
index 740b88c40c45..d6bf1ce81336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSchedulerConfiguration.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.ReservationACL;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import java.util.Map;
@@ -69,7 +70,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return true if the queue participates in reservation based scheduling
*/
- public abstract boolean isReservable(String queue);
+ public abstract boolean isReservable(QueuePath queue);
/**
* Gets a map containing the {@link AccessControlList} of users for each
@@ -80,7 +81,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* which contains a list of users that have the specified permission level.
*/
public abstract Map<ReservationACL, AccessControlList> getReservationAcls(
- String queue);
+ QueuePath queue);
/**
* Gets the length of time in milliseconds for which the {@link SharingPolicy}
@@ -89,7 +90,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @return length in time in milliseconds for which to check the
* {@link SharingPolicy}
*/
- public long getReservationWindow(String queue) {
+ public long getReservationWindow(QueuePath queue) {
return DEFAULT_RESERVATION_WINDOW;
}
@@ -100,7 +101,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return average capacity allowed by the {@link SharingPolicy}
*/
- public float getAverageCapacity(String queue) {
+ public float getAverageCapacity(QueuePath queue) {
return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER;
}
@@ -109,7 +110,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return maximum allowed capacity at any time
*/
- public float getInstantaneousMaxCapacity(String queue) {
+ public float getInstantaneousMaxCapacity(QueuePath queue) {
return DEFAULT_CAPACITY_OVER_TIME_MULTIPLIER;
}
@@ -118,7 +119,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return the class name of the {@link SharingPolicy}
*/
- public String getReservationAdmissionPolicy(String queue) {
+ public String getReservationAdmissionPolicy(QueuePath queue) {
return DEFAULT_RESERVATION_ADMISSION_POLICY;
}
@@ -128,7 +129,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return the class name of the {@code ReservationAgent}
*/
- public String getReservationAgent(String queue) {
+ public String getReservationAgent(QueuePath queue) {
return DEFAULT_RESERVATION_AGENT_NAME;
}
@@ -137,7 +138,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queuePath name of the queue
* @return true if reservation queues should be visible
*/
- public boolean getShowReservationAsQueues(String queuePath) {
+ public boolean getShowReservationAsQueues(QueuePath queuePath) {
return DEFAULT_SHOW_RESERVATIONS_AS_QUEUES;
}
@@ -147,7 +148,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return the class name of the {@code Planner}
*/
- public String getReplanner(String queue) {
+ public String getReplanner(QueuePath queue) {
return DEFAULT_RESERVATION_PLANNER_NAME;
}
@@ -158,7 +159,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @return true if application should be moved, false if they need to be
* killed
*/
- public boolean getMoveOnExpiry(String queue) {
+ public boolean getMoveOnExpiry(QueuePath queue) {
return DEFAULT_RESERVATION_MOVE_ON_EXPIRY;
}
@@ -168,7 +169,7 @@ public abstract class ReservationSchedulerConfiguration extends Configuration {
* @param queue name of the queue
* @return the time in milliseconds for which to check constraints
*/
- public long getEnforcementWindow(String queue) {
+ public long getEnforcementWindow(QueuePath queue) {
return DEFAULT_RESERVATION_ENFORCEMENT_WINDOW;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java
index 38880aeed524..f14a87ed8cd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/SimpleCapacityReplanner.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
@@ -72,7 +73,7 @@ public class SimpleCapacityReplanner implements Planner {
@Override
public void init(String planQueueName,
ReservationSchedulerConfiguration conf) {
- this.lengthOfCheckZone = conf.getEnforcementWindow(planQueueName);
+ this.lengthOfCheckZone = conf.getEnforcementWindow(new QueuePath(planQueueName));
}
@Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
index 112ca7b4fd48..c056396fe172 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
@@ -61,26 +61,26 @@ public class AbstractAutoCreatedLeafQueue extends AbstractLeafQueue {
}
@Override
- protected Resource getMinimumAbsoluteResource(String queuePath,
+ protected Resource getMinimumAbsoluteResource(QueuePath queuePath,
String label) {
- return super.getMinimumAbsoluteResource(queueContext.getConfiguration()
- .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
+ return super.getMinimumAbsoluteResource(QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()),
label);
}
@Override
- protected Resource getMaximumAbsoluteResource(String queuePath,
+ protected Resource getMaximumAbsoluteResource(QueuePath queuePath,
String label) {
- return super.getMaximumAbsoluteResource(queueContext.getConfiguration()
- .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
+ return super.getMaximumAbsoluteResource(QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()),
label);
}
@Override
- protected boolean checkConfigTypeIsAbsoluteResource(String queuePath,
+ protected boolean checkConfigTypeIsAbsoluteResource(QueuePath queuePath,
String label) {
- return super.checkConfigTypeIsAbsoluteResource(queueContext.getConfiguration()
- .getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
+ return super.checkConfigTypeIsAbsoluteResource(QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(this.getParent().getQueuePathObject()),
label);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index a84a68ac409a..1bdaebb3d89e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -346,7 +346,7 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.lock();
try {
CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
- this.acls = configuration.getAcls(getQueuePath());
+ this.acls = configuration.getAcls(getQueuePathObject());
if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
parseAndSetDynamicTemplates();
@@ -367,7 +367,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// Setup queue's maximumAllocation respecting the global
// and the queue settings
- this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePath(),
+ this.queueAllocationSettings.setupMaximumAllocation(configuration, getQueuePathObject(),
parent);
// Initialize the queue state based on previous state, configured state
@@ -382,10 +382,10 @@ public abstract class AbstractCSQueue implements CSQueue {
configuration.getReservationContinueLook();
this.configuredCapacityVectors = configuration
- .parseConfiguredResourceVector(queuePath.getFullPath(),
+ .parseConfiguredResourceVector(queuePath,
this.queueNodeLabelsSettings.getConfiguredNodeLabels());
this.configuredMaxCapacityVectors = configuration
- .parseConfiguredMaximumCapacityVector(queuePath.getFullPath(),
+ .parseConfiguredMaximumCapacityVector(queuePath,
this.queueNodeLabelsSettings.getConfiguredNodeLabels(),
QueueCapacityVector.newInstance());
@@ -420,11 +420,11 @@ public abstract class AbstractCSQueue implements CSQueue {
// Store preemption settings
this.preemptionSettings = new CSQueuePreemptionSettings(this, configuration);
this.priority = configuration.getQueuePriority(
- getQueuePath());
+ getQueuePathObject());
// Update multi-node sorting algorithm for scheduling as configured.
setMultiNodeSortingPolicyName(
- configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePath()));
+ configuration.getMultiNodesSortingAlgorithmPolicy(getQueuePathObject()));
// Setup application related limits
this.queueAppLifetimeSettings = new QueueAppLifetimeAndLimitSettings(configuration,
@@ -440,7 +440,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void parseAndSetDynamicTemplates() {
// Set the template properties from the parent to the queuepath of the child
((AbstractParentQueue) parent).getAutoCreatedQueueTemplate()
- .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePath(),
+ .setTemplateEntriesForChild(queueContext.getConfiguration(), getQueuePathObject(),
this instanceof AbstractLeafQueue);
String parentTemplate = String.format("%s.%s", parent.getQueuePath(),
@@ -488,21 +488,21 @@ public abstract class AbstractCSQueue implements CSQueue {
// Insert this queue's userWeights, overriding parent's userWeights if
// there is an overlap.
unionInheritedWeights.addFrom(
- queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePath()));
+ queueContext.getConfiguration().getAllUserWeightsForQueue(getQueuePathObject()));
return unionInheritedWeights;
}
- protected Resource getMinimumAbsoluteResource(String queuePath, String label) {
+ protected Resource getMinimumAbsoluteResource(QueuePath queuePath, String label) {
return queueContext.getConfiguration()
.getMinimumResourceRequirement(label, queuePath, resourceTypes);
}
- protected Resource getMaximumAbsoluteResource(String queuePath, String label) {
+ protected Resource getMaximumAbsoluteResource(QueuePath queuePath, String label) {
return queueContext.getConfiguration()
.getMaximumResourceRequirement(label, queuePath, resourceTypes);
}
- protected boolean checkConfigTypeIsAbsoluteResource(String queuePath,
+ protected boolean checkConfigTypeIsAbsoluteResource(QueuePath queuePath,
String label) {
return queueContext.getConfiguration().checkConfigTypeIsAbsoluteResource(label,
queuePath, resourceTypes);
@@ -518,7 +518,7 @@ public abstract class AbstractCSQueue implements CSQueue {
if (queueContext.getConfiguration().isLegacyQueueMode()) {
localType = checkConfigTypeIsAbsoluteResource(
- getQueuePath(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE
+ getQueuePathObject(), label) ? CapacityConfigType.ABSOLUTE_RESOURCE
: CapacityConfigType.PERCENTAGE;
} else {
// TODO: revisit this later
@@ -556,8 +556,8 @@ public abstract class AbstractCSQueue implements CSQueue {
*/
protected void updateConfigurableResourceLimits(Resource clusterResource) {
for (String label : queueNodeLabelsSettings.getConfiguredNodeLabels()) {
- final Resource minResource = getMinimumAbsoluteResource(getQueuePath(), label);
- Resource maxResource = getMaximumAbsoluteResource(getQueuePath(), label);
+ final Resource minResource = getMinimumAbsoluteResource(getQueuePathObject(), label);
+ Resource maxResource = getMaximumAbsoluteResource(getQueuePathObject(), label);
if (parent != null) {
final Resource parentMax = parent.getQueueResourceQuotas()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
index 280d3d182fb2..e40b8aaeb0bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractLeafQueue.java
@@ -193,16 +193,16 @@ public class AbstractLeafQueue extends AbstractCSQueue {
setQueueResourceLimitsInfo(clusterResource);
setOrderingPolicy(
- configuration.<FiCaSchedulerApp>getAppOrderingPolicy(getQueuePath()));
+ configuration.<FiCaSchedulerApp>getAppOrderingPolicy(getQueuePathObject()));
- usersManager.setUserLimit(configuration.getUserLimit(getQueuePath()));
- usersManager.setUserLimitFactor(configuration.getUserLimitFactor(getQueuePath()));
+ usersManager.setUserLimit(configuration.getUserLimit(getQueuePathObject()));
+ usersManager.setUserLimitFactor(configuration.getUserLimitFactor(getQueuePathObject()));
maxAMResourcePerQueuePercent =
configuration.getMaximumApplicationMasterResourcePerQueuePercent(
- getQueuePath());
+ getQueuePathObject());
- maxApplications = configuration.getMaximumApplicationsPerQueue(getQueuePath());
+ maxApplications = configuration.getMaximumApplicationsPerQueue(getQueuePathObject());
if (maxApplications < 0) {
int maxGlobalPerQueueApps =
configuration.getGlobalMaximumApplicationsPerQueue();
@@ -211,7 +211,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
}
- priorityAcls = configuration.getPriorityAcls(getQueuePath(),
+ priorityAcls = configuration.getPriorityAcls(getQueuePathObject(),
configuration.getClusterLevelApplicationMaxPriority());
Set<String> accessibleNodeLabels = this.queueNodeLabelsSettings.getAccessibleNodeLabels();
@@ -254,10 +254,10 @@ public class AbstractLeafQueue extends AbstractCSQueue {
}
defaultAppPriorityPerQueue = Priority.newInstance(
- configuration.getDefaultApplicationPriorityConfPerQueue(getQueuePath()));
+ configuration.getDefaultApplicationPriorityConfPerQueue(getQueuePathObject()));
// Validate leaf queue's user's weights.
- float queueUserLimit = Math.min(100.0f, configuration.getUserLimit(getQueuePath()));
+ float queueUserLimit = Math.min(100.0f, configuration.getUserLimit(getQueuePathObject()));
getUserWeights().validateForLeafQueue(queueUserLimit, getQueuePath());
usersManager.updateUserWeights();
@@ -1703,10 +1703,10 @@ public class AbstractLeafQueue extends AbstractCSQueue {
@Override
protected void parseAndSetDynamicTemplates() {
// set to -1, to disable it
- queueContext.getConfiguration().setUserLimitFactor(getQueuePath(), -1);
+ queueContext.getConfiguration().setUserLimitFactor(getQueuePathObject(), -1);
// Set Max AM percentage to a higher value
queueContext.getConfiguration().setMaximumApplicationMasterResourcePerQueuePercent(
- getQueuePath(), 1f);
+ getQueuePathObject(), 1f);
super.parseAndSetDynamicTemplates();
}
@@ -1716,7 +1716,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
if (parent instanceof AbstractManagedParentQueue) {
acls.putAll(queueContext.getConfiguration().getACLsForLegacyAutoCreatedLeafQueue(
- parent.getQueuePath()));
+ parent.getQueuePathObject()));
} else if (parent instanceof ParentQueue) {
acls.putAll(getACLsForFlexibleAutoCreatedLeafQueue(
((ParentQueue) parent).getAutoCreatedQueueTemplate()));
@@ -2432,7 +2432,7 @@ public class AbstractLeafQueue extends AbstractCSQueue {
void updateMaximumApplications(boolean absoluteCapacityIsReadyForUse) {
CapacitySchedulerConfiguration configuration = queueContext.getConfiguration();
- int maxAppsForQueue = configuration.getMaximumApplicationsPerQueue(getQueuePath());
+ int maxAppsForQueue = configuration.getMaximumApplicationsPerQueue(getQueuePathObject());
int maxDefaultPerQueueApps = configuration.getGlobalMaximumApplicationsPerQueue();
int maxSystemApps = configuration.getMaximumSystemApplications();
@@ -2566,6 +2566,6 @@ public class AbstractLeafQueue extends AbstractCSQueue {
public boolean isEligibleForAutoDeletion() {
return isDynamicQueue() && getNumApplications() == 0
&& queueContext.getConfiguration().
- isAutoExpiredDeletionEnabled(this.getQueuePath());
+ isAutoExpiredDeletionEnabled(this.getQueuePathObject());
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java
index 367abcb5375b..9cb545d37808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractParentQueue.java
@@ -134,7 +134,7 @@ public abstract class AbstractParentQueue extends AbstractCSQueue {
this.childQueues = new ArrayList<>();
this.allowZeroCapacitySum =
queueContext.getConfiguration()
- .getAllowZeroCapacitySum(getQueuePath());
+ .getAllowZeroCapacitySum(getQueuePathObject());
}
@@ -168,7 +168,7 @@ public abstract class AbstractParentQueue extends AbstractCSQueue {
// Initialize queue ordering policy
queueOrderingPolicy = configuration.getQueueOrderingPolicy(
- getQueuePath(), parent == null ?
+ getQueuePathObject(), parent == null ?
null :
((AbstractParentQueue) parent).getQueueOrderingPolicyConfigName());
queueOrderingPolicy.setQueues(childQueues);
@@ -229,7 +229,7 @@ public abstract class AbstractParentQueue extends AbstractCSQueue {
"{Queue=" + queue.getQueuePath() + ", label=" + nodeLabel
+ " uses weight mode}. ");
}
- if (checkConfigTypeIsAbsoluteResource(queue.getQueuePath(), nodeLabel)) {
+ if (checkConfigTypeIsAbsoluteResource(queue.getQueuePathObject(), nodeLabel)) {
absoluteMinResSet = true;
// There's a special handling: when absolute resource is configured,
// capacity will be calculated (and set) for UI/metrics purposes, so
@@ -549,7 +549,7 @@ public abstract class AbstractParentQueue extends AbstractCSQueue {
*/
public boolean isEligibleForAutoQueueCreation() {
return isDynamicQueue() || queueContext.getConfiguration().
- isAutoQueueCreationV2Enabled(getQueuePath());
+ isAutoQueueCreationV2Enabled(getQueuePathObject());
}
@Override
@@ -1617,7 +1617,7 @@ public abstract class AbstractParentQueue extends AbstractCSQueue {
public boolean isEligibleForAutoDeletion() {
return isDynamicQueue() && getChildQueues().size() == 0 &&
queueContext.getConfiguration().
- isAutoExpiredDeletionEnabled(this.getQueuePath());
+ isAutoExpiredDeletionEnabled(this.getQueuePathObject());
}
public AutoCreatedQueueTemplate getAutoCreatedQueueTemplate() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
index bbeca4b07245..d31f4307cc73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedQueueTemplate.java
@@ -25,10 +25,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hadoop.util.Lists;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_QUEUE_CREATION_V2_PREFIX;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix;
/**
* A handler for storing and setting auto created queue template settings.
@@ -41,7 +39,7 @@ public class AutoCreatedQueueTemplate {
public static final String AUTO_QUEUE_PARENT_TEMPLATE_PREFIX =
AUTO_QUEUE_CREATION_V2_PREFIX + "parent-template.";
- private static final String WILDCARD_QUEUE = "*";
+ public static final String WILDCARD_QUEUE = "*";
private final Map<String, String> templateProperties = new HashMap<>();
private final Map<String, String> leafOnlyProperties = new HashMap<>();
@@ -53,9 +51,8 @@ public class AutoCreatedQueueTemplate {
}
@VisibleForTesting
- public static String getAutoQueueTemplatePrefix(String queue) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queue)
- + AUTO_QUEUE_TEMPLATE_PREFIX;
+ public static String getAutoQueueTemplatePrefix(QueuePath queuePath) {
+ return getQueuePrefix(queuePath) + AUTO_QUEUE_TEMPLATE_PREFIX;
}
/**
@@ -89,7 +86,7 @@ public class AutoCreatedQueueTemplate {
* @param childQueuePath child queue path used for prefixing the properties
*/
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
- String childQueuePath) {
+ QueuePath childQueuePath) {
setTemplateEntriesForChild(conf, childQueuePath, false);
}
@@ -103,9 +100,9 @@ public class AutoCreatedQueueTemplate {
* @param childQueuePath child queue path used for prefixing the properties
*/
public void setTemplateEntriesForChild(CapacitySchedulerConfiguration conf,
- String childQueuePath,
+ QueuePath childQueuePath,
boolean isLeaf) {
- if (childQueuePath.equals(ROOT)) {
+ if (childQueuePath.isRoot()) {
return;
}
@@ -114,8 +111,7 @@ public class AutoCreatedQueueTemplate {
// Get all properties that are explicitly set
Set<String> alreadySetProps = configurationProperties
- .getPropertiesWithPrefix(CapacitySchedulerConfiguration
- .getQueuePrefix(childQueuePath)).keySet();
+ .getPropertiesWithPrefix(getQueuePrefix(childQueuePath)).keySet();
// Check template properties only set for leaf or parent queues
Map<String, String> queueTypeSpecificTemplates = parentOnlyProperties;
@@ -129,8 +125,7 @@ public class AutoCreatedQueueTemplate {
if (alreadySetProps.contains(entry.getKey())) {
continue;
}
- conf.set(CapacitySchedulerConfiguration.getQueuePrefix(
- childQueuePath) + entry.getKey(), entry.getValue());
+ conf.set(getQueuePrefix(childQueuePath) + entry.getKey(), entry.getValue());
}
for (Map.Entry<String, String> entry : templateProperties.entrySet()) {
@@ -140,8 +135,7 @@ public class AutoCreatedQueueTemplate {
|| queueTypeSpecificTemplates.containsKey(entry.getKey())) {
continue;
}
- conf.set(CapacitySchedulerConfiguration.getQueuePrefix(
- childQueuePath) + entry.getKey(), entry.getValue());
+ conf.set(getQueuePrefix(childQueuePath) + entry.getKey(), entry.getValue());
}
}
@@ -154,46 +148,26 @@ public class AutoCreatedQueueTemplate {
*/
private void setTemplateConfigEntries(CapacitySchedulerConfiguration configuration,
QueuePath queuePath) {
- ConfigurationProperties configurationProperties =
- configuration.getConfigurationProperties();
-
- List<String> queuePathParts = Lists.newArrayList(queuePath.iterator());
-
- if (queuePathParts.size() <= 1 && !queuePath.isRoot()) {
- // This is an invalid queue path
- return;
- }
- int queuePathMaxIndex = queuePathParts.size() - 1;
-
- // start with the most explicit format (without wildcard)
- int wildcardLevel = 0;
- // root can not be wildcarded
- int supportedWildcardLevel = Math.min(queuePathMaxIndex,
- configuration.getMaximumAutoCreatedQueueDepth(queuePath.getFullPath()));
- // Allow root to have template properties
- if (queuePath.isRoot()) {
- supportedWildcardLevel = 0;
- }
-
- // Collect all template entries
- while (wildcardLevel <= supportedWildcardLevel) {
- String templateQueuePath = String.join(".", queuePathParts);
- // Get all configuration entries with
- // yarn.scheduler.capacity.<queuePath> prefix
- Map<String, String> queueProps = configurationProperties
- .getPropertiesWithPrefix(getQueuePrefix(templateQueuePath));
-
- // Store template, parent-template and leaf-template properties
- for (Map.Entry<String, String> entry : queueProps.entrySet()) {
- storeConfiguredTemplates(entry.getKey(), entry.getValue());
+ if (!queuePath.isInvalid()) {
+ ConfigurationProperties configurationProperties =
+ configuration.getConfigurationProperties();
+
+ int maxAutoCreatedQueueDepth = configuration
+ .getMaximumAutoCreatedQueueDepth(queuePath);
+ List<QueuePath> wildcardedQueuePaths =
+ queuePath.getWildcardedQueuePaths(maxAutoCreatedQueueDepth);
+
+ for (QueuePath templateQueuePath: wildcardedQueuePaths) {
+ // Get all configuration entries with
+ // yarn.scheduler.capacity.<queuePath> prefix
+ Map<String, String> queueProps = configurationProperties
+ .getPropertiesWithPrefix(getQueuePrefix(templateQueuePath));
+
+ // Store template, parent-template and leaf-template properties
+ for (Map.Entry<String, String> entry : queueProps.entrySet()) {
+ storeConfiguredTemplates(entry.getKey(), entry.getValue());
+ }
}
-
- // Replace a queue part with a wildcard based on the wildcard level
- // eg. root.a -> root.*
- int queuePartToWildcard = queuePathMaxIndex - wildcardLevel;
- queuePathParts.set(queuePartToWildcard, WILDCARD_QUEUE);
-
- ++wildcardLevel;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java
index dc254747dfec..d429b208233b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueuePreemptionSettings.java
@@ -58,14 +58,14 @@ public class CSQueuePreemptionSettings {
// on, then q does not have preemption disabled (default=false, below)
// unless the preemption_disabled property is explicitly set.
if (parentQ == null) {
- return configuration.getPreemptionDisabled(q.getQueuePath(), false);
+ return configuration.getPreemptionDisabled(q.getQueuePathObject(), false);
}
// If this is not the root queue, inherit the default value for the
// preemption_disabled property from the parent. Preemptability will be
// inherited from the parent's hierarchy unless explicitly overridden at
// this level.
- return configuration.getPreemptionDisabled(q.getQueuePath(),
+ return configuration.getPreemptionDisabled(q.getQueuePathObject(),
parentQ.getPreemptionDisabled());
}
@@ -96,13 +96,13 @@ public class CSQueuePreemptionSettings {
CSQueue parentQ = q.getParent();
if (parentQ == null) {
return configuration
- .getIntraQueuePreemptionDisabled(q.getQueuePath(), false);
+ .getIntraQueuePreemptionDisabled(q.getQueuePathObject(), false);
}
// At this point, the master preemption switch is enabled down to this
// queue's level. Determine whether intra-queue preemption is enabled
// down to this queue's level and return that value.
- return configuration.getIntraQueuePreemptionDisabled(q.getQueuePath(),
+ return configuration.getIntraQueuePreemptionDisabled(q.getQueuePathObject(),
parentQ.getIntraQueuePreemptionDisabledInHierarchy());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 9e912d3dc4c5..76aab3179d75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -80,7 +80,7 @@ public class CSQueueUtils {
label,
csConf.getMaximumAMResourcePercentPerPartition(queuePath, label));
queueCapacities.setWeight(label,
- csConf.getNonLabeledQueueWeight(queuePath.getFullPath()));
+ csConf.getNonLabeledQueueWeight(queuePath));
} else{
queueCapacities.setCapacity(label,
csConf.getLabeledQueueCapacity(queuePath, label) / 100);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 6ebadcba1c91..ec5b074915ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2537,7 +2537,7 @@ public class CapacityScheduler extends
if (queue == null) {
// reservation has terminated during failover
if (isRecovering && conf.getMoveOnExpiry(
- getQueue(queueName).getQueuePath())) {
+ getQueue(queueName).getQueuePathObject())) {
// move to the default child queue of the plan
return getDefaultReservationQueueName(queueName);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index e40e978a8002..e2b61df9eb3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -136,8 +136,8 @@ public final class CapacitySchedulerConfigValidator {
}
final String queuePath = oldQueue.getQueuePath();
- final String configPrefix = CapacitySchedulerConfiguration.getQueuePrefix(
- oldQueue.getQueuePath());
+ final String configPrefix = QueuePrefixes.getQueuePrefix(
+ oldQueue.getQueuePathObject());
final QueueState newQueueState = createQueueState(newConf.get(configPrefix + "state"),
queuePath);
final CSQueue newQueue = newQueues.get(queuePath);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 5ab237d282a6..8d9cf2079301 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
@@ -78,6 +77,10 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getNodeLabelPrefix;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix;
+
public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration {
private static final Logger LOG =
@@ -438,13 +441,13 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return queueCapacityConfigParser;
}
- public int getMaximumAutoCreatedQueueDepth(String queuePath) {
+ public int getMaximumAutoCreatedQueueDepth(QueuePath queuePath) {
return getInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH,
getInt(PREFIX + MAXIMUM_QUEUE_DEPTH, DEFAULT_MAXIMUM_QUEUE_DEPTH));
}
- public void setMaximumAutoCreatedQueueDepth(String queuePath, int value) {
- setInt(getQueuePrefix(queuePath) + MAXIMUM_QUEUE_DEPTH, value);
+ public void setMaximumAutoCreatedQueueDepth(QueuePath queue, int value) {
+ setInt(getQueuePrefix(queue) + MAXIMUM_QUEUE_DEPTH, value);
}
public void setMaximumAutoCreatedQueueDepth(int value) {
@@ -476,27 +479,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
- public static String getQueuePrefix(String queue) {
- String queueName = PREFIX + queue + DOT;
- return queueName;
- }
-
- static String getQueueOrderingPolicyPrefix(String queue) {
- String queueName = PREFIX + queue + DOT + ORDERING_POLICY + DOT;
- return queueName;
- }
-
static String getUserPrefix(String user) {
return PREFIX + "user." + user + DOT;
}
- public static String getNodeLabelPrefix(String queue, String label) {
- if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
- return getQueuePrefix(queue);
- }
- return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT;
- }
-
public void setMaximumSystemApplications(int numMaxApps) {
setInt(MAXIMUM_SYSTEM_APPLICATIONS, numMaxApps);
}
@@ -519,10 +505,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
/**
* Get the maximum applications per queue setting.
- * @param queue name of the queue
+ * @param queue path of the queue
* @return setting specified or -1 if not set
*/
- public int getMaximumApplicationsPerQueue(String queue) {
+ public int getMaximumApplicationsPerQueue(QueuePath queue) {
int maxApplicationsPerQueue =
getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
(int)UNDEFINED);
@@ -530,7 +516,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@VisibleForTesting
- public void setMaximumApplicationsPerQueue(String queue,
+ public void setMaximumApplicationsPerQueue(QueuePath queue,
int numMaxApps) {
setInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
numMaxApps);
@@ -538,54 +524,54 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
/**
* Get the maximum am resource percent per queue setting.
- * @param queue name of the queue
+ * @param queue path of the queue
* @return per queue setting or defaults to the global am-resource-percent
* setting if per queue setting not present
*/
- public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) {
+ public float getMaximumApplicationMasterResourcePerQueuePercent(QueuePath queue) {
return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX,
getMaximumApplicationMasterResourcePercent());
}
- public void setMaximumApplicationMasterResourcePerQueuePercent(String queue,
+ public void setMaximumApplicationMasterResourcePerQueuePercent(QueuePath queue,
float percent) {
setFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, percent);
}
- private void throwExceptionForUnexpectedWeight(float weight, String queue,
+ private void throwExceptionForUnexpectedWeight(float weight, QueuePath queue,
String label) {
if ((weight < -1e-6 && Math.abs(weight + 1) > 1e-6) || weight > 10000) {
throw new IllegalArgumentException(
- "Illegal " + "weight=" + weight + " for queue=" + queue + "label="
+ "Illegal " + "weight=" + weight + " for queue=" + queue.getFullPath() + "label="
+ label
+ ". Acceptable values: [0, 10000], -1 is same as not set");
}
}
- public float getNonLabeledQueueWeight(String queue) {
+ public float getNonLabeledQueueWeight(QueuePath queue) {
String configuredValue = get(getQueuePrefix(queue) + CAPACITY);
float weight = extractFloatValueFromWeightConfig(configuredValue);
throwExceptionForUnexpectedWeight(weight, queue, "");
return weight;
}
- public void setNonLabeledQueueWeight(String queue, float weight) {
+ public void setNonLabeledQueueWeight(QueuePath queue, float weight) {
set(getQueuePrefix(queue) + CAPACITY, weight + WEIGHT_SUFFIX);
}
- public void setLabeledQueueWeight(String queue, String label, float weight) {
+ public void setLabeledQueueWeight(QueuePath queue, String label, float weight) {
set(getNodeLabelPrefix(queue, label) + CAPACITY, weight + WEIGHT_SUFFIX);
}
public float getLabeledQueueWeight(QueuePath queue, String label) {
- String configuredValue = get(getNodeLabelPrefix(queue.getFullPath(), label) + CAPACITY);
+ String configuredValue = get(getNodeLabelPrefix(queue, label) + CAPACITY);
float weight = extractFloatValueFromWeightConfig(configuredValue);
- throwExceptionForUnexpectedWeight(weight, queue.getFullPath(), label);
+ throwExceptionForUnexpectedWeight(weight, queue, label);
return weight;
}
public float getNonLabeledQueueCapacity(QueuePath queue) {
- String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + CAPACITY);
+ String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY);
boolean absoluteResourceConfigured = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find();
boolean isCapacityVectorFormat = queueCapacityConfigParser
@@ -607,16 +593,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
if (capacity < MINIMUM_CAPACITY_VALUE
|| capacity > MAXIMUM_CAPACITY_VALUE) {
throw new IllegalArgumentException(
- "Illegal " + "capacity of " + capacity + " for queue " + queue);
+ "Illegal " + "capacity of " + capacity + " for queue " + queue.getFullPath());
}
LOG.debug("CSConf - getCapacity: queuePrefix={}, capacity={}",
- getQueuePrefix(queue.getFullPath()), capacity);
+ getQueuePrefix(queue), capacity);
return capacity;
}
- public void setCapacity(String queue, float capacity) {
- if (queue.equals("root")) {
+ public void setCapacity(QueuePath queue, float capacity) {
+ if (queue.isRoot()) {
throw new IllegalArgumentException(
"Cannot set capacity, root queue has a fixed capacity of 100.0f");
}
@@ -627,8 +613,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@VisibleForTesting
- public void setCapacity(String queue, String absoluteResourceCapacity) {
- if (queue.equals("root")) {
+ public void setCapacity(QueuePath queue, String absoluteResourceCapacity) {
+ if (queue.isRoot()) {
throw new IllegalArgumentException(
"Cannot set capacity, root queue has a fixed capacity");
}
@@ -639,7 +625,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
public float getNonLabeledQueueMaximumCapacity(QueuePath queue) {
- String configuredCapacity = get(getQueuePrefix(queue.getFullPath()) + MAXIMUM_CAPACITY);
+ String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY);
boolean matcher = (configuredCapacity != null)
&& RESOURCE_PATTERN.matcher(configuredCapacity).find()
|| queueCapacityConfigParser.isCapacityVectorFormat(configuredCapacity);
@@ -660,38 +646,38 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return maxCapacity;
}
- public void setMaximumCapacity(String queue, float maxCapacity) {
+ public void setMaximumCapacity(QueuePath queue, float maxCapacity) {
if (maxCapacity > MAXIMUM_CAPACITY_VALUE) {
throw new IllegalArgumentException("Illegal " +
- "maximum-capacity of " + maxCapacity + " for queue " + queue);
+ "maximum-capacity of " + maxCapacity + " for queue " + queue.getFullPath());
}
setFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY, maxCapacity);
LOG.debug("CSConf - setMaxCapacity: queuePrefix={}, maxCapacity={}",
getQueuePrefix(queue), maxCapacity);
}
- public void setCapacityByLabel(String queue, String label, float capacity) {
+ public void setCapacityByLabel(QueuePath queue, String label, float capacity) {
setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity);
}
@VisibleForTesting
- public void setCapacityByLabel(String queue, String label,
+ public void setCapacityByLabel(QueuePath queue, String label,
String absoluteResourceCapacity) {
set(getNodeLabelPrefix(queue, label) + CAPACITY, absoluteResourceCapacity);
}
- public void setMaximumCapacityByLabel(String queue, String label,
+ public void setMaximumCapacityByLabel(QueuePath queue, String label,
float capacity) {
setFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, capacity);
}
- public void setMaximumCapacityByLabel(String queue, String label,
+ public void setMaximumCapacityByLabel(QueuePath queue, String label,
String absoluteResourceCapacity) {
set(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY,
absoluteResourceCapacity);
}
- public float getUserLimit(String queue) {
+ public float getUserLimit(QueuePath queue) {
float defaultUserLimit = getFloat(PREFIX + USER_LIMIT, DEFAULT_USER_LIMIT);
float userLimit = getFloat(getQueuePrefix(queue) + USER_LIMIT,
defaultUserLimit);
@@ -703,7 +689,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
// if possible.
@SuppressWarnings("unchecked")
public <S extends SchedulableEntity> OrderingPolicy<S> getAppOrderingPolicy(
- String queue) {
+ QueuePath queue) {
String policyType = get(getQueuePrefix(queue) + ORDERING_POLICY,
DEFAULT_APP_ORDERING_POLICY);
@@ -742,7 +728,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return orderingPolicy;
}
- public void setUserLimit(String queue, float userLimit) {
+ public void setUserLimit(QueuePath queue, float userLimit) {
setFloat(getQueuePrefix(queue) + USER_LIMIT, userLimit);
LOG.debug("here setUserLimit: queuePrefix={}, userLimit={}",
getQueuePrefix(queue), getUserLimit(queue));
@@ -753,7 +739,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
setFloat(PREFIX + USER_LIMIT, defaultUserLimit);
}
- public float getUserLimitFactor(String queue) {
+ public float getUserLimitFactor(QueuePath queue) {
float defaultUserLimitFactor = getFloat(PREFIX + USER_LIMIT_FACTOR, DEFAULT_USER_LIMIT_FACTOR);
float userLimitFactor =
getFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR,
@@ -761,8 +747,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return userLimitFactor;
}
- public void setUserLimitFactor(String queue, float userLimitFactor) {
- setFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, userLimitFactor);
+ public void setUserLimitFactor(QueuePath queuePath, float userLimitFactor) {
+ setFloat(getQueuePrefix(queuePath) + USER_LIMIT_FACTOR, userLimitFactor);
}
@VisibleForTesting
@@ -770,7 +756,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
setFloat(PREFIX + USER_LIMIT_FACTOR, defaultUserLimitFactor);
}
- public QueueState getConfiguredState(String queue) {
+ public QueueState getConfiguredState(QueuePath queue) {
String state = get(getQueuePrefix(queue) + STATE);
if (state == null) {
return null;
@@ -779,18 +765,18 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
- public QueueState getState(String queue) {
+ public QueueState getState(QueuePath queue) {
QueueState state = getConfiguredState(queue);
return (state == null) ? QueueState.RUNNING : state;
}
@Private
@VisibleForTesting
- public void setState(String queue, QueueState state) {
+ public void setState(QueuePath queue, QueueState state) {
set(getQueuePrefix(queue) + STATE, state.name());
}
- public void setAccessibleNodeLabels(String queue, Set<String> labels) {
+ public void setAccessibleNodeLabels(QueuePath queue, Set<String> labels) {
if (labels == null) {
return;
}
@@ -798,27 +784,27 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str);
}
- public Set<String> getAccessibleNodeLabels(String queue) {
+ public Set<String> getAccessibleNodeLabels(QueuePath queue) {
String accessibleLabelStr =
get(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS);
// When accessible-label is null,
if (accessibleLabelStr == null) {
// Only return null when queue is not ROOT
- if (!queue.equals(ROOT)) {
+ if (!queue.isRoot()) {
return null;
}
} else {
// print a warning when accessibleNodeLabel specified in config and queue
// is ROOT
- if (queue.equals(ROOT)) {
+ if (queue.isRoot()) {
LOG.warn("Accessible node labels for root queue will be ignored,"
+ " it will be automatically set to \"*\".");
}
}
// always return ANY for queue root
- if (queue.equals(ROOT)) {
+ if (queue.isRoot()) {
return ImmutableSet.of(RMNodeLabelsManager.ANY);
}
@@ -838,12 +824,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return Collections.unmodifiableSet(set);
}
- public void setCapacityVector(String queuePath, String label, String capacityVector) {
+ public void setCapacityVector(QueuePath queuePath, String label, String capacityVector) {
String capacityPropertyName = getNodeLabelPrefix(queuePath, label) + CAPACITY;
set(capacityPropertyName, capacityVector);
}
- public void setMaximumCapacityVector(String queuePath, String label, String capacityVector) {
+ public void setMaximumCapacityVector(QueuePath queuePath, String label, String capacityVector) {
String capacityPropertyName = getNodeLabelPrefix(queuePath, label) + MAXIMUM_CAPACITY;
set(capacityPropertyName, capacityVector);
}
@@ -866,7 +852,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
private float internalGetLabeledQueueCapacity(QueuePath queue, String label,
String suffix, float defaultValue) {
- String capacityPropertyName = getNodeLabelPrefix(queue.getFullPath(), label) + suffix;
+ String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
String configuredCapacity = get(capacityPropertyName);
boolean absoluteResourceConfigured =
(configuredCapacity != null) && RESOURCE_PATTERN.matcher(
@@ -891,7 +877,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
if (LOG.isDebugEnabled()) {
LOG.debug(
- "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue.getFullPath(),
+ "CSConf - getCapacityOfLabel: prefix=" + getNodeLabelPrefix(queue,
label) + ", capacity=" + capacity);
}
return capacity;
@@ -905,7 +891,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f);
}
- public String getDefaultNodeLabelExpression(String queue) {
+ public String getDefaultNodeLabelExpression(QueuePath queue) {
String defaultLabelExpression = get(getQueuePrefix(queue)
+ DEFAULT_NODE_LABEL_EXPRESSION);
if (defaultLabelExpression == null) {
@@ -914,7 +900,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return defaultLabelExpression.trim();
}
- public void setDefaultNodeLabelExpression(String queue, String exp) {
+ public void setDefaultNodeLabelExpression(QueuePath queue, String exp) {
set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp);
}
@@ -922,12 +908,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
String label) {
// If per-partition max-am-resource-percent is not configured,
// use default value as max-am-resource-percent for this queue.
- return getFloat(getNodeLabelPrefix(queue.getFullPath(), label)
+ return getFloat(getNodeLabelPrefix(queue, label)
+ MAXIMUM_AM_RESOURCE_SUFFIX,
- getMaximumApplicationMasterResourcePerQueuePercent(queue.getFullPath()));
+ getMaximumApplicationMasterResourcePerQueuePercent(queue));
}
- public void setMaximumAMResourcePercentPerPartition(String queue,
+ public void setMaximumAMResourcePercentPerPartition(QueuePath queue,
String label, float percent) {
setFloat(getNodeLabelPrefix(queue, label)
+ MAXIMUM_AM_RESOURCE_SUFFIX, percent);
@@ -953,16 +939,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return "acl_" + StringUtils.toLowerCase(acl.toString());
}
- public AccessControlList getAcl(String queue, QueueACL acl) {
+ public AccessControlList getAcl(QueuePath queue, QueueACL acl) {
String queuePrefix = getQueuePrefix(queue);
// The root queue defaults to all access if not defined
// Sub queues inherit access if not defined
- String defaultAcl = queue.equals(ROOT) ? ALL_ACL : NONE_ACL;
+ String defaultAcl = queue.isRoot() ? ALL_ACL : NONE_ACL;
String aclString = get(queuePrefix + getAclKey(acl), defaultAcl);
return new AccessControlList(aclString);
}
- public void setAcl(String queue, QueueACL acl, String aclString) {
+ public void setAcl(QueuePath queue, QueueACL acl, String aclString) {
String queuePrefix = getQueuePrefix(queue);
set(queuePrefix + getAclKey(acl), aclString);
}
@@ -982,9 +968,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* @return A mapping of the queue ACLs.
*/
public Map<AccessType, AccessControlList> getACLsForLegacyAutoCreatedLeafQueue(
- String parentQueuePath) {
+ QueuePath parentQueuePath) {
final String prefix =
- getQueuePrefix(getAutoCreatedQueueTemplateConfPrefix(
+ getQueuePrefix(getAutoCreatedQueueObjectTemplateConfPrefix(
parentQueuePath));
Map<String, String> properties = new HashMap<>();
@@ -1060,7 +1046,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Override
- public Map<ReservationACL, AccessControlList> getReservationAcls(String
+ public Map<ReservationACL, AccessControlList> getReservationAcls(QueuePath
queue) {
Map<ReservationACL, AccessControlList> resAcls = new HashMap<>();
for (ReservationACL acl : ReservationACL.values()) {
@@ -1069,7 +1055,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return resAcls;
}
- private AccessControlList getReservationAcl(String queue, ReservationACL
+ private AccessControlList getReservationAcl(QueuePath queue, ReservationACL
acl) {
String queuePrefix = getQueuePrefix(queue);
// The root queue defaults to all access if not defined
@@ -1079,17 +1065,17 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return new AccessControlList(aclString);
}
- private void setAcl(String queue, ReservationACL acl, String aclString) {
+ private void setAcl(QueuePath queue, ReservationACL acl, String aclString) {
String queuePrefix = getQueuePrefix(queue);
set(queuePrefix + getAclKey(acl), aclString);
}
- private void setAcl(String queue, AccessType acl, String aclString) {
+ private void setAcl(QueuePath queue, AccessType acl, String aclString) {
String queuePrefix = getQueuePrefix(queue);
set(queuePrefix + getAclKey(acl), aclString);
}
- public Map<AccessType, AccessControlList> getAcls(String queue) {
+ public Map<AccessType, AccessControlList> getAcls(QueuePath queue) {
Map<AccessType, AccessControlList> acls =
new HashMap<AccessType, AccessControlList>();
for (QueueACL acl : QueueACL.values()) {
@@ -1098,14 +1084,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return acls;
}
- public void setAcls(String queue, Map<QueueACL, AccessControlList> acls) {
+ public void setAcls(QueuePath queue, Map<QueueACL, AccessControlList> acls) {
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
setAcl(queue, e.getKey(), e.getValue().getAclString());
}
}
@VisibleForTesting
- public void setReservationAcls(String queue,
+ public void setReservationAcls(QueuePath queue,
Map<ReservationACL, AccessControlList> acls) {
for (Map.Entry<ReservationACL, AccessControlList> e : acls.entrySet()) {
setAcl(queue, e.getKey(), e.getValue().getAclString());
@@ -1113,7 +1099,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@VisibleForTesting
- public void setPriorityAcls(String queue, Priority priority,
+ public void setPriorityAcls(QueuePath queue, Priority priority,
Priority defaultPriority, String[] acls) {
StringBuilder aclString = new StringBuilder();
@@ -1130,7 +1116,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
setAcl(queue, AccessType.APPLICATION_MAX_PRIORITY, aclString.toString());
}
- public List<AppPriorityACLGroup> getPriorityAcls(String queue,
+ public List<AppPriorityACLGroup> getPriorityAcls(QueuePath queue,
Priority clusterMaxPriority) {
String queuePrefix = getQueuePrefix(queue);
String defaultAcl = ALL_ACL;
@@ -1141,7 +1127,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return priorityACLConfig.getPriorityAcl(clusterMaxPriority, aclString);
}
- public String[] getQueues(String queue) {
+ public List<String> getQueues(QueuePath queue) {
LOG.debug("CSConf - getQueues called for: queuePrefix={}",
getQueuePrefix(queue));
String[] queues = getStrings(getQueuePrefix(queue) + QUEUES);
@@ -1150,17 +1136,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
for (String s : queues) {
trimmedQueueNames.add(s.trim());
}
- queues = trimmedQueueNames.toArray(new String[0]);
}
LOG.debug("CSConf - getQueues: queuePrefix={}, queues={}",
getQueuePrefix(queue),
((queues == null) ? "" : StringUtils.arrayToString(queues)));
- return queues;
+ return trimmedQueueNames;
}
- public void setQueues(String queue, String[] subQueues) {
+ public void setQueues(QueuePath queue, String[] subQueues) {
set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues));
LOG.debug("CSConf - setQueues: qPrefix={}, queues={}",
getQueuePrefix(queue), StringUtils.arrayToString(subQueues));
@@ -1177,7 +1162,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Private
- public Priority getQueuePriority(String queue) {
+ public Priority getQueuePriority(QueuePath queue) {
String queuePolicyPrefix = getQueuePrefix(queue);
Priority pri = Priority.newInstance(
getInt(queuePolicyPrefix + "priority", 0));
@@ -1185,7 +1170,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Private
- public void setQueuePriority(String queue, int priority) {
+ public void setQueuePriority(QueuePath queue, int priority) {
String queuePolicyPrefix = getQueuePrefix(queue);
setInt(queuePolicyPrefix + "priority", priority);
}
@@ -1198,7 +1183,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* name of the queue
* @return Resource object or Resource.none if not set
*/
- public Resource getQueueMaximumAllocation(String queue) {
+ public Resource getQueueMaximumAllocation(QueuePath queue) {
String queuePrefix = getQueuePrefix(queue);
String rawQueueMaxAllocation = get(queuePrefix + MAXIMUM_ALLOCATION, null);
if (Strings.isNullOrEmpty(rawQueueMaxAllocation)) {
@@ -1209,7 +1194,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
}
- public void setQueueMaximumAllocation(String queue, String maximumAllocation) {
+ public void setQueueMaximumAllocation(QueuePath queue, String maximumAllocation) {
String queuePrefix = getQueuePrefix(queue);
set(queuePrefix + MAXIMUM_ALLOCATION, maximumAllocation);
}
@@ -1237,22 +1222,22 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
configurationProperties = new ConfigurationProperties(props);
}
- public void setQueueMaximumAllocationMb(String queue, int value) {
+ public void setQueueMaximumAllocationMb(QueuePath queue, int value) {
String queuePrefix = getQueuePrefix(queue);
setInt(queuePrefix + MAXIMUM_ALLOCATION_MB, value);
}
- public void setQueueMaximumAllocationVcores(String queue, int value) {
+ public void setQueueMaximumAllocationVcores(QueuePath queue, int value) {
String queuePrefix = getQueuePrefix(queue);
setInt(queuePrefix + MAXIMUM_ALLOCATION_VCORES, value);
}
- public long getQueueMaximumAllocationMb(String queue) {
+ public long getQueueMaximumAllocationMb(QueuePath queue) {
String queuePrefix = getQueuePrefix(queue);
return getInt(queuePrefix + MAXIMUM_ALLOCATION_MB, (int)UNDEFINED);
}
- public int getQueueMaximumAllocationVcores(String queue) {
+ public int getQueueMaximumAllocationVcores(QueuePath queue) {
String queuePrefix = getQueuePrefix(queue);
return getInt(queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
}
@@ -1595,20 +1580,20 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
.getWorkflowPriorityMappingStr(workflowPriorityMappings));
}
- public boolean isReservable(String queue) {
+ public boolean isReservable(QueuePath queue) {
boolean isReservable =
getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false);
return isReservable;
}
- public void setReservable(String queue, boolean isReservable) {
+ public void setReservable(QueuePath queue, boolean isReservable) {
setBoolean(getQueuePrefix(queue) + IS_RESERVABLE, isReservable);
LOG.debug("here setReservableQueue: queuePrefix={}, isReservableQueue={}",
getQueuePrefix(queue), isReservable(queue));
}
@Override
- public long getReservationWindow(String queue) {
+ public long getReservationWindow(QueuePath queue) {
long reservationWindow =
getLong(getQueuePrefix(queue) + RESERVATION_WINDOW,
DEFAULT_RESERVATION_WINDOW);
@@ -1616,7 +1601,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Override
- public float getAverageCapacity(String queue) {
+ public float getAverageCapacity(QueuePath queue) {
float avgCapacity =
getFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY,
MAXIMUM_CAPACITY_VALUE);
@@ -1624,53 +1609,53 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Override
- public float getInstantaneousMaxCapacity(String queue) {
+ public float getInstantaneousMaxCapacity(QueuePath queue) {
float instMaxCapacity =
getFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY,
MAXIMUM_CAPACITY_VALUE);
return instMaxCapacity;
}
- public void setInstantaneousMaxCapacity(String queue, float instMaxCapacity) {
+ public void setInstantaneousMaxCapacity(QueuePath queue, float instMaxCapacity) {
setFloat(getQueuePrefix(queue) + INSTANTANEOUS_MAX_CAPACITY,
instMaxCapacity);
}
- public void setReservationWindow(String queue, long reservationWindow) {
+ public void setReservationWindow(QueuePath queue, long reservationWindow) {
setLong(getQueuePrefix(queue) + RESERVATION_WINDOW, reservationWindow);
}
- public void setAverageCapacity(String queue, float avgCapacity) {
+ public void setAverageCapacity(QueuePath queue, float avgCapacity) {
setFloat(getQueuePrefix(queue) + AVERAGE_CAPACITY, avgCapacity);
}
@Override
- public String getReservationAdmissionPolicy(String queue) {
+ public String getReservationAdmissionPolicy(QueuePath queue) {
String reservationPolicy =
get(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY,
DEFAULT_RESERVATION_ADMISSION_POLICY);
return reservationPolicy;
}
- public void setReservationAdmissionPolicy(String queue,
+ public void setReservationAdmissionPolicy(QueuePath queue,
String reservationPolicy) {
set(getQueuePrefix(queue) + RESERVATION_ADMISSION_POLICY, reservationPolicy);
}
@Override
- public String getReservationAgent(String queue) {
+ public String getReservationAgent(QueuePath queue) {
String reservationAgent =
get(getQueuePrefix(queue) + RESERVATION_AGENT_NAME,
DEFAULT_RESERVATION_AGENT_NAME);
return reservationAgent;
}
- public void setReservationAgent(String queue, String reservationPolicy) {
+ public void setReservationAgent(QueuePath queue, String reservationPolicy) {
set(getQueuePrefix(queue) + RESERVATION_AGENT_NAME, reservationPolicy);
}
@Override
- public boolean getShowReservationAsQueues(String queuePath) {
+ public boolean getShowReservationAsQueues(QueuePath queuePath) {
boolean showReservationAsQueues =
getBoolean(getQueuePrefix(queuePath)
+ RESERVATION_SHOW_RESERVATION_AS_QUEUE,
@@ -1679,7 +1664,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Override
- public String getReplanner(String queue) {
+ public String getReplanner(QueuePath queue) {
String replanner =
get(getQueuePrefix(queue) + RESERVATION_PLANNER_NAME,
DEFAULT_RESERVATION_PLANNER_NAME);
@@ -1687,7 +1672,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Override
- public boolean getMoveOnExpiry(String queue) {
+ public boolean getMoveOnExpiry(QueuePath queue) {
boolean killOnExpiry =
getBoolean(getQueuePrefix(queue) + RESERVATION_MOVE_ON_EXPIRY,
DEFAULT_RESERVATION_MOVE_ON_EXPIRY);
@@ -1695,7 +1680,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@Override
- public long getEnforcementWindow(String queue) {
+ public long getEnforcementWindow(QueuePath queue) {
long enforcementWindow =
getLong(getQueuePrefix(queue) + RESERVATION_ENFORCEMENT_WINDOW,
DEFAULT_RESERVATION_ENFORCEMENT_WINDOW);
@@ -1710,7 +1695,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* @param queue queue path
* @param preemptionDisabled true if preemption is disabled on queue
*/
- public void setPreemptionDisabled(String queue, boolean preemptionDisabled) {
+ public void setPreemptionDisabled(QueuePath queue, boolean preemptionDisabled) {
setBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED,
preemptionDisabled);
}
@@ -1723,7 +1708,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* is not set in the configuration
* @return true if preemption is disabled on <em>queue</em>, false otherwise
*/
- public boolean getPreemptionDisabled(String queue, boolean defaultVal) {
+ public boolean getPreemptionDisabled(QueuePath queue, boolean defaultVal) {
boolean preemptionDisabled =
getBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED,
defaultVal);
@@ -1738,7 +1723,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* configuration
* @return true if preemption is disabled on queue, false otherwise
*/
- public boolean getIntraQueuePreemptionDisabled(String queue,
+ public boolean getIntraQueuePreemptionDisabled(QueuePath queue,
boolean defaultVal) {
return
getBoolean(getQueuePrefix(queue) + INTRA_QUEUE_PREEMPTION_CONFIG_PREFIX
@@ -1759,7 +1744,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* @param queuePath queue path.
* @return configured node labels.
*/
- public Set<String> getConfiguredNodeLabels(String queuePath) {
+ public Set<String> getConfiguredNodeLabels(QueuePath queuePath) {
Set<String> configuredNodeLabels = new HashSet<String>();
Entry<String, String> e = null;
@@ -1830,7 +1815,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
YarnConfiguration.DEFAULT_CLUSTER_LEVEL_APPLICATION_PRIORITY));
}
- public Integer getDefaultApplicationPriorityConfPerQueue(String queue) {
+ public Integer getDefaultApplicationPriorityConfPerQueue(QueuePath queue) {
Integer defaultPriority = getInt(getQueuePrefix(queue)
+ DEFAULT_APPLICATION_PRIORITY,
DEFAULT_CONFIGURATION_APPLICATION_PRIORITY);
@@ -1838,12 +1823,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
@VisibleForTesting
- public void setOrderingPolicy(String queue, String policy) {
+ public void setOrderingPolicy(QueuePath queue, String policy) {
set(getQueuePrefix(queue) + ORDERING_POLICY, policy);
}
@VisibleForTesting
- public void setOrderingPolicyParameter(String queue,
+ public void setOrderingPolicyParameter(QueuePath queue,
String parameterKey, String parameterValue) {
set(getQueuePrefix(queue) + ORDERING_POLICY + "." + parameterKey,
parameterValue);
@@ -1888,11 +1873,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
: getDefaultMaxParallelAppsPerUser();
}
- public void setMaxParallelAppsForQueue(String queue, String value) {
+ public void setMaxParallelAppsForQueue(QueuePath queue, String value) {
set(getQueuePrefix(queue) + MAX_PARALLEL_APPLICATIONS, value);
}
- public Integer getMaxParallelAppsForQueue(String queue) {
+ public Integer getMaxParallelAppsForQueue(QueuePath queue) {
String maxParallelAppsForQueue = get(getQueuePrefix(queue)
+ MAX_PARALLEL_APPLICATIONS);
@@ -1901,12 +1886,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
: getDefaultMaxParallelApps();
}
- public boolean getAllowZeroCapacitySum(String queue) {
+ public boolean getAllowZeroCapacitySum(QueuePath queue) {
return getBoolean(getQueuePrefix(queue)
+ ALLOW_ZERO_CAPACITY_SUM, DEFAULT_ALLOW_ZERO_CAPACITY_SUM);
}
- public void setAllowZeroCapacitySum(String queue, boolean value) {
+ public void setAllowZeroCapacitySum(QueuePath queue, boolean value) {
setBoolean(getQueuePrefix(queue)
+ ALLOW_ZERO_CAPACITY_SUM, value);
}
@@ -2104,12 +2089,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
- public void setQueueOrderingPolicy(String queue, String policy) {
+ public void setQueueOrderingPolicy(QueuePath queue, String policy) {
set(getQueuePrefix(queue) + ORDERING_POLICY, policy);
}
@Private
- public QueueOrderingPolicy getQueueOrderingPolicy(String queue,
+ public QueueOrderingPolicy getQueueOrderingPolicy(QueuePath queue,
String parentPolicy) {
String defaultPolicy = parentPolicy;
if (null == defaultPolicy) {
@@ -2131,7 +2116,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
qop = (QueueOrderingPolicy) Class.forName(policyType).newInstance();
} catch (Exception e) {
String message = "Unable to construct queue ordering policy="
- + policyType + " queue=" + queue;
+ + policyType + " queue=" + queue.getFullPath();
throw new YarnRuntimeException(message, e);
}
}
@@ -2250,7 +2235,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* @param queuePath full queue path
* @return map of user weights, if they exist. Otherwise, return empty map.
*/
- public UserWeights getAllUserWeightsForQueue(String queuePath) {
+ public UserWeights getAllUserWeightsForQueue(QueuePath queuePath) {
return UserWeights.createByConfig(this, getConfigurationProperties(), queuePath);
}
@@ -2268,23 +2253,23 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String DEFAULT_LIFETIME_SUFFIX =
"default-application-lifetime";
- public long getMaximumLifetimePerQueue(String queue) {
+ public long getMaximumLifetimePerQueue(QueuePath queue) {
long maximumLifetimePerQueue = getLong(
getQueuePrefix(queue) + MAXIMUM_LIFETIME_SUFFIX, (long) UNDEFINED);
return maximumLifetimePerQueue;
}
- public void setMaximumLifetimePerQueue(String queue, long maximumLifetime) {
+ public void setMaximumLifetimePerQueue(QueuePath queue, long maximumLifetime) {
setLong(getQueuePrefix(queue) + MAXIMUM_LIFETIME_SUFFIX, maximumLifetime);
}
- public long getDefaultLifetimePerQueue(String queue) {
+ public long getDefaultLifetimePerQueue(QueuePath queue) {
long maximumLifetimePerQueue = getLong(
getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, (long) UNDEFINED);
return maximumLifetimePerQueue;
}
- public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
+ public void setDefaultLifetimePerQueue(QueuePath queue, long defaultLifetime) {
setLong(getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, defaultLifetime);
}
@@ -2345,7 +2330,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* is false
*/
@Private
- public boolean isAutoCreateChildQueueEnabled(String queuePath) {
+ public boolean isAutoCreateChildQueueEnabled(QueuePath queuePath) {
boolean isAutoCreateEnabled = getBoolean(
getQueuePrefix(queuePath) + AUTO_CREATE_CHILD_QUEUE_ENABLED,
DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED);
@@ -2354,44 +2339,27 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
@VisibleForTesting
- public void setAutoCreateChildQueueEnabled(String queuePath,
+ public void setAutoCreateChildQueueEnabled(QueuePath queuePath,
boolean autoCreationEnabled) {
setBoolean(getQueuePrefix(queuePath) +
AUTO_CREATE_CHILD_QUEUE_ENABLED,
autoCreationEnabled);
}
- public void setAutoQueueCreationV2Enabled(String queuePath,
+ public void setAutoQueueCreationV2Enabled(QueuePath queuePath,
boolean autoQueueCreation) {
setBoolean(
getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_V2_ENABLED,
autoQueueCreation);
}
- public boolean isAutoQueueCreationV2Enabled(String queuePath) {
+ public boolean isAutoQueueCreationV2Enabled(QueuePath queuePath) {
boolean isAutoQueueCreation = getBoolean(
getQueuePrefix(queuePath) + AUTO_QUEUE_CREATION_V2_ENABLED,
DEFAULT_AUTO_QUEUE_CREATION_ENABLED);
return isAutoQueueCreation;
}
- /**
- * Get the auto created leaf queue's template configuration prefix
- * Leaf queue's template capacities are configured at the parent queue
- *
- * @param queuePath parent queue's path
- * @return Config prefix for leaf queue template configurations
- */
- @Private
- public String getAutoCreatedQueueTemplateConfPrefix(String queuePath) {
- return queuePath + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
- }
-
- @Private
- public QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(String queuePath) {
- return new QueuePath(queuePath, AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX);
- }
-
@Private
public static final String FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY =
"auto-create-child-queue.fail-on-exceeding-parent-capacity";
@@ -2409,7 +2377,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
*/
@Private
public boolean getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
- String queuePath) {
+ QueuePath queuePath) {
boolean shouldFailAutoQueueCreationOnExceedingGuaranteedCapacity =
getBoolean(getQueuePrefix(queuePath)
+ FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY,
@@ -2420,7 +2388,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@VisibleForTesting
@Private
public void setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
- String queuePath, boolean autoCreationEnabled) {
+ QueuePath queuePath, boolean autoCreationEnabled) {
setBoolean(
getQueuePrefix(queuePath) +
FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY,
@@ -2435,7 +2403,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* @return the max number of leaf queues allowed to be auto created
*/
@Private
- public int getAutoCreatedQueuesMaxChildQueuesLimit(String queuePath) {
+ public int getAutoCreatedQueuesMaxChildQueuesLimit(QueuePath queuePath) {
return getInt(getQueuePrefix(queuePath) +
AUTO_CREATE_QUEUE_MAX_QUEUES,
DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES);
@@ -2450,14 +2418,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* in new auto created.
*/
@Private
- public int getAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath) {
+ public int getAutoCreatedQueuesV2MaxChildQueuesLimit(QueuePath queuePath) {
return getInt(getQueuePrefix(queuePath) +
AUTO_QUEUE_CREATION_V2_MAX_QUEUES,
DEFAULT_AUTO_QUEUE_CREATION_V2_MAX_QUEUES);
}
@VisibleForTesting
- public void setAutoCreatedQueuesV2MaxChildQueuesLimit(String queuePath,
+ public void setAutoCreatedQueuesV2MaxChildQueuesLimit(QueuePath queuePath,
int maxQueues) {
setInt(getQueuePrefix(queuePath) +
AUTO_QUEUE_CREATION_V2_MAX_QUEUES, maxQueues);
@@ -2514,7 +2482,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* is true.
*/
@Private
- public boolean isAutoExpiredDeletionEnabled(String queuePath) {
+ public boolean isAutoExpiredDeletionEnabled(QueuePath queuePath) {
boolean isAutoExpiredDeletionEnabled = getBoolean(
getQueuePrefix(queuePath) +
AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE,
@@ -2524,7 +2492,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
@VisibleForTesting
- public void setAutoExpiredDeletionEnabled(String queuePath,
+ public void setAutoExpiredDeletionEnabled(QueuePath queuePath,
boolean autoRemovalEnable) {
setBoolean(getQueuePrefix(queuePath) +
AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE,
@@ -2562,7 +2530,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* @return Configured policy class name
*/
@Private
- public String getAutoCreatedQueueManagementPolicy(String queue) {
+ public String getAutoCreatedQueueManagementPolicy(QueuePath queue) {
String autoCreatedQueueManagementPolicy =
get(getQueuePrefix(queue) + AUTO_CREATED_QUEUE_MANAGEMENT_POLICY,
DEFAULT_AUTO_CREATED_QUEUE_MANAGEMENT_POLICY);
@@ -2573,19 +2541,19 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* Get The policy class configured to manage capacities for auto created leaf
* queues under the specified parent
*
- * @param queueName The parent queue's name
+ * @param queue The parent queue's path
* @return The policy class configured to manage capacities for auto created
* leaf queues under the specified parent queue
*/
@Private
protected AutoCreatedQueueManagementPolicy
getAutoCreatedQueueManagementPolicyClass(
- String queueName) {
+ QueuePath queue) {
String queueManagementPolicyClassName =
- getAutoCreatedQueueManagementPolicy(queueName);
+ getAutoCreatedQueueManagementPolicy(queue);
LOG.info("Using Auto Created Queue Management Policy: "
- + queueManagementPolicyClassName + " for queue: " + queueName);
+ + queueManagementPolicyClassName + " for queue: " + queue.getFullPath());
try {
Class<?> queueManagementPolicyClazz = getClassByName(
queueManagementPolicyClassName);
@@ -2601,36 +2569,36 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
} catch (ClassNotFoundException e) {
throw new YarnRuntimeException(
"Could not instantiate " + "AutoCreatedQueueManagementPolicy: "
- + queueManagementPolicyClassName + " for queue: " + queueName,
+ + queueManagementPolicyClassName + " for queue: " + queue.getFullPath(),
e);
}
}
@VisibleForTesting
@Private
- public void setAutoCreatedLeafQueueConfigCapacity(String queuePath,
+ public void setAutoCreatedLeafQueueConfigCapacity(QueuePath queuePath,
float val) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setCapacity(leafQueueConfPrefix, val);
}
@VisibleForTesting
@Private
- public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath,
+ public void setAutoCreatedLeafQueueTemplateCapacityByLabel(QueuePath queuePath,
String label, float val) {
- String leafQueueConfPrefix =
- getAutoCreatedQueueTemplateConfPrefix(queuePath);
+ QueuePath leafQueueConfPrefix =
+ getAutoCreatedQueueObjectTemplateConfPrefix(queuePath);
setCapacityByLabel(leafQueueConfPrefix, label, val);
}
@VisibleForTesting
@Private
- public void setAutoCreatedLeafQueueTemplateCapacityByLabel(String queuePath,
+ public void setAutoCreatedLeafQueueTemplateCapacityByLabel(QueuePath queuePath,
String label, Resource resource) {
- String leafQueueConfPrefix =
- getAutoCreatedQueueTemplateConfPrefix(queuePath);
+ QueuePath leafQueueConfPrefix =
+ getAutoCreatedQueueObjectTemplateConfPrefix(queuePath);
StringBuilder resourceString = new StringBuilder();
@@ -2647,27 +2615,27 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
@VisibleForTesting
- public void setAutoCreatedLeafQueueConfigMaxCapacity(String queuePath,
+ public void setAutoCreatedLeafQueueConfigMaxCapacity(QueuePath queuePath,
float val) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setMaximumCapacity(leafQueueConfPrefix, val);
}
@Private
@VisibleForTesting
- public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath,
+ public void setAutoCreatedLeafQueueTemplateMaxCapacity(QueuePath queuePath,
String label, float val) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setMaximumCapacityByLabel(leafQueueConfPrefix, label, val);
}
@Private
@VisibleForTesting
- public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath,
+ public void setAutoCreatedLeafQueueTemplateMaxCapacity(QueuePath queuePath,
String label, Resource resource) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
StringBuilder resourceString = new StringBuilder();
@@ -2685,37 +2653,37 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@VisibleForTesting
@Private
- public void setAutoCreatedLeafQueueConfigUserLimit(String queuePath,
+ public void setAutoCreatedLeafQueueConfigUserLimit(QueuePath queuePath,
int val) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setUserLimit(leafQueueConfPrefix, val);
}
@VisibleForTesting
@Private
- public void setAutoCreatedLeafQueueConfigUserLimitFactor(String queuePath,
+ public void setAutoCreatedLeafQueueConfigUserLimitFactor(QueuePath queuePath,
float val) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setUserLimitFactor(leafQueueConfPrefix, val);
}
@Private
@VisibleForTesting
- public void setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression(String
+ public void setAutoCreatedLeafQueueConfigDefaultNodeLabelExpression(QueuePath
queuePath,
String expression) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setDefaultNodeLabelExpression(leafQueueConfPrefix, expression);
}
@Private
@VisibleForTesting
- public void setAutoCreatedLeafQueueConfigMaximumAllocation(String
+ public void setAutoCreatedLeafQueueConfigMaximumAllocation(QueuePath
queuePath, String expression) {
- String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ QueuePath leafQueueConfPrefix = getAutoCreatedQueueObjectTemplateConfPrefix(
queuePath);
setQueueMaximumAllocation(leafQueueConfPrefix, expression);
}
@@ -2744,7 +2712,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* Resource types
* @return ResourceInformation
*/
- public Resource getMinimumResourceRequirement(String label, String queue,
+ public Resource getMinimumResourceRequirement(String label, QueuePath queue,
Set<String> resourceTypes) {
return internalGetLabeledResourceRequirementForQueue(queue, label,
resourceTypes, CAPACITY);
@@ -2761,7 +2729,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
* Resource types
* @return Resource
*/
- public Resource getMaximumResourceRequirement(String label, String queue,
+ public Resource getMaximumResourceRequirement(String label, QueuePath queue,
Set<String> resourceTypes) {
return internalGetLabeledResourceRequirementForQueue(queue, label,
resourceTypes, MAXIMUM_CAPACITY);
@@ -2780,10 +2748,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
public Map<String, QueueCapacityVector> parseConfiguredResourceVector(
- String queuePath, Set<String> labels) {
+ QueuePath queuePath, Set<String> labels) {
Map<String, QueueCapacityVector> queueResourceVectors = new HashMap<>();
for (String label : labels) {
- String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix(
+ String propertyName = QueuePrefixes.getNodeLabelPrefix(
queuePath, label) + CapacitySchedulerConfiguration.CAPACITY;
String capacityString = get(propertyName);
queueResourceVectors.put(label, queueCapacityConfigParser.parse(capacityString, queuePath));
@@ -2793,10 +2761,10 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
public Map<String, QueueCapacityVector> parseConfiguredMaximumCapacityVector(
- String queuePath, Set<String> labels, QueueCapacityVector defaultVector) {
+ QueuePath queuePath, Set<String> labels, QueueCapacityVector defaultVector) {
Map<String, QueueCapacityVector> queueResourceVectors = new HashMap<>();
for (String label : labels) {
- String propertyName = CapacitySchedulerConfiguration.getNodeLabelPrefix(
+ String propertyName = QueuePrefixes.getNodeLabelPrefix(
queuePath, label) + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY;
String capacityString = get(propertyName);
QueueCapacityVector capacityVector = queueCapacityConfigParser.parse(capacityString,
@@ -2827,15 +2795,15 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
+ ResourceUtils.
getCustomResourcesStrings(resource) + "]");
- String prefix = getQueuePrefix(queue.getFullPath()) + type;
+ String prefix = getQueuePrefix(queue) + type;
if (!label.isEmpty()) {
- prefix = getQueuePrefix(queue.getFullPath()) + ACCESSIBLE_NODE_LABELS + DOT + label
+ prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label
+ DOT + type;
}
set(prefix, resourceString.toString());
}
- public boolean checkConfigTypeIsAbsoluteResource(String label, String queue,
+ public boolean checkConfigTypeIsAbsoluteResource(String label, QueuePath queue,
Set<String> resourceTypes) {
String propertyName = getNodeLabelPrefix(queue, label) + CAPACITY;
String resourceString = get(propertyName);
@@ -2850,7 +2818,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return false;
}
- private Resource internalGetLabeledResourceRequirementForQueue(String queue,
+ private Resource internalGetLabeledResourceRequirementForQueue(QueuePath queue,
String label, Set<String> resourceTypes, String suffix) {
String propertyName = getNodeLabelPrefix(queue, label) + suffix;
String resourceString = get(propertyName);
@@ -2969,7 +2937,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final boolean DEFAULT_MULTI_NODE_PLACEMENT_ENABLED = false;
public String getMultiNodesSortingAlgorithmPolicy(
- String queue) {
+ QueuePath queue) {
String policyName = get(
getQueuePrefix(queue) + "multi-node-sorting.policy");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 9e4dc55708d5..bf9df5b694f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -20,8 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -239,46 +237,47 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
CSQueue parent, String queueName, CSQueueStore newQueues, CSQueueStore oldQueues,
QueueHook hook) throws IOException {
CSQueue queue;
- String fullQueueName = (parent == null) ? queueName :
- (QueuePath.createFromQueues(parent.getQueuePath(), queueName).getFullPath());
- String[] staticChildQueueNames = conf.getQueues(fullQueueName);
- List<String> childQueueNames = staticChildQueueNames != null ?
- Arrays.asList(staticChildQueueNames) : Collections.emptyList();
- CSQueue oldQueue = oldQueues.get(fullQueueName);
-
- boolean isReservableQueue = conf.isReservable(fullQueueName);
- boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(fullQueueName);
+ QueuePath queueToParse = (parent == null) ? new QueuePath(queueName) :
+ (QueuePath.createFromQueues(parent.getQueuePath(), queueName));
+ List<String> childQueueNames = conf.getQueues(queueToParse);
+ CSQueue oldQueue = oldQueues.get(queueToParse.getFullPath());
+
+ boolean isReservableQueue = conf.isReservable(queueToParse);
+ boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(queueToParse);
// if a queue is eligible for auto queue creation v2 it must be a ParentQueue
// (even if it is empty)
final boolean isDynamicParent = oldQueue instanceof AbstractParentQueue &&
oldQueue.isDynamicQueue();
boolean isAutoQueueCreationEnabledParent = isDynamicParent || conf.isAutoQueueCreationV2Enabled(
- fullQueueName) || isAutoCreateEnabled;
+ queueToParse) || isAutoCreateEnabled;
if (childQueueNames.size() == 0 && !isAutoQueueCreationEnabledParent) {
validateParent(parent, queueName);
// Check if the queue will be dynamically managed by the Reservation system
if (isReservableQueue) {
- queue = new PlanQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName));
+ queue = new PlanQueue(queueContext, queueName, parent,
+ oldQueues.get(queueToParse.getFullPath()));
ReservationQueue defaultResQueue = ((PlanQueue) queue).initializeDefaultInternalQueue();
newQueues.add(defaultResQueue);
} else {
- queue = new LeafQueue(queueContext, queueName, parent, oldQueues.get(fullQueueName));
+ queue = new LeafQueue(queueContext, queueName, parent,
+ oldQueues.get(queueToParse.getFullPath()));
}
queue = hook.hook(queue);
} else {
if (isReservableQueue) {
- throw new IllegalStateException("Only Leaf Queues can be reservable for " + fullQueueName);
+ throw new IllegalStateException("Only Leaf Queues can be reservable for " +
+ queueToParse.getFullPath());
}
AbstractParentQueue parentQueue;
if (isAutoCreateEnabled) {
parentQueue = new ManagedParentQueue(queueContext, queueName, parent, oldQueues.get(
- fullQueueName));
+ queueToParse.getFullPath()));
} else {
parentQueue = new ParentQueue(queueContext, queueName, parent, oldQueues.get(
- fullQueueName));
+ queueToParse.getFullPath()));
}
queue = hook.hook(parentQueue);
@@ -297,7 +296,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
newQueues.add(queue);
- LOG.info("Initialized queue: " + fullQueueName);
+ LOG.info("Initialized queue: " + queueToParse.getFullPath());
return queue;
}
@@ -323,7 +322,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
boolean isRemovable = isDanglingDynamicQueue || !isDynamicQueue(queue)
&& newQueues.get(queue.getQueuePath()) == null
&& !(queue instanceof AutoCreatedLeafQueue &&
- conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePath()));
+ conf.isAutoCreateChildQueueEnabled(queue.getParent().getQueuePathObject()));
if (isRemovable) {
existingQueues.remove(queue);
@@ -513,7 +512,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
CSQueue parentQueue = getQueue(parentQueueName);
if (parentQueue != null && csContext.getConfiguration()
- .isAutoCreateChildQueueEnabled(parentQueue.getQueuePath())) {
+ .isAutoCreateChildQueueEnabled(parentQueue.getQueuePathObject())) {
return createLegacyAutoQueue(queue);
} else {
return createAutoQueue(queue);
@@ -584,7 +583,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
}
int maximumDepthOfStaticParent = csContext.getConfiguration().getMaximumAutoCreatedQueueDepth(
- firstExistingStaticParent.getQueuePath());
+ firstExistingStaticParent.getQueuePathObject());
if (firstStaticParentDistance > maximumDepthOfStaticParent) {
throw new SchedulerDynamicEditException(
"Could not auto create queue " + queue.getFullPath()
@@ -634,7 +633,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
if (parentQueue instanceof AbstractManagedParentQueue) {
// An AbstractManagedParentQueue must have been found for Legacy AQC
permissions.add(new Permission(privilegedEntity,
- csConf.getACLsForLegacyAutoCreatedLeafQueue(queuePath.getParent())));
+ csConf.getACLsForLegacyAutoCreatedLeafQueue(queuePath.getParentObject())));
} else {
// Every other case must be a Flexible Leaf Queue
permissions.add(new Permission(privilegedEntity,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
index 66dd316be744..fa749f077964 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
@@ -39,7 +39,6 @@ import java.util.Set;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueueCapacityConfigParser;
@@ -66,7 +65,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
queueContext.getConfiguration()
.getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
- getQueuePath());
+ getQueuePathObject());
leafQueueTemplate = initializeLeafQueueConfigs().build();
@@ -84,7 +83,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
queueContext.getConfiguration()
.getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
- getQueuePath());
+ getQueuePathObject());
//validate if capacity is exceeded for child queues
if (shouldFailAutoCreationWhenGuaranteedCapacityExceeded) {
@@ -134,7 +133,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
private void initializeQueueManagementPolicy() throws IOException {
queueManagementPolicy =
queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
- getQueuePath());
+ getQueuePathObject());
queueManagementPolicy.init(this);
}
@@ -142,7 +141,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
private void reinitializeQueueManagementPolicy() throws IOException {
AutoCreatedQueueManagementPolicy managementPolicy =
queueContext.getConfiguration().getAutoCreatedQueueManagementPolicyClass(
- getQueuePath());
+ getQueuePathObject());
if (!(managementPolicy.getClass().equals(
this.queueManagementPolicy.getClass()))) {
@@ -162,8 +161,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
queueContext.getConfiguration();
// TODO load configs into CapacitySchedulerConfiguration instead of duplicating them
- String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
- configuration);
+ String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix();
//Load template configuration into CapacitySchedulerConfiguration
CapacitySchedulerConfiguration autoCreatedTemplateConfig =
super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix);
@@ -171,8 +169,8 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
QueueResourceQuotas queueResourceQuotas = new QueueResourceQuotas();
setAbsoluteResourceTemplates(configuration, queueResourceQuotas);
- QueuePath templateQueuePath = configuration
- .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath());
+ QueuePath templateQueuePath = QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject());
Set<String> templateConfiguredNodeLabels = queueContext
.getQueueManager().getConfiguredNodeLabelsForAllQueues()
.getLabelsByQueue(templateQueuePath.getFullPath());
@@ -199,15 +197,15 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
private void setAbsoluteResourceTemplates(CapacitySchedulerConfiguration configuration,
QueueResourceQuotas queueResourceQuotas) throws IOException {
- QueuePath templateQueuePath = configuration
- .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePath());
+ QueuePath templateQueuePath = QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject());
Set<String> templateConfiguredNodeLabels = queueContext
.getQueueManager().getConfiguredNodeLabelsForAllQueues()
.getLabelsByQueue(templateQueuePath.getFullPath());
for (String nodeLabel : templateConfiguredNodeLabels) {
Resource templateMinResource = configuration.getMinimumResourceRequirement(
- nodeLabel, templateQueuePath.getFullPath(), resourceTypes);
+ nodeLabel, templateQueuePath, resourceTypes);
queueResourceQuotas.setConfiguredMinResource(nodeLabel, templateMinResource);
if (this.capacityConfigType.equals(CapacityConfigType.PERCENTAGE)
@@ -228,15 +226,15 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
queueContext.getClusterResource(),
configuration.getMinimumResourceRequirement(
label,
- configuration
- .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
+ QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()),
resourceTypes),
getQueueResourceQuotas().getConfiguredMinResource(label)));
Resource childMaxResource = configuration
.getMaximumResourceRequirement(label,
- configuration
- .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
+ QueuePrefixes
+ .getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()),
resourceTypes);
Resource parentMaxRes = getQueueResourceQuotas()
.getConfiguredMaxResource(label);
@@ -296,7 +294,7 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
String leafQueuePath = childQueue.getQueuePath();
int maxQueues = conf.getAutoCreatedQueuesMaxChildQueuesLimit(
- parentQueue.getQueuePath());
+ parentQueue.getQueuePathObject());
if (parentQueue.getChildQueues().size() >= maxQueues) {
throw new SchedulerDynamicEditException(
@@ -355,14 +353,15 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
.getLabelsByQueue(queuePath.getFullPath());
for (String label : templateConfiguredNodeLabels) {
final String leafConfigPath =
- CapacitySchedulerConfiguration.getNodeLabelPrefix(
- getQueuePath() + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX, label);
+ QueuePrefixes.getNodeLabelPrefix(
+ QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix(getQueuePathObject()),
+ label);
String capacityString = leafConfig.get(leafConfigPath + CAPACITY, "0");
leafQueue.setConfiguredMinCapacityVector(label,
- getQueueCapacityConfigParser().parse(capacityString, leafQueue.getQueuePath()));
+ getQueueCapacityConfigParser().parse(capacityString, leafQueue.getQueuePathObject()));
String maxCapacityString = leafConfig.get(leafConfigPath + MAXIMUM_CAPACITY, "100");
leafQueue.setConfiguredMaxCapacityVector(label,
- getQueueCapacityConfigParser().parse(maxCapacityString, leafQueue.getQueuePath()));
+ getQueueCapacityConfigParser().parse(maxCapacityString, leafQueue.getQueuePathObject()));
}
}
@@ -405,9 +404,9 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
}
}
- public String getLeafQueueConfigPrefix(CapacitySchedulerConfiguration conf) {
- return CapacitySchedulerConfiguration.PREFIX + conf
- .getAutoCreatedQueueTemplateConfPrefix(getQueuePath());
+ public String getLeafQueueConfigPrefix() {
+ return CapacitySchedulerConfiguration.PREFIX + QueuePrefixes
+ .getAutoCreatedQueueTemplateConfPrefix(getQueuePathObject());
}
public boolean shouldFailAutoCreationWhenGuaranteedCapacityExceeded() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 0b9f36779cfb..aac901d6d091 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -75,7 +75,7 @@ public class ParentQueue extends AbstractParentQueue {
// Check if the max queue limit is exceeded.
int maxQueues = queueContext.getConfiguration().
- getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePath());
+ getAutoCreatedQueuesV2MaxChildQueuesLimit(getQueuePathObject());
if (childQueues.size() >= maxQueues) {
throw new SchedulerDynamicEditException(
"Cannot auto create queue " + childQueuePath + ". Max Child "
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 903539fcf185..847c90e54589 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -52,7 +52,7 @@ public class PlanQueue extends AbstractManagedParentQueue {
// Set the reservation queue attributes for the Plan
CapacitySchedulerConfiguration conf = queueContext.getConfiguration();
- String queuePath = super.getQueuePath();
+ QueuePath queuePath = super.getQueuePathObject();
int maxAppsForReservation = conf.getMaximumApplicationsPerQueue(queuePath);
showReservationsAsQueues = conf.getShowReservationAsQueues(queuePath);
if (maxAppsForReservation < 0) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java
index 101c8076fdc0..7028423e8c22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAllocationSettings.java
@@ -36,7 +36,7 @@ public class QueueAllocationSettings {
this.minimumAllocation = minimumAllocation;
}
- void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, String queuePath,
+ void setupMaximumAllocation(CapacitySchedulerConfiguration configuration, QueuePath queuePath,
CSQueue parent) {
Resource clusterMax = ResourceUtils
.fetchMaximumAllocationFromConfig(configuration);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java
index 2e71d8eef3ed..9218c877e71b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAppLifetimeAndLimitSettings.java
@@ -37,7 +37,7 @@ public class QueueAppLifetimeAndLimitSettings {
public QueueAppLifetimeAndLimitSettings(CapacitySchedulerConfiguration configuration,
AbstractCSQueue q, QueuePath queuePath) {
// Store max parallel apps property
- this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath.getFullPath());
+ this.maxParallelApps = configuration.getMaxParallelAppsForQueue(queuePath);
this.maxApplicationLifetime = getInheritedMaxAppLifetime(q, configuration);
this.defaultApplicationLifetime = setupInheritedDefaultAppLifetime(q, queuePath, configuration,
maxApplicationLifetime);
@@ -45,7 +45,7 @@ public class QueueAppLifetimeAndLimitSettings {
private long getInheritedMaxAppLifetime(CSQueue q, CapacitySchedulerConfiguration conf) {
CSQueue parentQ = q.getParent();
- long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePath());
+ long maxAppLifetime = conf.getMaximumLifetimePerQueue(q.getQueuePathObject());
// If q is the root queue, then get max app lifetime from conf.
if (q.getQueuePathObject().isRoot()) {
@@ -64,7 +64,7 @@ public class QueueAppLifetimeAndLimitSettings {
private long setupInheritedDefaultAppLifetime(CSQueue q,
QueuePath queuePath, CapacitySchedulerConfiguration conf, long myMaxAppLifetime) {
CSQueue parentQ = q.getParent();
- long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath.getFullPath());
+ long defaultAppLifetime = conf.getDefaultLifetimePerQueue(queuePath);
defaultAppLifetimeWasSpecifiedInConfig =
(defaultAppLifetime >= 0
|| (!queuePath.isRoot() &&
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java
index c431d2bb4554..adf5f6e0ffcb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueNodeLabelsSettings.java
@@ -53,7 +53,7 @@ public class QueueNodeLabelsSettings {
}
private void initializeAccessibleLabels(CapacitySchedulerConfiguration configuration) {
- this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath.getFullPath());
+ this.accessibleLabels = configuration.getAccessibleNodeLabels(queuePath);
// Inherit labels from parent if not set
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
@@ -62,7 +62,7 @@ public class QueueNodeLabelsSettings {
private void initializeDefaultLabelExpression(CapacitySchedulerConfiguration configuration) {
this.defaultLabelExpression = configuration.getDefaultNodeLabelExpression(
- queuePath.getFullPath());
+ queuePath);
// If the accessible labels is not null and the queue has a parent with a
// similar set of labels copy the defaultNodeLabelExpression from the parent
if (this.accessibleLabels != null && parent != null
@@ -83,7 +83,7 @@ public class QueueNodeLabelsSettings {
}
} else {
// Fallback to suboptimal but correct logic
- this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath.getFullPath());
+ this.configuredNodeLabels = configuration.getConfiguredNodeLabels(queuePath);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java
index 692be1132eac..4e34c3a6ee93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePath.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
+import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.WILDCARD_QUEUE;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
@@ -110,6 +113,14 @@ public class QueuePath implements Iterable<String> {
return false;
}
+ /**
+ * Simple helper method to determine if the queue path is invalid or not.
+ * @return true if the queue path is invalid.
+ */
+ public boolean isInvalid() {
+ return getPathComponents().length <= 1 && !isRoot();
+ }
+
/**
* Getter for the parent part of the path.
* @return Parent path of the queue, null if there is no parent.
@@ -118,6 +129,14 @@ public class QueuePath implements Iterable<String> {
return parent;
}
+ /**
+ * Getter for the parent object of the path.
+ * @return Parent QueuePath object of the queue, null if there is no parent.
+ */
+ public QueuePath getParentObject() {
+ return hasParent() ? new QueuePath(parent) : null;
+ }
+
/**
* Getter for the leaf part of the path.
* @return The leaf queue name
@@ -167,7 +186,7 @@ public class QueuePath implements Iterable<String> {
*/
@Override
public Iterator<String> iterator() {
- return Arrays.asList(getFullPath().split(QUEUE_REGEX_DELIMITER)).iterator();
+ return Arrays.asList(getPathComponents()).iterator();
}
/**
@@ -204,6 +223,54 @@ public class QueuePath implements Iterable<String> {
};
}
+ /**
+ * Returns the list of wildcarded queue paths based on the autoCreatedQueueDepth config value.
+ * An example template precedence hierarchy for root.a ParentQueue from highest to lowest:
+ * yarn.scheduler.capacity.root.a.auto-queue-creation-v2.template.capacity
+ * yarn.scheduler.capacity.root.*.auto-queue-creation-v2.template.capacity
+ * @param maxAutoCreatedQueueDepth the maximum depth of auto-created queues stored in the
+ * configuration
+ * @return list of wildcarded QueuePath objects
+ */
+ public List<QueuePath> getWildcardedQueuePaths(int maxAutoCreatedQueueDepth) {
+ List<QueuePath> wildcardedPaths = new ArrayList<>();
+ // Start with the most explicit format (without wildcard)
+ wildcardedPaths.add(this);
+
+ String[] pathComponents = getPathComponents();
+ int supportedWildcardLevel = getSupportedWildcardLevel(maxAutoCreatedQueueDepth);
+
+ // Collect all template entries
+ for (int wildcardLevel = 1; wildcardLevel <= supportedWildcardLevel; wildcardLevel++) {
+ int wildcardedComponentIndex = pathComponents.length - wildcardLevel;
+ pathComponents[wildcardedComponentIndex] = WILDCARD_QUEUE;
+ QueuePath wildcardedPath = createFromQueues(pathComponents);
+ wildcardedPaths.add(wildcardedPath);
+ }
+
+ return wildcardedPaths;
+ }
+
+ /**
+ * Returns the supported wildcard level for this queue path.
+ * @param maxAutoCreatedQueueDepth the maximum depth of auto-created queues stored in the
+ * configuration
+ * @return int value of the supported wildcard level
+ */
+ private int getSupportedWildcardLevel(int maxAutoCreatedQueueDepth) {
+ int queuePathMaxIndex = getPathComponents().length - 1;
+ // Allow root to have template properties
+ return isRoot() ? 0 : Math.min(queuePathMaxIndex, maxAutoCreatedQueueDepth);
+ }
+
+ /**
+ * Returns queue path components.
+ * @return String array containing the queue names.
+ */
+ public String[] getPathComponents() {
+ return getFullPath().split(QUEUE_REGEX_DELIMITER);
+ }
+
@Override
public String toString() {
return getFullPath();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java
new file mode 100644
index 000000000000..75b7f77a5339
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueuePrefixes.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ACCESSIBLE_NODE_LABELS;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+
+public final class QueuePrefixes {
+
+ private QueuePrefixes() {
+ }
+
+ public static String getQueuePrefix(QueuePath queuePath) {
+ return PREFIX + queuePath.getFullPath() + DOT;
+ }
+
+ public static String getNodeLabelPrefix(QueuePath queuePath, String label) {
+ if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
+ return getQueuePrefix(queuePath);
+ }
+ return getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT;
+ }
+
+ /**
+ * Get the auto created leaf queue's template configuration prefix.
+ * Leaf queue's template capacities are configured at the parent queue.
+ *
+ * @param queuePath parent queue's path
+ * @return Config prefix for leaf queue template configurations
+ */
+ public static String getAutoCreatedQueueTemplateConfPrefix(QueuePath queuePath) {
+ return queuePath.getFullPath() + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
+ }
+
+ public static QueuePath getAutoCreatedQueueObjectTemplateConfPrefix(QueuePath queuePath) {
+ return new QueuePath(getAutoCreatedQueueTemplateConfPrefix(queuePath));
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java
index 5ec7d01bce9b..7e561c6fe074 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueStateHelper.java
@@ -42,7 +42,7 @@ public final class QueueStateHelper {
public static void setQueueState(AbstractCSQueue queue) {
QueueState previousState = queue.getState();
QueueState configuredState = queue.getQueueContext().getConfiguration().getConfiguredState(
- queue.getQueuePath());
+ queue.getQueuePathObject());
QueueState parentState = (queue.getParent() == null) ? null : queue.getParent().getState();
// verify that we can not any value for State other than RUNNING/STOPPED
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
index a9d755227ffc..6535de1c61d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserWeights.java
@@ -24,7 +24,7 @@ import java.util.regex.Matcher;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_SETTINGS;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_WEIGHT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.USER_WEIGHT_PATTERN;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix;
public final class UserWeights {
public static final float DEFAULT_WEIGHT = 1.0F;
@@ -43,7 +43,7 @@ public final class UserWeights {
public static UserWeights createByConfig(
CapacitySchedulerConfiguration conf,
ConfigurationProperties configurationProperties,
- String queuePath) {
+ QueuePath queuePath) {
String queuePathPlusPrefix = getQueuePrefix(queuePath) + USER_SETTINGS;
Map<String, String> props = configurationProperties
.getPropertiesWithPrefix(queuePathPlusPrefix);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
index 88c93019680c..6213edb6a76f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ConfigurationUpdateAssembler.java
@@ -21,18 +21,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
-
public final class ConfigurationUpdateAssembler {
private ConfigurationUpdateAssembler() {
@@ -65,38 +64,34 @@ public final class ConfigurationUpdateAssembler {
if (queueToRemove == null) {
return;
}
- if (queueToRemove.lastIndexOf('.') == -1) {
- throw new IOException("Can't remove queue " + queueToRemove);
+ QueuePath queuePath = new QueuePath(queueToRemove);
+ if (queuePath.isRoot() || queuePath.isInvalid()) {
+ throw new IOException("Can't remove queue " + queuePath.getFullPath());
}
- String queueName = queueToRemove.substring(
- queueToRemove.lastIndexOf('.') + 1);
- List<String> siblingQueues = getSiblingQueues(queueToRemove,
+ String queueName = queuePath.getLeafName();
+ List<String> siblingQueues = getSiblingQueues(queuePath,
proposedConf);
if (!siblingQueues.contains(queueName)) {
- throw new IOException("Queue " + queueToRemove + " not found");
+ throw new IOException("Queue " + queuePath.getFullPath() + " not found");
}
siblingQueues.remove(queueName);
- String parentQueuePath = queueToRemove.substring(0, queueToRemove
- .lastIndexOf('.'));
- proposedConf.setQueues(parentQueuePath, siblingQueues.toArray(
+
+ QueuePath parentPath = queuePath.getParentObject();
+ proposedConf.setQueues(parentPath, siblingQueues.toArray(
new String[0]));
- String queuesConfig = CapacitySchedulerConfiguration.PREFIX
- + parentQueuePath + CapacitySchedulerConfiguration.DOT
- + CapacitySchedulerConfiguration.QUEUES;
+ String queuesConfig = getQueuesConfig(parentPath);
if (siblingQueues.isEmpty()) {
confUpdate.put(queuesConfig, null);
// Unset Ordering Policy of Leaf Queue converted from
// Parent Queue after removeQueue
- String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
- + parentQueuePath + CapacitySchedulerConfiguration.DOT
- + ORDERING_POLICY;
+ String queueOrderingPolicy = getOrderingPolicyConfig(parentPath);
proposedConf.unset(queueOrderingPolicy);
confUpdate.put(queueOrderingPolicy, null);
} else {
confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
}
for (Map.Entry<String, String> confRemove : proposedConf.getValByRegex(
- ".*" + queueToRemove + "\\..*")
+ ".*" + queuePath.getFullPath() + "\\..*")
.entrySet()) {
proposedConf.unset(confRemove.getKey());
confUpdate.put(confRemove.getKey(), null);
@@ -109,27 +104,23 @@ public final class ConfigurationUpdateAssembler {
if (addInfo == null) {
return;
}
- String queuePath = addInfo.getQueue();
- String queueName = queuePath.substring(queuePath.lastIndexOf('.') + 1);
- if (queuePath.lastIndexOf('.') == -1) {
+ QueuePath queuePath = new QueuePath(addInfo.getQueue());
+ String queueName = queuePath.getLeafName();
+ if (queuePath.isRoot() || queuePath.isInvalid()) {
throw new IOException("Can't add invalid queue " + queuePath);
} else if (getSiblingQueues(queuePath, proposedConf).contains(
queueName)) {
throw new IOException("Can't add existing queue " + queuePath);
}
- String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
- String[] siblings = proposedConf.getQueues(parentQueue);
- List<String> siblingQueues = siblings == null ? new ArrayList<>() :
- new ArrayList<>(Arrays.asList(siblings));
- siblingQueues.add(queuePath.substring(queuePath.lastIndexOf('.') + 1));
- proposedConf.setQueues(parentQueue,
+
+ QueuePath parentPath = queuePath.getParentObject();
+ List<String> siblingQueues = proposedConf.getQueues(parentPath);
+ siblingQueues.add(queueName);
+ proposedConf.setQueues(parentPath,
siblingQueues.toArray(new String[0]));
- confUpdate.put(CapacitySchedulerConfiguration.PREFIX
- + parentQueue + CapacitySchedulerConfiguration.DOT
- + CapacitySchedulerConfiguration.QUEUES,
+ confUpdate.put(getQueuesConfig(parentPath),
Joiner.on(',').join(siblingQueues));
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX
- + queuePath + CapacitySchedulerConfiguration.DOT;
+ String keyPrefix = QueuePrefixes.getQueuePrefix(queuePath);
for (Map.Entry<String, String> kv : addInfo.getParams().entrySet()) {
String keyValue = kv.getValue();
if (keyValue == null || keyValue.isEmpty()) {
@@ -142,8 +133,7 @@ public final class ConfigurationUpdateAssembler {
}
// Unset Ordering Policy of Parent Queue converted from
// Leaf Queue after addQueue
- String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
- + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ String queueOrderingPolicy = getOrderingPolicyConfig(parentPath);
if (siblingQueues.size() == 1) {
proposedConf.unset(queueOrderingPolicy);
confUpdate.put(queueOrderingPolicy, null);
@@ -156,9 +146,8 @@ public final class ConfigurationUpdateAssembler {
if (updateInfo == null) {
return;
}
- String queuePath = updateInfo.getQueue();
- String keyPrefix = CapacitySchedulerConfiguration.PREFIX
- + queuePath + CapacitySchedulerConfiguration.DOT;
+ QueuePath queuePath = new QueuePath(updateInfo.getQueue());
+ String keyPrefix = QueuePrefixes.getQueuePrefix(queuePath);
for (Map.Entry<String, String> kv : updateInfo.getParams().entrySet()) {
String keyValue = kv.getValue();
if (keyValue == null || keyValue.isEmpty()) {
@@ -171,11 +160,16 @@ public final class ConfigurationUpdateAssembler {
}
}
- private static List<String> getSiblingQueues(String queuePath, Configuration conf) {
- String parentQueue = queuePath.substring(0, queuePath.lastIndexOf('.'));
- String childQueuesKey = CapacitySchedulerConfiguration.PREFIX +
- parentQueue + CapacitySchedulerConfiguration.DOT +
- CapacitySchedulerConfiguration.QUEUES;
+ private static List<String> getSiblingQueues(QueuePath queuePath, Configuration conf) {
+ String childQueuesKey = getQueuesConfig(queuePath.getParentObject());
return new ArrayList<>(conf.getTrimmedStringCollection(childQueuesKey));
}
+
+ private static String getQueuesConfig(QueuePath queuePath) {
+ return QueuePrefixes.getQueuePrefix(queuePath) + CapacitySchedulerConfiguration.QUEUES;
+ }
+
+ private static String getOrderingPolicyConfig(QueuePath queuePath) {
+ return QueuePrefixes.getQueuePrefix(queuePath) + CapacitySchedulerConfiguration.ORDERING_POLICY;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java
index 644938e33e77..5ebaa5c867f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/QueueCapacityConfigParser.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacityVector.ResourceUnitCapacityType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import java.util.ArrayList;
@@ -65,9 +65,9 @@ public class QueueCapacityConfigParser {
* @param queuePath queue for which the capacity property is parsed
* @return a parsed capacity vector
*/
- public QueueCapacityVector parse(String capacityString, String queuePath) {
+ public QueueCapacityVector parse(String capacityString, QueuePath queuePath) {
- if (queuePath.equals(CapacitySchedulerConfiguration.ROOT)) {
+ if (queuePath.isRoot()) {
return QueueCapacityVector.of(100f, ResourceUnitCapacityType.PERCENTAGE);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index b3705c600575..870f7b1d7a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ReservationACL;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.AllocationFileParser;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocation.QueueProperties;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -194,9 +195,9 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
* Get the map of reservation ACLs to {@link AccessControlList} for the
* specified queue.
*/
- public Map<ReservationACL, AccessControlList> getReservationAcls(String
+ public Map<ReservationACL, AccessControlList> getReservationAcls(QueuePath
queue) {
- return this.resAcls.get(queue);
+ return this.resAcls.get(queue.getFullPath());
}
/**
@@ -341,52 +342,52 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
}
@Override
- public boolean isReservable(String queue) {
- return reservableQueues.contains(queue);
+ public boolean isReservable(QueuePath queue) {
+ return reservableQueues.contains(queue.getFullPath());
}
@Override
- public long getReservationWindow(String queue) {
+ public long getReservationWindow(QueuePath queue) {
return globalReservationQueueConfig.getReservationWindowMsec();
}
@Override
- public float getAverageCapacity(String queue) {
+ public float getAverageCapacity(QueuePath queue) {
return globalReservationQueueConfig.getAvgOverTimeMultiplier() * 100;
}
@Override
- public float getInstantaneousMaxCapacity(String queue) {
+ public float getInstantaneousMaxCapacity(QueuePath queue) {
return globalReservationQueueConfig.getMaxOverTimeMultiplier() * 100;
}
@Override
- public String getReservationAdmissionPolicy(String queue) {
+ public String getReservationAdmissionPolicy(QueuePath queue) {
return globalReservationQueueConfig.getReservationAdmissionPolicy();
}
@Override
- public String getReservationAgent(String queue) {
+ public String getReservationAgent(QueuePath queue) {
return globalReservationQueueConfig.getReservationAgent();
}
@Override
- public boolean getShowReservationAsQueues(String queue) {
+ public boolean getShowReservationAsQueues(QueuePath queue) {
return globalReservationQueueConfig.shouldShowReservationAsQueues();
}
@Override
- public String getReplanner(String queue) {
+ public String getReplanner(QueuePath queue) {
return globalReservationQueueConfig.getPlanner();
}
@Override
- public boolean getMoveOnExpiry(String queue) {
+ public boolean getMoveOnExpiry(QueuePath queue) {
return globalReservationQueueConfig.shouldMoveOnExpiry();
}
@Override
- public long getEnforcementWindow(String queue) {
+ public long getEnforcementWindow(QueuePath queue) {
return globalReservationQueueConfig.getEnforcementWindowMsec();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 32a3b7be4be6..dc4e842518c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils.MaxResourceValidationResult;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -1363,7 +1364,11 @@ public class FairScheduler extends
readLock.lock();
try {
FSQueue queue = queueMgr.getQueue(queueName);
- if ((queue == null) || !allocConf.isReservable(queue.getQueueName())) {
+ if (queue == null) {
+ return queueName;
+ }
+ QueuePath queuePath = new QueuePath(queue.getQueueName());
+ if (!allocConf.isReservable(queuePath)) {
return queueName;
}
// Use fully specified name from now on (including root. prefix)
@@ -1373,7 +1378,7 @@ public class FairScheduler extends
queue = queueMgr.getQueue(resQName);
if (queue == null) {
// reservation has terminated during failover
- if (isRecovering && allocConf.getMoveOnExpiry(queueName)) {
+ if (isRecovering && allocConf.getMoveOnExpiry(queuePath)) {
// move to the default child queue of the plan
return getDefaultQueueForPlanQueue(queueName);
}
@@ -1972,7 +1977,7 @@ public class FairScheduler extends
Set<String> planQueues = new HashSet<String>();
for (FSQueue fsQueue : queueMgr.getQueues()) {
String queueName = fsQueue.getName();
- if (allocConf.isReservable(queueName)) {
+ if (allocConf.isReservable(new QueuePath(queueName))) {
planQueues.add(queueName);
}
}
@@ -2013,7 +2018,7 @@ public class FairScheduler extends
private String handleMoveToPlanQueue(String targetQueueName) {
FSQueue dest = queueMgr.getQueue(targetQueueName);
- if (dest != null && allocConf.isReservable(dest.getQueueName())) {
+ if (dest != null && allocConf.isReservable(new QueuePath(dest.getQueueName()))) {
// use the default child reservation queue of the plan
targetQueueName = getDefaultQueueForPlanQueue(targetQueueName);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
index 6c8fed8182b8..ff47b606c740 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverter.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfigurationException;
@@ -422,13 +423,13 @@ public class FSConfigToCSConfigConverter {
if (!submitAcls.getGroups().isEmpty() ||
!submitAcls.getUsers().isEmpty() || submitAcls.isAllAllowed()) {
- capacitySchedulerConfig.setAcl(queue, QueueACL.SUBMIT_APPLICATIONS,
+ capacitySchedulerConfig.setAcl(new QueuePath(queue), QueueACL.SUBMIT_APPLICATIONS,
submitAcls.getAclString());
}
if (!adminAcls.getGroups().isEmpty() ||
!adminAcls.getUsers().isEmpty() || adminAcls.isAllAllowed()) {
- capacitySchedulerConfig.setAcl(queue, QueueACL.ADMINISTER_QUEUE,
+ capacitySchedulerConfig.setAcl(new QueuePath(queue), QueueACL.ADMINISTER_QUEUE,
adminAcls.getAclString());
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
index 1bbf056c148d..510705ce14cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSQueueConverter.java
@@ -23,6 +23,7 @@ import java.util.stream.Collectors;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.ConfigurableResource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -104,7 +105,8 @@ public class FSQueueConverter {
List<String> childQueues = children.stream()
.map(child -> getQueueShortName(child.getName()))
.collect(Collectors.toList());
- capacitySchedulerConfig.setQueues(queueName, childQueues.toArray(new String[0]));
+ capacitySchedulerConfig.setQueues(new QueuePath(queueName),
+ childQueues.toArray(new String[0]));
}
}
@@ -122,13 +124,13 @@ public class FSQueueConverter {
&& queueMaxAmShare != queueMaxAMShareDefault
&& queueMaxAmShare != QUEUE_MAX_AM_SHARE_DISABLED) {
capacitySchedulerConfig.setMaximumApplicationMasterResourcePerQueuePercent(
- queueName, queueMaxAmShare);
+ new QueuePath(queueName), queueMaxAmShare);
}
if (queueMaxAmShare == QUEUE_MAX_AM_SHARE_DISABLED
&& queueMaxAmShare != queueMaxAMShareDefault) {
capacitySchedulerConfig.setMaximumApplicationMasterResourcePerQueuePercent(
- queueName, 1.0f);
+ new QueuePath(queueName), 1.0f);
}
}
@@ -141,7 +143,7 @@ public class FSQueueConverter {
private void emitMaxParallelApps(String queueName, FSQueue queue) {
if (queue.getMaxRunningApps() != MAX_RUNNING_APPS_UNSET
&& queue.getMaxRunningApps() != queueMaxAppsDefault) {
- capacitySchedulerConfig.setMaxParallelAppsForQueue(queueName,
+ capacitySchedulerConfig.setMaxParallelAppsForQueue(new QueuePath(queueName),
String.valueOf(queue.getMaxRunningApps()));
}
}
@@ -161,7 +163,7 @@ public class FSQueueConverter {
ruleHandler.handleMaxResources();
}
- capacitySchedulerConfig.setMaximumCapacity(queueName,
+ capacitySchedulerConfig.setMaximumCapacity(new QueuePath(queueName),
100.0f);
}
@@ -194,10 +196,10 @@ public class FSQueueConverter {
// only emit max allocation if it differs from the parent's setting
if (maxVcores != parentMaxVcores || maxMemory != parentMaxMemory) {
capacitySchedulerConfig.setQueueMaximumAllocationMb(
- queueName, (int) maxMemory);
+ new QueuePath(queueName), (int) maxMemory);
capacitySchedulerConfig.setQueueMaximumAllocationVcores(
- queueName, maxVcores);
+ new QueuePath(queueName), maxVcores);
}
}
}
@@ -210,14 +212,14 @@ public class FSQueueConverter {
*/
private void emitPreemptionDisabled(String queueName, FSQueue queue) {
if (preemptionEnabled && !queue.isPreemptable()) {
- capacitySchedulerConfig.setPreemptionDisabled(queueName, true);
+ capacitySchedulerConfig.setPreemptionDisabled(new QueuePath(queueName), true);
}
}
public void emitDefaultUserLimitFactor(String queueName, List<FSQueue> children) {
if (children.isEmpty() &&
- !capacitySchedulerConfig.isAutoQueueCreationV2Enabled(queueName)) {
- capacitySchedulerConfig.setUserLimitFactor(queueName, -1.0f);
+ !capacitySchedulerConfig.isAutoQueueCreationV2Enabled(new QueuePath(queueName))) {
+ capacitySchedulerConfig.setUserLimitFactor(new QueuePath(queueName), -1.0f);
}
}
@@ -246,16 +248,16 @@ public class FSQueueConverter {
switch (policy) {
case DominantResourceFairnessPolicy.NAME:
- capacitySchedulerConfig.setOrderingPolicy(queueName, FAIR_POLICY);
+ capacitySchedulerConfig.setOrderingPolicy(new QueuePath(queueName), FAIR_POLICY);
break;
case FairSharePolicy.NAME:
- capacitySchedulerConfig.setOrderingPolicy(queueName, FAIR_POLICY);
+ capacitySchedulerConfig.setOrderingPolicy(new QueuePath(queueName), FAIR_POLICY);
if (drfUsed) {
ruleHandler.handleFairAsDrf(queueName);
}
break;
case FifoPolicy.NAME:
- capacitySchedulerConfig.setOrderingPolicy(queueName, FIFO_POLICY);
+ capacitySchedulerConfig.setOrderingPolicy(new QueuePath(queueName), FIFO_POLICY);
break;
default:
String msg = String.format("Unexpected ordering policy " +
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
index 6ff19e21a6e7..10777a61bd58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.SecondaryGroupExi
import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule.FallbackResult;
@@ -121,7 +122,7 @@ class QueuePlacementConverter {
PlacementRule parentRule = userRule.getParentRule();
boolean parentCreate = ((FSPlacementRule) parentRule).getCreateFlag();
Policy policy;
- String queueName = null;
+ QueuePath queueName = null;
if (parentRule instanceof PrimaryGroupPlacementRule) {
policy = Policy.PRIMARY_GROUP_USER;
@@ -130,7 +131,7 @@ class QueuePlacementConverter {
} else if (parentRule instanceof DefaultPlacementRule) {
DefaultPlacementRule defaultRule = (DefaultPlacementRule) parentRule;
policy = Policy.USER;
- queueName = defaultRule.defaultQueueName;
+ queueName = new QueuePath(defaultRule.defaultQueueName);
} else {
throw new IllegalArgumentException(
"Unsupported parent nested rule: "
@@ -187,7 +188,7 @@ class QueuePlacementConverter {
boolean create,
FSConfigToCSConfigRuleHandler ruleHandler,
boolean fsParentCreate,
- String parentQueue,
+ QueuePath parentQueue,
CapacitySchedulerConfiguration csConf,
boolean usePercentages) {
@@ -196,7 +197,7 @@ class QueuePlacementConverter {
// "parent" is already set to "root" at this point,
// so we override it if necessary
if (parentQueue != null) {
- rule.setParentQueue(parentQueue);
+ rule.setParentQueue(parentQueue.getFullPath());
}
if (usePercentages) {
@@ -207,13 +208,13 @@ class QueuePlacementConverter {
} else if (policy == Policy.SECONDARY_GROUP_USER) {
ruleHandler.handleFSParentCreateFlag("root.<secondaryGroup>");
} else {
- ruleHandler.handleFSParentCreateFlag(parentQueue);
+ ruleHandler.handleFSParentCreateFlag(parentQueue.getFullPath());
}
}
// check if parent conflicts with existing static queues
if (create && policy == Policy.USER) {
- ruleHandler.handleRuleAutoCreateFlag(parentQueue);
+ ruleHandler.handleRuleAutoCreateFlag(parentQueue.getFullPath());
checkStaticDynamicConflict(parentQueue, csConf, ruleHandler);
}
} else {
@@ -230,15 +231,15 @@ class QueuePlacementConverter {
return rule;
}
- private void checkStaticDynamicConflict(String parentPath,
+ private void checkStaticDynamicConflict(QueuePath parentPath,
CapacitySchedulerConfiguration csConf,
FSConfigToCSConfigRuleHandler ruleHandler) {
- String[] childQueues = csConf.getQueues(parentPath);
+ List<String> childQueues = csConf.getQueues(parentPath);
// User must be warned: static + dynamic queues are under the
// same parent
- if (childQueues != null && childQueues.length > 0) {
- ruleHandler.handleChildStaticDynamicConflict(parentPath);
+ if (childQueues != null && childQueues.size() > 0) {
+ ruleHandler.handleChildStaticDynamicConflict(parentPath.getFullPath());
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java
index 600b92626db9..0e437c112e07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToPercentConverter.java
@@ -29,6 +29,7 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.classification.VisibleForTesting;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
public class WeightToPercentConverter
@@ -50,11 +51,11 @@ public class WeightToPercentConverter
boolean shouldAllowZeroSumCapacity = result.getRight();
capacities
- .forEach((key, value) -> csConfig.setCapacity(key, value.toString()));
+ .forEach((key, value) -> csConfig.setCapacity(new QueuePath(key), value.toString()));
if (shouldAllowZeroSumCapacity) {
String queueName = queue.getName();
- csConfig.setAllowZeroCapacitySum(queueName, true);
+ csConfig.setAllowZeroCapacitySum(new QueuePath(queueName), true);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
index 75554fc4d2e6..bd13e7a9a711 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.w
import java.util.List;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -34,13 +35,14 @@ public class WeightToWeightConverter
List<FSQueue> children = queue.getChildQueues();
if (queue instanceof FSParentQueue || !children.isEmpty()) {
+ QueuePath queuePath = new QueuePath(queue.getName());
if (queue.getName().equals(ROOT_QUEUE)) {
- csConfig.setNonLabeledQueueWeight(queue.getName(), queue.getWeight());
+ csConfig.setNonLabeledQueueWeight(queuePath, queue.getWeight());
}
children.forEach(fsQueue -> csConfig.setNonLabeledQueueWeight(
- fsQueue.getName(), fsQueue.getWeight()));
- csConfig.setAutoQueueCreationV2Enabled(queue.getName(), true);
+ new QueuePath(fsQueue.getName()), fsQueue.getWeight()));
+ csConfig.setAutoQueueCreationV2Enabled(queuePath, true);
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java
index 531d2a315331..418f822187fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityReservationsACLsManager.java
@@ -39,7 +39,7 @@ public class CapacityReservationsACLsManager extends ReservationsACLsManager {
for (String planQueue : scheduler.getPlanQueues()) {
CSQueue queue = ((CapacityScheduler) scheduler).getQueue(planQueue);
reservationAcls.put(planQueue,
- csConf.getReservationAcls(queue.getQueuePath()));
+ csConf.getReservationAcls(queue.getQueuePathObject()));
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java
index 09f147f89ea1..a2d05de4e39b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/FairReservationsACLsManager.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
@@ -35,7 +36,7 @@ public class FairReservationsACLsManager extends ReservationsACLsManager {
AllocationConfiguration aConf = ((FairScheduler) scheduler)
.getAllocationConfiguration();
for (String planQueue : scheduler.getPlanQueues()) {
- reservationAcls.put(planQueue, aConf.getReservationAcls(planQueue));
+ reservationAcls.put(planQueue, aConf.getReservationAcls(new QueuePath(planQueue)));
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 5d5b3f26cc1f..dbf57cfa05df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Abstract
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
import java.util.ArrayList;
@@ -100,7 +101,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
- queueAcls.addAll(getSortedQueueAclInfoList(parent, queueName, conf));
+ queueAcls.addAll(getSortedQueueAclInfoList(parent, new QueuePath(queuePath), conf));
queuePriority = parent.getPriority().getPriority();
if (parent instanceof AbstractParentQueue) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 1b5a3714d68c..318adee1caac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -40,6 +40,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
@XmlRootElement
@@ -99,6 +102,7 @@ public class CapacitySchedulerQueueInfo {
CapacitySchedulerQueueInfo(CapacityScheduler cs, CSQueue q) {
queuePath = q.getQueuePath();
+ QueuePath queuePathObject = new QueuePath(queuePath);
capacity = q.getCapacity() * 100;
usedCapacity = q.getUsedCapacity() * 100;
@@ -153,7 +157,7 @@ public class CapacitySchedulerQueueInfo {
CapacitySchedulerConfiguration conf = cs.getConfiguration();
queueAcls = new QueueAclsInfo();
- queueAcls.addAll(getSortedQueueAclInfoList(q, queuePath, conf));
+ queueAcls.addAll(getSortedQueueAclInfoList(q, queuePathObject, conf));
queuePriority = q.getPriority().getPriority();
if (q instanceof AbstractParentQueue) {
@@ -175,12 +179,12 @@ public class CapacitySchedulerQueueInfo {
AbstractCSQueue.CapacityConfigType.ABSOLUTE_RESOURCE;
autoCreateChildQueueEnabled = conf.
- isAutoCreateChildQueueEnabled(queuePath);
- leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePath);
+ isAutoCreateChildQueueEnabled(queuePathObject);
+ leafQueueTemplate = new LeafQueueTemplateInfo(conf, queuePathObject);
}
public static ArrayList<QueueAclInfo> getSortedQueueAclInfoList(
- CSQueue queue, String queuePath, CapacitySchedulerConfiguration conf) {
+ CSQueue queue, QueuePath queuePath, CapacitySchedulerConfiguration conf) {
ArrayList<QueueAclInfo> queueAclsInfo = new ArrayList<>();
for (Map.Entry<AccessType, AccessControlList> e :
((AbstractCSQueue) queue).getACLs().entrySet()) {
@@ -191,7 +195,7 @@ public class CapacitySchedulerQueueInfo {
String aclApplicationMaxPriority = "acl_" +
StringUtils.toLowerCase(AccessType.APPLICATION_MAX_PRIORITY.toString());
- String priorityAcls = conf.get(CapacitySchedulerConfiguration
+ String priorityAcls = conf.get(QueuePrefixes
.getQueuePrefix(queuePath) + aclApplicationMaxPriority,
CapacitySchedulerConfiguration.ALL_ACL);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 446b7928bf93..65e5a700d6be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -27,6 +27,8 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -111,8 +113,9 @@ public class FairSchedulerQueueInfo {
reservedContainers = queue.getMetrics().getReservedContainers();
pendingContainers = queue.getMetrics().getPendingContainers();
- if (allocConf.isReservable(queueName) &&
- !allocConf.getShowReservationAsQueues(queueName)) {
+ QueuePath queuePath = new QueuePath(queueName);
+ if (allocConf.isReservable(queuePath) &&
+ !allocConf.getShowReservationAsQueues(queuePath)) {
return;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java
index f528a47449b0..c4a20494bc38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LeafQueueTemplateInfo.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -40,8 +41,8 @@ public class LeafQueueTemplateInfo {
public LeafQueueTemplateInfo() {
} // JAXB needs this
- public LeafQueueTemplateInfo(Configuration conf, String queuePath) {
- String configPrefix = CapacitySchedulerConfiguration.
+ public LeafQueueTemplateInfo(Configuration conf, QueuePath queuePath) {
+ String configPrefix = QueuePrefixes.
getQueuePrefix(queuePath) + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
+ DOT;
conf.forEach(entry -> {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
index a011c8a8b35a..26e3ee2920d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -47,6 +49,13 @@ public abstract class ACLsTestBase {
protected static final String QUEUEA = "queueA";
protected static final String QUEUEB = "queueB";
protected static final String QUEUEC = "queueC";
+ protected static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ protected static final QueuePath A_QUEUE_PATH = new QueuePath(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEA);
+ protected static final QueuePath B_QUEUE_PATH = new QueuePath(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEB);
+ protected static final QueuePath C_QUEUE_PATH = new QueuePath(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEC);
protected static final Logger LOG =
LoggerFactory.getLogger(TestApplicationACLs.class);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
index af4fe7d6c6e5..a0fe708c77e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
@@ -499,19 +499,15 @@ public class ReservationACLsTestBase extends ACLsTestBase {
private static Configuration createCapacitySchedulerConfiguration() {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {
+ csConf.setQueues(ROOT, new String[] {
QUEUEA, QUEUEB, QUEUEC });
- String absoluteQueueA = CapacitySchedulerConfiguration.ROOT + "." + QUEUEA;
- String absoluteQueueB = CapacitySchedulerConfiguration.ROOT + "." + QUEUEB;
- String absoluteQueueC = CapacitySchedulerConfiguration.ROOT + "." + QUEUEC;
-
- csConf.setCapacity(absoluteQueueA, 50f);
- csConf.setCapacity(absoluteQueueB, 20f);
- csConf.setCapacity(absoluteQueueC, 30f);
- csConf.setReservable(absoluteQueueA, true);
- csConf.setReservable(absoluteQueueB, true);
- csConf.setReservable(absoluteQueueC, true);
+ csConf.setCapacity(A_QUEUE_PATH, 50f);
+ csConf.setCapacity(B_QUEUE_PATH, 20f);
+ csConf.setCapacity(C_QUEUE_PATH, 30f);
+ csConf.setReservable(A_QUEUE_PATH, true);
+ csConf.setReservable(B_QUEUE_PATH, true);
+ csConf.setReservable(C_QUEUE_PATH, true);
// Set up ACLs on Queue A
Map<ReservationACL, AccessControlList> reservationAclsOnQueueA =
@@ -528,7 +524,7 @@ public class ReservationACLsTestBase extends ACLsTestBase {
reservationAclsOnQueueA.put(ReservationACL.LIST_RESERVATIONS,
listACLonQueueA);
- csConf.setReservationAcls(absoluteQueueA, reservationAclsOnQueueA);
+ csConf.setReservationAcls(A_QUEUE_PATH, reservationAclsOnQueueA);
// Set up ACLs on Queue B
Map<ReservationACL, AccessControlList> reservationAclsOnQueueB =
@@ -545,7 +541,7 @@ public class ReservationACLsTestBase extends ACLsTestBase {
reservationAclsOnQueueB.put(ReservationACL.LIST_RESERVATIONS,
listACLonQueueB);
- csConf.setReservationAcls(absoluteQueueB, reservationAclsOnQueueB);
+ csConf.setReservationAcls(B_QUEUE_PATH, reservationAclsOnQueueB);
csConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
csConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 0369feb130c4..50fc8fd67dea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -116,7 +116,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.A
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getAutoCreatedQueueTemplateConfPrefix;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePrefixes.getQueuePrefix;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
@@ -149,7 +150,12 @@ public class TestAppManager extends AppManagerTestBase{
private ResourceScheduler scheduler;
private static final String USER_ID_PREFIX = "userid=";
- private static final String ROOT_PARENT = PREFIX + "root.parent.";
+ private static final String ROOT_PARENT_PATH = PREFIX + "root.parent.";
+ private static final QueuePath ROOT = new QueuePath("root");
+ private static final QueuePath DEFAULT = new QueuePath("root.default");
+ private static final QueuePath TEST = new QueuePath("root.test");
+ private static final QueuePath PARENT = new QueuePath("root.parent");
+ private static final QueuePath MANAGED_PARENT = new QueuePath("root.managedparent");
public synchronized RMAppEventType getAppEventType() {
return appEventType;
@@ -321,11 +327,11 @@ public class TestAppManager extends AppManagerTestBase{
CapacitySchedulerConfiguration(conf, false);
csConf.set(PREFIX + "root.queues", "default,test");
- csConf.setCapacity("root.default", 50.0f);
- csConf.setMaximumCapacity("root.default", 100.0f);
+ csConf.setCapacity(DEFAULT, 50.0f);
+ csConf.setMaximumCapacity(DEFAULT, 100.0f);
- csConf.setCapacity("root.test", 50.0f);
- csConf.setMaximumCapacity("root.test", 100.0f);
+ csConf.setCapacity(TEST, 50.0f);
+ csConf.setMaximumCapacity(TEST, 100.0f);
csConf.set(PREFIX + "root.acl_submit_applications", " ");
csConf.set(PREFIX + "root.acl_administer_queue", " ");
@@ -363,11 +369,11 @@ public class TestAppManager extends AppManagerTestBase{
CapacitySchedulerConfiguration(conf, false);
csConf.set(PREFIX + "root.queues", "default,test");
- csConf.setCapacity("root.default", 50.0f);
- csConf.setMaximumCapacity("root.default", 100.0f);
+ csConf.setCapacity(DEFAULT, 50.0f);
+ csConf.setMaximumCapacity(DEFAULT, 100.0f);
- csConf.setCapacity("root.test", 50.0f);
- csConf.setMaximumCapacity("root.test", 100.0f);
+ csConf.setCapacity(TEST, 50.0f);
+ csConf.setMaximumCapacity(TEST, 100.0f);
MockRM newMockRM = new MockRM(csConf);
RMContext newMockRMContext = newMockRM.getRMContext();
@@ -390,11 +396,11 @@ public class TestAppManager extends AppManagerTestBase{
conf, false);
csConf.set(PREFIX + "root.queues", "default,managedparent");
- csConf.setCapacity("root.default", 50.0f);
- csConf.setMaximumCapacity("root.default", 100.0f);
+ csConf.setCapacity(DEFAULT, 50.0f);
+ csConf.setMaximumCapacity(DEFAULT, 100.0f);
- csConf.setCapacity("root.managedparent", 50.0f);
- csConf.setMaximumCapacity("root.managedparent", 100.0f);
+ csConf.setCapacity(MANAGED_PARENT, 50.0f);
+ csConf.setMaximumCapacity(MANAGED_PARENT, 100.0f);
csConf.set(PREFIX + "root.acl_submit_applications", " ");
csConf.set(PREFIX + "root.acl_administer_queue", " ");
@@ -405,9 +411,9 @@ public class TestAppManager extends AppManagerTestBase{
csConf.set(PREFIX + "root.managedparent.acl_administer_queue", "admin");
csConf.set(PREFIX + "root.managedparent.acl_submit_applications", "user1");
- csConf.setAutoCreateChildQueueEnabled("root.managedparent", true);
- csConf.setAutoCreatedLeafQueueConfigCapacity("root.managedparent", 30f);
- csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.managedparent", 100f);
+ csConf.setAutoCreateChildQueueEnabled(MANAGED_PARENT, true);
+ csConf.setAutoCreatedLeafQueueConfigCapacity(MANAGED_PARENT, 30f);
+ csConf.setAutoCreatedLeafQueueConfigMaxCapacity(MANAGED_PARENT, 100f);
MockRM newMockRM = new MockRM(csConf);
CapacityScheduler cs =
@@ -441,21 +447,27 @@ public class TestAppManager extends AppManagerTestBase{
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
conf, false);
+
+ QueuePath parentQueuePath = new QueuePath("root.parent");
+ QueuePath user1QueuePath = new QueuePath("root.parent.user1");
+ QueuePath user2QueuePath = new QueuePath("root.parent.user2");
+
csConf.set(PREFIX + "root.queues", "parent");
csConf.set(PREFIX + "root.acl_submit_applications", " ");
csConf.set(PREFIX + "root.acl_administer_queue", " ");
- csConf.setCapacity("root.parent", 100.0f);
+ csConf.setCapacity(parentQueuePath, 100.0f);
csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1,user4");
csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1,user4");
- csConf.setAutoCreateChildQueueEnabled("root.parent", true);
- csConf.setAutoCreatedLeafQueueConfigCapacity("root.parent", 50f);
- csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.parent", 100f);
- csConf.set(getQueuePrefix(csConf.getAutoCreatedQueueTemplateConfPrefix("root.parent")) +
- "acl_administer_queue", "user2,user4");
- csConf.set(getQueuePrefix(csConf.getAutoCreatedQueueTemplateConfPrefix("root.parent")) +
- "acl_submit_applications", "user2,user4");
+ csConf.setAutoCreateChildQueueEnabled(parentQueuePath, true);
+ csConf.setAutoCreatedLeafQueueConfigCapacity(parentQueuePath, 50f);
+ csConf.setAutoCreatedLeafQueueConfigMaxCapacity(parentQueuePath, 100f);
+ String autoCreatedQueuePrefix =
+ getAutoCreatedQueueTemplateConfPrefix(parentQueuePath);
+ QueuePath autoCreatedQueuePath = new QueuePath(autoCreatedQueuePrefix);
+ csConf.set(getQueuePrefix(autoCreatedQueuePath) + "acl_administer_queue", "user2,user4");
+ csConf.set(getQueuePrefix(autoCreatedQueuePath) + "acl_submit_applications", "user2,user4");
MockRM newMockRM = new MockRM(csConf);
@@ -464,22 +476,22 @@ public class TestAppManager extends AppManagerTestBase{
// user1 has permission on root.parent so a queue would be created
newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
- "user1", "user1", "root.parent"));
+ "user1", "user1", parentQueuePath.getFullPath()));
verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(1)),
newAppMonitor,
newMockRMContext,
"user1",
- "root.parent.user1");
+ user1QueuePath.getFullPath());
newMockRMContext.setQueuePlacementManager(createMockPlacementManager(
- "user1|user2|user3|user4", "user2", "root.parent"));
+ "user1|user2|user3|user4", "user2", parentQueuePath.getFullPath()));
// user2 has permission (due to ACL templates)
verifyAppSubmission(createAppSubmissionContext(MockApps.newAppID(2)),
newAppMonitor,
newMockRMContext,
"user2",
- "root.parent.user2");
+ user2QueuePath.getFullPath());
// user3 doesn't have permission
verifyAppSubmissionFailure(newAppMonitor,
@@ -491,12 +503,12 @@ public class TestAppManager extends AppManagerTestBase{
newAppMonitor,
newMockRMContext,
"user4",
- "root.parent.user2");
+ user2QueuePath.getFullPath());
// create the root.parent.user2 manually
CapacityScheduler cs =
((CapacityScheduler) newMockRM.getResourceScheduler());
- cs.getCapacitySchedulerQueueManager().createQueue(new QueuePath("root.parent.user2"));
+ cs.getCapacitySchedulerQueueManager().createQueue(user2QueuePath);
AutoCreatedLeafQueue autoCreatedLeafQueue = (AutoCreatedLeafQueue) cs.getQueue("user2");
Assert.assertNotNull("Auto Creation of Queue failed", autoCreatedLeafQueue);
ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue("parent");
@@ -509,7 +521,7 @@ public class TestAppManager extends AppManagerTestBase{
newAppMonitor,
newMockRMContext,
"user2",
- "root.parent.user2");
+ user2QueuePath.getFullPath());
// user3 doesn't have permission for root.parent.user2 queue
verifyAppSubmissionFailure(newAppMonitor,
@@ -521,7 +533,7 @@ public class TestAppManager extends AppManagerTestBase{
newAppMonitor,
newMockRMContext,
"user1",
- "root.parent.user2");
+ user2QueuePath.getFullPath());
}
@Test
@@ -530,19 +542,19 @@ public class TestAppManager extends AppManagerTestBase{
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
- csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_administer_queue",
"user2");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "acl_submit_applications",
"user2");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
- "user3");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
"user3");
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX
+ + "acl_submit_applications", "user3");
MockRM newMockRM = new MockRM(csConf);
@@ -621,19 +633,19 @@ public class TestAppManager extends AppManagerTestBase{
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
"user2");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
"user2");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
- "user3");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
"user3");
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX
+ + "acl_submit_applications", "user3");
testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf);
}
@@ -644,18 +656,18 @@ public class TestAppManager extends AppManagerTestBase{
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
"user2");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
"user2");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
"user3");
- csConf.set(ROOT_PARENT + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + "*." + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
"user3");
testFlexibleAQCDWithMixedTemplatesDynamicParentACLScenario(conf, csConf);
@@ -744,11 +756,11 @@ public class TestAppManager extends AppManagerTestBase{
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_administer_queue",
"user2");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_TEMPLATE_PREFIX + "acl_submit_applications",
"user2");
testFlexibleAQCLeafOnly(conf, csConf);
@@ -760,11 +772,11 @@ public class TestAppManager extends AppManagerTestBase{
YarnConfiguration conf = createYarnACLEnabledConfiguration();
CapacitySchedulerConfiguration csConf = createFlexibleAQCBaseACLConfiguration(conf);
- csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "capacity",
"1w");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_administer_queue",
"user2");
- csConf.set(ROOT_PARENT + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
+ csConf.set(ROOT_PARENT_PATH + AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + "acl_submit_applications",
"user2");
testFlexibleAQCLeafOnly(conf, csConf);
@@ -844,7 +856,7 @@ public class TestAppManager extends AppManagerTestBase{
csConf.set(PREFIX + "root.acl_submit_applications", "user1");
csConf.set(PREFIX + "root.acl_administer_queue", "admin1");
- csConf.setAutoQueueCreationV2Enabled("root", true);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
"1w");
@@ -943,7 +955,7 @@ public class TestAppManager extends AppManagerTestBase{
csConf.set(PREFIX + "root.acl_submit_applications", "user1");
csConf.set(PREFIX + "root.acl_administer_queue", "admin1");
- csConf.setAutoQueueCreationV2Enabled("root", true);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
csConf.set(PREFIX + "root." + AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + "capacity",
"1w");
@@ -1021,11 +1033,11 @@ public class TestAppManager extends AppManagerTestBase{
csConf.set(PREFIX + "root.acl_submit_applications", " ");
csConf.set(PREFIX + "root.acl_administer_queue", " ");
- csConf.setCapacity("root.parent", "1w");
+ csConf.setCapacity(PARENT, "1w");
csConf.set(PREFIX + "root.parent.acl_administer_queue", "user1");
csConf.set(PREFIX + "root.parent.acl_submit_applications", "user1");
- csConf.setAutoQueueCreationV2Enabled("root.parent", true);
+ csConf.setAutoQueueCreationV2Enabled(PARENT, true);
return csConf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java
index 80f84c34503f..ab6851591165 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterServiceCapacity.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -260,23 +261,26 @@ public class TestApplicationMasterServiceCapacity extends
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
-
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
- conf.setCapacity(A, 50);
- conf.setMaximumCapacity(A, 100);
- conf.setAccessibleNodeLabels(A, toSet("x"));
- conf.setDefaultNodeLabelExpression(A, "x");
- conf.setCapacityByLabel(A, "x", 100);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
- conf.setCapacity(B, 50);
- conf.setMaximumCapacity(B, 100);
- conf.setAccessibleNodeLabels(B, toSet("y"));
- conf.setDefaultNodeLabelExpression(B, "y");
- conf.setCapacityByLabel(B, "y", 100);
+ final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ conf.setQueues(root, new String[] {"a", "b"});
+ conf.setCapacityByLabel(root, "x", 100);
+ conf.setCapacityByLabel(root, "y", 100);
+
+ final String aPath = CapacitySchedulerConfiguration.ROOT + ".a";
+ final QueuePath a = new QueuePath(aPath);
+ conf.setCapacity(a, 50);
+ conf.setMaximumCapacity(a, 100);
+ conf.setAccessibleNodeLabels(a, toSet("x"));
+ conf.setDefaultNodeLabelExpression(a, "x");
+ conf.setCapacityByLabel(a, "x", 100);
+
+ final String bPath = CapacitySchedulerConfiguration.ROOT + ".b";
+ final QueuePath b = new QueuePath(bPath);
+ conf.setCapacity(b, 50);
+ conf.setMaximumCapacity(b, 100);
+ conf.setAccessibleNodeLabels(b, toSet("y"));
+ conf.setDefaultNodeLabelExpression(b, "y");
+ conf.setCapacityByLabel(b, "y", 100);
return conf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index ce4a7a84fdcd..76cf0db9a02b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.TestCapacitySchedulerAutoCreatedQueueBase;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt;
@@ -572,46 +573,48 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
private static final String QUEUE_DOESNT_EXIST = "NoSuchQueue";
private static final String USER_1 = "user1";
private static final String USER_2 = "user2";
+ private static final String Q_R_PATH = CapacitySchedulerConfiguration.ROOT + "." + R;
+ private static final String Q_A_PATH = Q_R_PATH + "." + A;
+ private static final String Q_B_PATH = Q_R_PATH + "." + B;
+ private static final String Q_B1_PATH = Q_B_PATH + "." + B1;
+ private static final String Q_B2_PATH = Q_B_PATH + "." + B2;
+
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath R_QUEUE_PATH = new QueuePath(Q_R_PATH);
+ private static final QueuePath A_QUEUE_PATH = new QueuePath(Q_A_PATH);
+ private static final QueuePath B_QUEUE_PATH = new QueuePath(Q_B_PATH);
+ private static final QueuePath B1_QUEUE_PATH = new QueuePath(Q_B1_PATH);
+ private static final QueuePath B2_QUEUE_PATH = new QueuePath(Q_B2_PATH);
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R });
- final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R;
- conf.setCapacity(Q_R, 100);
- final String Q_A = Q_R + "." + A;
- final String Q_B = Q_R + "." + B;
- conf.setQueues(Q_R, new String[] {A, B});
- conf.setCapacity(Q_A, 50);
- conf.setCapacity(Q_B, 50);
+ conf.setQueues(ROOT, new String[] {R});
+ conf.setCapacity(R_QUEUE_PATH, 100);
+ conf.setQueues(R_QUEUE_PATH, new String[] {A, B});
+ conf.setCapacity(A_QUEUE_PATH, 50);
+ conf.setCapacity(B_QUEUE_PATH, 50);
conf.setDouble(CapacitySchedulerConfiguration
.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.5f);
}
private void setupQueueConfigurationOnlyA(
CapacitySchedulerConfiguration conf) {
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R });
- final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R;
- conf.setCapacity(Q_R, 100);
- final String Q_A = Q_R + "." + A;
- conf.setQueues(Q_R, new String[] {A});
- conf.setCapacity(Q_A, 100);
+ conf.setQueues(ROOT, new String[] {R});
+ conf.setCapacity(R_QUEUE_PATH, 100);
+ conf.setQueues(R_QUEUE_PATH, new String[] {A});
+ conf.setCapacity(A_QUEUE_PATH, 100);
conf.setDouble(CapacitySchedulerConfiguration
.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 1.0f);
}
private void setupQueueConfigurationChildOfB(CapacitySchedulerConfiguration conf) {
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { R });
- final String Q_R = CapacitySchedulerConfiguration.ROOT + "." + R;
- conf.setCapacity(Q_R, 100);
- final String Q_A = Q_R + "." + A;
- final String Q_B = Q_R + "." + B;
- final String Q_B1 = Q_B + "." + B1;
- final String Q_B2 = Q_B + "." + B2;
- conf.setQueues(Q_R, new String[] {A, B});
- conf.setCapacity(Q_A, 50);
- conf.setCapacity(Q_B, 50);
- conf.setQueues(Q_B, new String[] {B1, B2});
- conf.setCapacity(Q_B1, 50);
- conf.setCapacity(Q_B2, 50);
+ conf.setQueues(ROOT, new String[] {R});
+ conf.setCapacity(R_QUEUE_PATH, 100);
+ conf.setQueues(R_QUEUE_PATH, new String[] {A, B});
+ conf.setCapacity(A_QUEUE_PATH, 50);
+ conf.setCapacity(B_QUEUE_PATH, 50);
+ conf.setQueues(B_QUEUE_PATH, new String[] {B1, B2});
+ conf.setCapacity(B1_QUEUE_PATH, 50);
+ conf.setCapacity(B2_QUEUE_PATH, 50);
conf.setDouble(CapacitySchedulerConfiguration
.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.5f);
}
@@ -641,9 +644,9 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{QUEUE_DOESNT_EXIST});
+ csConf.setQueues(ROOT, new String[]{QUEUE_DOESNT_EXIST});
final String noQueue = CapacitySchedulerConfiguration.ROOT + "." + QUEUE_DOESNT_EXIST;
- csConf.setCapacity(noQueue, 100);
+ csConf.setCapacity(new QueuePath(noQueue), 100);
rm2 = new MockRM(csConf, memStore);
rm2.start();
@@ -1669,24 +1672,19 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
return;
}
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
-
- csConf.setQueues(
- CapacitySchedulerConfiguration.ROOT, new String[] {"default", "joe", "john"});
- csConf.setCapacity(
- CapacitySchedulerConfiguration.ROOT + "." + "joe", 25);
- csConf.setCapacity(
- CapacitySchedulerConfiguration.ROOT + "." + "john", 25);
- csConf.setCapacity(
- CapacitySchedulerConfiguration.ROOT + "." + "default", 50);
-
- final String q1 = CapacitySchedulerConfiguration.ROOT + "." + "joe";
- final String q2 = CapacitySchedulerConfiguration.ROOT + "." + "john";
- csConf.setQueues(q1, new String[] {"test"});
- csConf.setQueues(q2, new String[] {"test"});
- csConf.setCapacity(
- CapacitySchedulerConfiguration.ROOT + "." + "joe.test", 100);
- csConf.setCapacity(
- CapacitySchedulerConfiguration.ROOT + "." + "john.test", 100);
+ final QueuePath defaultPath = new QueuePath(ROOT + "." + "default");
+ final QueuePath joe = new QueuePath(ROOT + "." + "joe");
+ final QueuePath john = new QueuePath(ROOT + "." + "john");
+
+ csConf.setQueues(ROOT, new String[] {"default", "joe", "john"});
+ csConf.setCapacity(joe, 25);
+ csConf.setCapacity(john, 25);
+ csConf.setCapacity(defaultPath, 50);
+
+ csConf.setQueues(joe, new String[] {"test"});
+ csConf.setQueues(john, new String[] {"test"});
+ csConf.setCapacity(new QueuePath(joe.getFullPath(), "test"), 100);
+ csConf.setCapacity(new QueuePath(john.getFullPath(), "test"), 100);
csConf.set(CapacitySchedulerConfiguration.MAPPING_RULE_JSON,
"{\"rules\" : [{\"type\": \"user\", \"policy\" : \"specified\", " +
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index fe89a698cf2e..1bac22743b0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
@@ -127,6 +128,14 @@ public class TestProportionalCapacityPreemptionPolicy {
ApplicationId.newInstance(TS, 4), 0);
final ArgumentCaptor<ContainerPreemptEvent> evtCaptor =
ArgumentCaptor.forClass(ContainerPreemptEvent.class);
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath QUEUE_A = new QueuePath("root.queueA");
+ private static final QueuePath QUEUE_A_QUEUE_B = new QueuePath("root.queueA.queueB");
+ private static final QueuePath QUEUE_B = new QueuePath("root.queueB");
+ private static final QueuePath QUEUE_D = new QueuePath("root.queueD");
+ private static final QueuePath QUEUE_D_QUEUE_E = new QueuePath("root.queueD.queueE");
+ private static final QueuePath QUEUE_A_QUEUE_C = new QueuePath("root.queueA.queueC");
+
public enum priority {
AMCONTAINER(0), CONTAINER(1), LABELEDCONTAINER(2);
@@ -346,7 +355,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ 3, 0, 0, 0 }, // subqueues
};
- conf.setPreemptionDisabled("root.queueB", true);
+ conf.setPreemptionDisabled(QUEUE_B, true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
@@ -359,7 +368,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// event handler will count only events from the following test and not the
// previous one.
setup();
- conf.setPreemptionDisabled("root.queueB", false);
+ conf.setPreemptionDisabled(QUEUE_B, false);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
@@ -398,7 +407,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// Need to call setup() again to reset mDisp
setup();
// Turn off preemption for queueB and it's children
- conf.setPreemptionDisabled("root.queueA.queueB", true);
+ conf.setPreemptionDisabled(QUEUE_A_QUEUE_B, true);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
ApplicationAttemptId expectedAttemptOnQueueC =
@@ -445,7 +454,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// Need to call setup() again to reset mDisp
setup();
// Turn off preemption for queueB(appA)
- conf.setPreemptionDisabled("root.queueA.queueB", true);
+ conf.setPreemptionDisabled(QUEUE_A_QUEUE_B, true);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
// Now that queueB(appA) is not preemptable, verify that resources come
@@ -455,8 +464,8 @@ public class TestProportionalCapacityPreemptionPolicy {
setup();
// Turn off preemption for two of the 3 queues with over-capacity.
- conf.setPreemptionDisabled("root.queueD.queueE", true);
- conf.setPreemptionDisabled("root.queueA.queueB", true);
+ conf.setPreemptionDisabled(QUEUE_D_QUEUE_E, true);
+ conf.setPreemptionDisabled(QUEUE_A_QUEUE_B, true);
ProportionalCapacityPreemptionPolicy policy3 = buildPolicy(qData);
policy3.editSchedule();
@@ -497,7 +506,7 @@ public class TestProportionalCapacityPreemptionPolicy {
// Turn off preemption for queueA and it's children. queueF(appC)'s request
// should starve.
setup(); // Call setup() to reset mDisp
- conf.setPreemptionDisabled("root.queueA", true);
+ conf.setPreemptionDisabled(QUEUE_A, true);
ProportionalCapacityPreemptionPolicy policy2 = buildPolicy(qData);
policy2.editSchedule();
verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA))); // queueC
@@ -521,7 +530,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ -1, -1, 1, 1, 1, -1, 1, 1, 1 }, // req granularity
{ 2, 3, 0, 0, 0, 3, 0, 0, 0 }, // subqueues
};
- conf.setPreemptionDisabled("root.queueA.queueC", true);
+ conf.setPreemptionDisabled(QUEUE_A_QUEUE_C, true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// Although queueC(appB) is way over capacity and is untouchable,
@@ -545,7 +554,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ 3, 2, 0, 0, 2, 0, 0, 2, 0, 0 }, // subqueues
};
- conf.setPreemptionDisabled("root", true);
+ conf.setPreemptionDisabled(ROOT, true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// All queues should be non-preemptable, so request should starve.
@@ -572,7 +581,7 @@ public class TestProportionalCapacityPreemptionPolicy {
{ 2, 2, 0, 0, 2, 0, 0 }, // subqueues
};
// QueueE inherits non-preemption from QueueD
- conf.setPreemptionDisabled("root.queueD", true);
+ conf.setPreemptionDisabled(QUEUE_D, true);
ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
policy.editSchedule();
// appC is running on QueueE. QueueE is over absMaxCap, but is not
@@ -1325,7 +1334,7 @@ public class TestProportionalCapacityPreemptionPolicy {
String qName = "";
while(tokenizer.hasMoreTokens()) {
qName += tokenizer.nextToken();
- preemptionDisabled = conf.getPreemptionDisabled(qName, preemptionDisabled);
+ preemptionDisabled = conf.getPreemptionDisabled(new QueuePath(qName), preemptionDisabled);
qName += ".";
}
return preemptionDisabled;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index 96e91d5945b7..2ddb0d8ae1c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.mockframework.ProportionalCapacityPreemptionPolicyMockFramework;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Before;
import org.junit.Test;
@@ -366,7 +367,7 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions
"c\t" // app3 in c
+ "(1,1,n1,x,20,false)"; // 20x in n1
- conf.setPreemptionDisabled("root.b", true);
+ conf.setPreemptionDisabled(new QueuePath("root.b"), true);
buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
policy.editSchedule();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
index 2fb69f555728..a11153a0482f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueue.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import java.io.IOException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.junit.Before;
import org.junit.Test;
@@ -120,10 +121,15 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueue
* purpose is to test that disabling preemption on a specific queue will
* avoid intra-queue preemption.
*/
- conf.setPreemptionDisabled("root.a", true);
- conf.setPreemptionDisabled("root.b", true);
- conf.setPreemptionDisabled("root.c", true);
- conf.setPreemptionDisabled("root.d", true);
+ QueuePath a = new QueuePath("root.a");
+ QueuePath b = new QueuePath("root.b");
+ QueuePath c = new QueuePath("root.c");
+ QueuePath d = new QueuePath("root.d");
+
+ conf.setPreemptionDisabled(a, true);
+ conf.setPreemptionDisabled(b, true);
+ conf.setPreemptionDisabled(c, true);
+ conf.setPreemptionDisabled(d, true);
String labelsConfig = "=100,true;";
String nodesConfig = // n1 has no label
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java
index ae4ff5a663ed..44075b322d3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/mockframework/MockQueueHierarchy.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
@@ -273,7 +274,7 @@ class MockQueueHierarchy {
// Setup preemption disabled
when(queue.getPreemptionDisabled()).thenReturn(
- conf.getPreemptionDisabled(queuePath, false));
+ conf.getPreemptionDisabled(new QueuePath(queuePath), false));
// Setup other queue configurations
Map<String, String> otherConfigs = getOtherConfigurations(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index 0400f7dfcbf0..affd984e5334 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
@@ -667,13 +668,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
CapacitySchedulerConfiguration conf =
new CapacitySchedulerConfiguration(config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a" });
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
-
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
- conf.setCapacity(A, 100);
- conf.setAccessibleNodeLabels(A, ImmutableSet.of("x"));
- conf.setCapacityByLabel(A, "x", 100);
+ final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ conf.setQueues(root, new String[] {"a"});
+ conf.setCapacityByLabel(root, "x", 100);
+
+ final QueuePath a = root.createNewLeaf("a");
+ conf.setCapacity(a, 100);
+ conf.setAccessibleNodeLabels(a, ImmutableSet.of("x"));
+ conf.setCapacityByLabel(a, "x", 100);
return conf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
index bf5fb8ca1a4d..7e172aee3530 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
@@ -75,6 +76,18 @@ public class ReservationSystemTestUtil {
public final static String RESERVATION_Q_SHORT = "dedicated";
public final static String reservationQ = "root." + RESERVATION_Q_SHORT;
+ public final static String DEDICATED_PATH = CapacitySchedulerConfiguration.ROOT
+ + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT;
+ private final static String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default";
+ private final static String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ private final static String A1_PATH = A_PATH + ".a1";
+ private final static String A2_PATH = A_PATH + ".a2";
+ private final static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private final static QueuePath DEDICATED = new QueuePath(DEDICATED_PATH);
+ private final static QueuePath DEFAULT = new QueuePath(DEFAULT_PATH);
+ private final static QueuePath A = new QueuePath(A_PATH);
+ private final static QueuePath A1 = new QueuePath(A1_PATH);
+ private final static QueuePath A2 = new QueuePath(A2_PATH);
public static ReservationId getNewReservationId() {
return ReservationId.newInstance(rand.nextLong(), rand.nextLong());
@@ -87,10 +100,11 @@ public class ReservationSystemTestUtil {
ReservationSchedulerConfiguration realConf =
new CapacitySchedulerConfiguration();
ReservationSchedulerConfiguration conf = spy(realConf);
- when(conf.getReservationWindow(reservationQ)).thenReturn(timeWindow);
- when(conf.getInstantaneousMaxCapacity(reservationQ))
+ QueuePath reservationQueuePath = new QueuePath(reservationQ);
+ when(conf.getReservationWindow(reservationQueuePath)).thenReturn(timeWindow);
+ when(conf.getInstantaneousMaxCapacity(reservationQueuePath))
.thenReturn(instConstraint);
- when(conf.getAverageCapacity(reservationQ)).thenReturn(avgConstraint);
+ when(conf.getAverageCapacity(reservationQueuePath)).thenReturn(avgConstraint);
return conf;
}
@@ -292,25 +306,19 @@ public class ReservationSystemTestUtil {
public static void setupQueueConfiguration(
CapacitySchedulerConfiguration conf) {
// Define default queue
- final String defQ = CapacitySchedulerConfiguration.ROOT + ".default";
+ final String defQPath = CapacitySchedulerConfiguration.ROOT + ".default";
+ final QueuePath defQ = new QueuePath(defQPath);
conf.setCapacity(defQ, 10);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] {"default", "a", RESERVATION_Q_SHORT});
-
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
-
- final String dedicated = CapacitySchedulerConfiguration.ROOT
- + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT;
- conf.setCapacity(dedicated, 80);
+ conf.setCapacity(DEDICATED, 80);
// Set as reservation queue
- conf.setReservable(dedicated, true);
+ conf.setReservable(DEDICATED, true);
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
conf.setQueues(A, new String[] {"a1", "a2"});
conf.setCapacity(A1, 30);
conf.setCapacity(A2, 70);
@@ -319,13 +327,11 @@ public class ReservationSystemTestUtil {
public static void setupDynamicQueueConfiguration(
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] {RESERVATION_Q_SHORT});
- final String dedicated = CapacitySchedulerConfiguration.ROOT
- + CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT;
- conf.setCapacity(dedicated, 100);
+ conf.setCapacity(DEDICATED, 100);
// Set as reservation queue
- conf.setReservable(dedicated, true);
+ conf.setReservable(DEDICATED, true);
}
public static String getFullReservationQueueName() {
@@ -333,6 +339,10 @@ public class ReservationSystemTestUtil {
+ CapacitySchedulerConfiguration.DOT + RESERVATION_Q_SHORT;
}
+ public static QueuePath getFullReservationQueuePath() {
+ return new QueuePath(getFullReservationQueueName());
+ }
+
public static String getReservationQueueName() {
return reservationQ;
}
@@ -341,29 +351,23 @@ public class ReservationSystemTestUtil {
CapacitySchedulerConfiguration conf, String newQ) {
// Define default queue
final String prefix = CapacitySchedulerConfiguration.ROOT
- + CapacitySchedulerConfiguration.DOT;
- final String defQ = prefix + "default";
- conf.setCapacity(defQ, 5);
+ + CapacitySchedulerConfiguration.DOT;
+ conf.setCapacity(DEFAULT, 5);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] {"default", "a", RESERVATION_Q_SHORT, newQ});
-
- final String A = prefix + "a";
conf.setCapacity(A, 5);
-
- final String dedicated = prefix + RESERVATION_Q_SHORT;
- conf.setCapacity(dedicated, 10);
+ conf.setCapacity(DEDICATED, 10);
// Set as reservation queue
- conf.setReservable(dedicated, true);
+ conf.setReservable(DEDICATED, true);
- conf.setCapacity(prefix + newQ, 80);
+ final QueuePath newQueue = new QueuePath(prefix + newQ);
+ conf.setCapacity(newQueue, 80);
// Set as reservation queue
- conf.setReservable(prefix + newQ, true);
+ conf.setReservable(newQueue, true);
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
conf.setQueues(A, new String[] { "a1", "a2" });
conf.setCapacity(A1, 30);
conf.setCapacity(A2, 70);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
index b4ad8be8587f..20aa5511f822 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -128,10 +129,12 @@ public class TestCapacitySchedulerPlanFollower extends
String reservationQ =
ReservationSystemTestUtil.getFullReservationQueueName();
+ QueuePath reservationQueuePath =
+ ReservationSystemTestUtil.getFullReservationQueuePath();
CapacitySchedulerConfiguration csConf = cs.getConfiguration();
- csConf.setReservationWindow(reservationQ, 20L);
- csConf.setMaximumCapacity(reservationQ, 40);
- csConf.setAverageCapacity(reservationQ, 20);
+ csConf.setReservationWindow(reservationQueuePath, 20L);
+ csConf.setMaximumCapacity(reservationQueuePath, 40);
+ csConf.setAverageCapacity(reservationQueuePath, 20);
policy.init(reservationQ, csConf);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
index b8521969b343..1209c7e757ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestSimpleCapacityReplanner.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst
import org.apache.hadoop.yarn.server.resourcemanager.reservation.SharingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -74,7 +75,7 @@ public class TestSimpleCapacityReplanner {
RMContext context = ReservationSystemTestUtil.createMockRMContext();
ReservationSchedulerConfiguration conf =
mock(ReservationSchedulerConfiguration.class);
- when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L);
+ when(conf.getEnforcementWindow(any(QueuePath.class))).thenReturn(6L);
enf.init("blah", conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
index 2e003151e79a..569919c01c4f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestApplicationLifetimeMonitor.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.util.Times;
import org.slf4j.event.Level;
@@ -75,6 +76,10 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class TestApplicationLifetimeMonitor {
private final long maxLifetime = 30L;
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final String CQ1 = "child1";
+ private static final QueuePath CQ1_QUEUE_PATH = ROOT.createNewLeaf(CQ1);
+ private static final QueuePath DEFAULT_QUEUE_PATH = ROOT.createNewLeaf("default");
private YarnConfiguration conf;
@@ -390,7 +395,6 @@ public class TestApplicationLifetimeMonitor {
}
}
- static final String CQ1 = "child1";
@Test(timeout = 120000)
public void testInheritAppLifetimeFromParentQueue() throws Exception {
YarnConfiguration yarnConf = conf;
@@ -399,13 +403,10 @@ public class TestApplicationLifetimeMonitor {
if (scheduler.equals(CapacityScheduler.class)) {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {CQ1});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + CQ1, 100);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT, maxRootLifetime);
- csConf.setDefaultLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT, defaultRootLifetime);
+ csConf.setQueues(ROOT, new String[] {CQ1});
+ csConf.setCapacity(CQ1_QUEUE_PATH, 100);
+ csConf.setMaximumLifetimePerQueue(ROOT, maxRootLifetime);
+ csConf.setDefaultLifetimePerQueue(ROOT, defaultRootLifetime);
yarnConf = new YarnConfiguration(csConf);
}
@@ -465,17 +466,12 @@ public class TestApplicationLifetimeMonitor {
if (scheduler.equals(CapacityScheduler.class)) {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {CQ1});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + CQ1, 100);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT, maxRootLifetime);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + "." + CQ1, maxChildLifetime);
- csConf.setDefaultLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT, defaultRootLifetime);
- csConf.setDefaultLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + "." + CQ1, maxChildLifetime);
+ csConf.setQueues(ROOT, new String[] {CQ1});
+ csConf.setCapacity(CQ1_QUEUE_PATH, 100);
+ csConf.setMaximumLifetimePerQueue(ROOT, maxRootLifetime);
+ csConf.setMaximumLifetimePerQueue(CQ1_QUEUE_PATH, maxChildLifetime);
+ csConf.setDefaultLifetimePerQueue(ROOT, defaultRootLifetime);
+ csConf.setDefaultLifetimePerQueue(CQ1_QUEUE_PATH, maxChildLifetime);
yarnConf = new YarnConfiguration(csConf);
}
@@ -529,16 +525,11 @@ public class TestApplicationLifetimeMonitor {
if (scheduler.equals(CapacityScheduler.class)) {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {CQ1});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + CQ1, 100);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT, maxRootLifetime);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + "." + CQ1, maxChildLifetime);
- csConf.setDefaultLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + "." + CQ1,
- defaultChildLifetime);
+ csConf.setQueues(ROOT, new String[] {CQ1});
+ csConf.setCapacity(CQ1_QUEUE_PATH, 100);
+ csConf.setMaximumLifetimePerQueue(ROOT, maxRootLifetime);
+ csConf.setMaximumLifetimePerQueue(CQ1_QUEUE_PATH, maxChildLifetime);
+ csConf.setDefaultLifetimePerQueue(CQ1_QUEUE_PATH, defaultChildLifetime);
yarnConf = new YarnConfiguration(csConf);
}
@@ -592,13 +583,11 @@ public class TestApplicationLifetimeMonitor {
long defaultLifetime) {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {"default"});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".default", 100);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + ".default", maxLifetime);
- csConf.setDefaultLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + ".default", defaultLifetime);
+ csConf.setCapacity(DEFAULT_QUEUE_PATH, 100);
+ csConf.setMaximumLifetimePerQueue(DEFAULT_QUEUE_PATH, maxLifetime);
+ csConf.setDefaultLifetimePerQueue(DEFAULT_QUEUE_PATH, defaultLifetime);
return csConf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java
index 087b797f2d84..db033a4be1de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigGeneratorForTest.java
@@ -65,8 +65,8 @@ public final class CapacitySchedulerConfigGeneratorForTest {
}
public static void setMaxAllocMb(CapacitySchedulerConfiguration conf,
- String queueName, int maxAllocMb) {
- String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+ QueuePath queuePath, int maxAllocMb) {
+ String propName = QueuePrefixes.getQueuePrefix(queuePath)
+ MAXIMUM_ALLOCATION_MB;
conf.setInt(propName, maxAllocMb);
}
@@ -82,22 +82,22 @@ public final class CapacitySchedulerConfigGeneratorForTest {
}
public static void setMaxAllocVcores(CapacitySchedulerConfiguration conf,
- String queueName, int maxAllocVcores) {
- String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+ QueuePath queuePath, int maxAllocVcores) {
+ String propName = QueuePrefixes.getQueuePrefix(queuePath)
+ CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_VCORES;
conf.setInt(propName, maxAllocVcores);
}
public static void setMaxAllocation(CapacitySchedulerConfiguration conf,
- String queueName, String maxAllocation) {
- String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+ QueuePath queuePath, String maxAllocation) {
+ String propName = QueuePrefixes.getQueuePrefix(queuePath)
+ MAXIMUM_ALLOCATION;
conf.set(propName, maxAllocation);
}
public static void unsetMaxAllocation(CapacitySchedulerConfiguration conf,
- String queueName) {
- String propName = CapacitySchedulerConfiguration.getQueuePrefix(queueName)
+ QueuePath queuePath) {
+ String propName = QueuePrefixes.getQueuePrefix(queuePath)
+ MAXIMUM_ALLOCATION;
conf.unset(propName);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java
index e61412d71c9d..0f539f8b74e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueCalculationTestBase.java
@@ -33,14 +33,15 @@ import java.io.IOException;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.GB;
public class CapacitySchedulerQueueCalculationTestBase {
- protected static final String A = "root.a";
- protected static final String A1 = "root.a.a1";
- protected static final String A11 = "root.a.a1.a11";
- protected static final String A12 = "root.a.a1.a12";
- protected static final String A2 = "root.a.a2";
- protected static final String B = "root.b";
- protected static final String B1 = "root.b.b1";
- protected static final String C = "root.c";
+ protected static final QueuePath ROOT = new QueuePath("root");
+ protected static final QueuePath A = new QueuePath("root.a");
+ protected static final QueuePath A1 = new QueuePath("root.a.a1");
+ protected static final QueuePath A11 = new QueuePath("root.a.a1.a11");
+ protected static final QueuePath A12 = new QueuePath("root.a.a1.a12");
+ protected static final QueuePath A2 = new QueuePath("root.a.a2");
+ protected static final QueuePath B = new QueuePath("root.b");
+ protected static final QueuePath B1 = new QueuePath("root.b.b1");
+ protected static final QueuePath C = new QueuePath("root.c");
private static final String CAPACITY_VECTOR_TEMPLATE = "[memory=%s, vcores=%s]";
@@ -57,14 +58,14 @@ public class CapacitySchedulerQueueCalculationTestBase {
csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
- csConf.setQueues("root", new String[]{"a", "b"});
- csConf.setCapacity("root.a", 50f);
- csConf.setCapacity("root.b", 50f);
- csConf.setQueues("root.a", new String[]{"a1", "a2"});
- csConf.setCapacity("root.a.a1", 100f);
- csConf.setQueues("root.a.a1", new String[]{"a11", "a12"});
- csConf.setCapacity("root.a.a1.a11", 50f);
- csConf.setCapacity("root.a.a1.a12", 50f);
+ csConf.setQueues(ROOT, new String[]{"a", "b"});
+ csConf.setCapacity(A, 50f);
+ csConf.setCapacity(B, 50f);
+ csConf.setQueues(A, new String[]{"a1", "a2"});
+ csConf.setCapacity(A1, 100f);
+ csConf.setQueues(A1, new String[]{"a11", "a12"});
+ csConf.setCapacity(A11, 50f);
+ csConf.setCapacity(A12, 50f);
mgr = new NullRMNodeLabelsManager();
mgr.init(csConf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
index 1e11d3b41783..757c86de305d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueHelpers.java
@@ -27,22 +27,43 @@ import static org.junit.Assert.assertEquals;
public final class CapacitySchedulerQueueHelpers {
- public static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
- public static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
- public static final String A_CHILD = A + ".a";
- public static final String A1 = A + ".a1";
- public static final String A2 = A + ".a2";
- public static final String B1 = B + ".b1";
- public static final String B2 = B + ".b2";
- public static final String B3 = B + ".b3";
+ public static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default";
+ public static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ public static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ public static final String A_CHILD_PATH = A_PATH + ".a";
+ public static final String A1_PATH = A_PATH + ".a1";
+ public static final String A2_PATH = A_PATH + ".a2";
+ public static final String A3_PATH = A_PATH + ".a3";
+ public static final String B1_PATH = B_PATH + ".b1";
+ public static final String B2_PATH = B_PATH + ".b2";
+ public static final String B3_PATH = B_PATH + ".b3";
+ public static final String A1_B1_PATH = A1_PATH + ".b1";
+ public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ public static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH);
+ public static final QueuePath A = new QueuePath(A_PATH);
+ public static final QueuePath A_CHILD = new QueuePath(A_CHILD_PATH);
+ public static final QueuePath A1 = new QueuePath(A1_PATH);
+ public static final QueuePath A2 = new QueuePath(A2_PATH);
+ public static final QueuePath A3 = new QueuePath(A3_PATH);
+ public static final QueuePath B = new QueuePath(B_PATH);
+ public static final QueuePath B1 = new QueuePath(B1_PATH);
+ public static final QueuePath B2 = new QueuePath(B2_PATH);
+ public static final QueuePath B3 = new QueuePath(B3_PATH);
+ public static final QueuePath A1_B1 = new QueuePath(A1_B1_PATH);
public static final float A_CAPACITY = 10.5f;
public static final float B_CAPACITY = 89.5f;
- public static final String P1 = CapacitySchedulerConfiguration.ROOT + ".p1";
- public static final String P2 = CapacitySchedulerConfiguration.ROOT + ".p2";
- public static final String X1 = P1 + ".x1";
- public static final String X2 = P1 + ".x2";
- public static final String Y1 = P2 + ".y1";
- public static final String Y2 = P2 + ".y2";
+ public static final String P1_PATH = CapacitySchedulerConfiguration.ROOT + ".p1";
+ public static final String P2_PATH = CapacitySchedulerConfiguration.ROOT + ".p2";
+ public static final String X1_PATH = P1_PATH + ".x1";
+ public static final String X2_PATH = P1_PATH + ".x2";
+ public static final String Y1_PATH = P2_PATH + ".y1";
+ public static final String Y2_PATH = P2_PATH + ".y2";
+ public static final QueuePath P1 = new QueuePath(P1_PATH);
+ public static final QueuePath P2 = new QueuePath(P2_PATH);
+ public static final QueuePath X1 = new QueuePath(X1_PATH);
+ public static final QueuePath X2 = new QueuePath(X2_PATH);
+ public static final QueuePath Y1 = new QueuePath(Y1_PATH);
+ public static final QueuePath Y2 = new QueuePath(Y2_PATH);
public static final float A1_CAPACITY = 30;
public static final float A2_CAPACITY = 70;
public static final float B1_CAPACITY = 79.2f;
@@ -67,7 +88,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"});
+ conf.setQueues(ROOT, new String[]{"a", "b"});
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);
@@ -94,7 +115,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b"});
+ conf.setQueues(ROOT, new String[]{"a", "b"});
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);
@@ -105,8 +126,8 @@ public final class CapacitySchedulerQueueHelpers {
conf.setUserLimitFactor(A1, 100.0f);
conf.setCapacity(A2, 30.0f);
conf.setUserLimitFactor(A2, 100.0f);
- conf.setCapacity("root.a.a3", 40.0f);
- conf.setUserLimitFactor("root.a.a3", 100.0f);
+ conf.setCapacity(A3, 40.0f);
+ conf.setUserLimitFactor(A3, 100.0f);
conf.setQueues(B, new String[]{"b1", "b2", "b3"});
conf.setCapacity(B1, B1_CAPACITY);
@@ -132,7 +153,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"a", "b"});
conf.setCapacity(A, A_CAPACITY);
@@ -161,7 +182,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"a", "b"});
conf.setCapacity(A, A_CAPACITY);
@@ -190,7 +211,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"a", "b"});
conf.setCapacity(A, A_CAPACITY);
@@ -227,7 +248,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"a", "b"});
conf.setCapacity(A, A_CAPACITY);
@@ -250,7 +271,8 @@ public final class CapacitySchedulerQueueHelpers {
// Set childQueue for B1
conf.setQueues(B1, new String[]{"b11"});
- final String b11 = B1 + ".b11";
+ final String b11Path = B1 + ".b11";
+ final QueuePath b11 = new QueuePath(b11Path);
conf.setCapacity(b11, 100.0f);
conf.setUserLimitFactor(b11, 100.0f);
@@ -265,7 +287,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"});
+ conf.setQueues(ROOT, new String[]{"a"});
conf.setCapacity(A, A_CAPACITY + B_CAPACITY);
@@ -283,7 +305,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"a", "b"});
conf.setCapacity(A, 80f);
@@ -299,7 +321,7 @@ public final class CapacitySchedulerQueueHelpers {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"p1", "p2"});
conf.setCapacity(P1, 50f);
@@ -343,13 +365,13 @@ public final class CapacitySchedulerQueueHelpers {
public static Map<String, ExpectedCapacities> getDefaultCapacities(float capA, float capB) {
Map<String, ExpectedCapacities> capacities = new HashMap<>();
- capacities.put(A, new ExpectedCapacities(capA, 1.0f));
- capacities.put(B, new ExpectedCapacities(capB, 1.0f));
- capacities.put(A1, new ExpectedCapacities((A1_CAPACITY / 100.0f), capA));
- capacities.put(A2, new ExpectedCapacities((A2_CAPACITY / 100.0f), capA));
- capacities.put(B1, new ExpectedCapacities((B1_CAPACITY / 100.0f), capB));
- capacities.put(B2, new ExpectedCapacities((B2_CAPACITY / 100.0f), capB));
- capacities.put(B3, new ExpectedCapacities((B3_CAPACITY / 100.0f), capB));
+ capacities.put(A.getFullPath(), new ExpectedCapacities(capA, 1.0f));
+ capacities.put(B.getFullPath(), new ExpectedCapacities(capB, 1.0f));
+ capacities.put(A1.getFullPath(), new ExpectedCapacities((A1_CAPACITY / 100.0f), capA));
+ capacities.put(A2.getFullPath(), new ExpectedCapacities((A2_CAPACITY / 100.0f), capA));
+ capacities.put(B1.getFullPath(), new ExpectedCapacities((B1_CAPACITY / 100.0f), capB));
+ capacities.put(B2.getFullPath(), new ExpectedCapacities((B2_CAPACITY / 100.0f), capB));
+ capacities.put(B3.getFullPath(), new ExpectedCapacities((B3_CAPACITY / 100.0f), capB));
return capacities;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java
index 1c066719dd09..329d8b1bedbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueAssertionBuilder.java
@@ -64,15 +64,15 @@ class QueueAssertionBuilder {
}
public class QueueAssertion {
- private final String queuePath;
+ private final QueuePath queuePath;
private final List<QueueAssertion.ValueAssertion> assertions = new ArrayList<>();
- QueueAssertion(String queuePath) {
+ QueueAssertion(QueuePath queuePath) {
this.queuePath = queuePath;
}
- public QueueAssertion withQueue(String queuePath) {
+ public QueueAssertion withQueue(QueuePath queuePath) {
return QueueAssertionBuilder.this.withQueue(queuePath);
}
@@ -139,7 +139,7 @@ class QueueAssertionBuilder {
public void withResourceSupplier(
BiFunction<QueueResourceQuotas, String, Resource> assertion, String messageInfo) {
- CSQueue queue = cs.getQueue(queuePath);
+ CSQueue queue = cs.getQueue(queuePath.getFullPath());
if (queue == null) {
Assert.fail("Queue " + queuePath + " is not found");
}
@@ -150,7 +150,7 @@ class QueueAssertionBuilder {
public void withCapacitySupplier(
BiFunction<QueueCapacities, String, Float> assertion, String messageInfo) {
- CSQueue queue = cs.getQueue(queuePath);
+ CSQueue queue = cs.getQueue(queuePath.getFullPath());
if (queue == null) {
Assert.fail("Queue " + queuePath + " is not found");
}
@@ -161,7 +161,7 @@ class QueueAssertionBuilder {
}
- private final Map<String, QueueAssertion> assertions = new LinkedHashMap<>();
+ private final Map<QueuePath, QueueAssertion> assertions = new LinkedHashMap<>();
public QueueAssertionBuilder build() {
return this;
@@ -172,7 +172,7 @@ class QueueAssertionBuilder {
* @param queuePath path of the queue
* @return queue assertion group
*/
- public QueueAssertion withQueue(String queuePath) {
+ public QueueAssertion withQueue(QueuePath queuePath) {
assertions.putIfAbsent(queuePath, new QueueAssertion(queuePath));
return assertions.get(queuePath);
}
@@ -181,7 +181,7 @@ class QueueAssertionBuilder {
* Executes assertions created for all queues.
*/
public void finishAssertion() {
- for (Map.Entry<String, QueueAssertion> assertionEntry : assertions.entrySet()) {
+ for (Map.Entry<QueuePath, QueueAssertion> assertionEntry : assertions.entrySet()) {
for (QueueAssertion.ValueAssertion assertion : assertionEntry.getValue().assertions) {
if (assertion.resourceSupplier != null) {
String errorMessage = String.format(RESOURCE_ASSERTION_ERROR_MESSAGE,
@@ -204,7 +204,7 @@ class QueueAssertionBuilder {
* Returns all queues that have defined assertions.
* @return queue paths
*/
- public Set<String> getQueues() {
+ public Set<QueuePath> getQueues() {
return assertions.keySet();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 09b52d3b246d..2443f723653c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -48,6 +48,8 @@ public class TestAbsoluteResourceConfiguration {
private static final String QUEUEA2 = "queueA2";
private static final String QUEUEB1 = "queueB1";
+ private static final QueuePath ROOT =
+ new QueuePath(CapacitySchedulerConfiguration.ROOT);
private static final QueuePath QUEUEA_FULL =
new QueuePath(CapacitySchedulerConfiguration.ROOT, QUEUEA);
private static final QueuePath QUEUEB_FULL =
@@ -103,9 +105,9 @@ public class TestAbsoluteResourceConfiguration {
private CapacitySchedulerConfiguration setupNormalizationConfiguration() {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[]{QUEUEA, QUEUEB});
- csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
// 60, 28
csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resource.newInstance(50 * GB, 20));
@@ -119,23 +121,23 @@ public class TestAbsoluteResourceConfiguration {
private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
boolean isCapacityNeeded) {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[]{QUEUEA, QUEUEB, QUEUEC, QUEUED});
// Set default capacities like normal configuration.
if (isCapacityNeeded) {
- csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f);
- csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f);
- csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f);
- csConf.setCapacity(QUEUED_FULL.getFullPath(), 25f);
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ csConf.setCapacity(QUEUED_FULL, 25f);
}
- csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL.getFullPath(), true);
+ csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true);
// Setup leaf queue template configs
- csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL.getFullPath(), "",
+ csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "",
QUEUE_D_TEMPL_MINRES);
- csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL.getFullPath(), "",
+ csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "",
QUEUE_D_TEMPL_MAXRES);
return csConf;
@@ -144,19 +146,19 @@ public class TestAbsoluteResourceConfiguration {
private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
boolean isCapacityNeeded) {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[]{QUEUEA, QUEUEB, QUEUEC});
- csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[]{QUEUEA1, QUEUEA2});
- csConf.setQueues(QUEUEB_FULL.getFullPath(), new String[]{QUEUEB1});
+ csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
// Set default capacities like normal configuration.
if (isCapacityNeeded) {
- csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f);
- csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f);
- csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f);
- csConf.setCapacity(QUEUEA1_FULL.getFullPath(), 50f);
- csConf.setCapacity(QUEUEA2_FULL.getFullPath(), 50f);
- csConf.setCapacity(QUEUEB1_FULL.getFullPath(), 100f);
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ csConf.setCapacity(QUEUEA1_FULL, 50f);
+ csConf.setCapacity(QUEUEA2_FULL, 50f);
+ csConf.setCapacity(QUEUEB1_FULL, 100f);
}
return csConf;
@@ -225,22 +227,22 @@ public class TestAbsoluteResourceConfiguration {
Assert.assertEquals("Min resource configured for QUEUEA is not correct",
QUEUE_A_MINRES,
- csConf.getMinimumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes));
+ csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
Assert.assertEquals("Max resource configured for QUEUEA is not correct",
QUEUE_A_MAXRES,
- csConf.getMaximumResourceRequirement("", QUEUEA_FULL.getFullPath(), resourceTypes));
+ csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
Assert.assertEquals("Min resource configured for QUEUEB is not correct",
QUEUE_B_MINRES,
- csConf.getMinimumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes));
+ csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
Assert.assertEquals("Max resource configured for QUEUEB is not correct",
QUEUE_B_MAXRES,
- csConf.getMaximumResourceRequirement("", QUEUEB_FULL.getFullPath(), resourceTypes));
+ csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
Assert.assertEquals("Min resource configured for QUEUEC is not correct",
QUEUE_C_MINRES,
- csConf.getMinimumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes));
+ csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
Assert.assertEquals("Max resource configured for QUEUEC is not correct",
QUEUE_C_MAXRES,
- csConf.getMaximumResourceRequirement("", QUEUEC_FULL.getFullPath(), resourceTypes));
+ csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@@ -564,9 +566,9 @@ public class TestAbsoluteResourceConfiguration {
// 1. Explicitly set percentage based config for parent queues. This will
// make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
// resource.
- csConf.setCapacity(QUEUEA_FULL.getFullPath(), 50f);
- csConf.setCapacity(QUEUEB_FULL.getFullPath(), 25f);
- csConf.setCapacity(QUEUEC_FULL.getFullPath(), 25f);
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
// Get queue object to verify min/max resource configuration.
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -618,15 +620,15 @@ public class TestAbsoluteResourceConfiguration {
// create conf with basic queue configuration.
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {QUEUEA, QUEUEB});
- csConf.setQueues(QUEUEA_FULL.getFullPath(), new String[] {QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEA_FULL, new String[] {QUEUEA1, QUEUEA2});
// Set default capacities like normal configuration.
- csConf.setCapacity(QUEUEA_FULL.getFullPath(), "[memory=125]");
- csConf.setCapacity(QUEUEB_FULL.getFullPath(), "[memory=0]");
- csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=100]");
- csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=25]");
+ csConf.setCapacity(QUEUEA_FULL, "[memory=125]");
+ csConf.setCapacity(QUEUEB_FULL, "[memory=0]");
+ csConf.setCapacity(QUEUEA1_FULL, "[memory=100]");
+ csConf.setCapacity(QUEUEA2_FULL, "[memory=25]");
// Update min/max resource to queueA
csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
@@ -646,8 +648,8 @@ public class TestAbsoluteResourceConfiguration {
// doesnt throw exception saying "Parent queue 'root.A' and
// child queue 'root.A.A2' should use either percentage
// based capacityconfiguration or absolute resource together for label"
- csConf.setCapacity(QUEUEA1_FULL.getFullPath(), "[memory=125]");
- csConf.setCapacity(QUEUEA2_FULL.getFullPath(), "[memory=0]");
+ csConf.setCapacity(QUEUEA1_FULL, "[memory=125]");
+ csConf.setCapacity(QUEUEA2_FULL, "[memory=0]");
// Get queue object to verify min/max resource configuration.
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
index 54fb966ded84..66ab96308f98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceWithAutoQueue.java
@@ -71,6 +71,13 @@ public class TestAbsoluteResourceWithAutoQueue
private static final String QUEUED_FULL =
CapacitySchedulerConfiguration.ROOT + "." + QUEUED;
+ private static final QueuePath ROOT_QUEUE_PATH =
+ new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A_QUEUE_PATH = new QueuePath(QUEUEA_FULL);
+ private static final QueuePath B_QUEUE_PATH = new QueuePath(QUEUEB_FULL);
+ private static final QueuePath C_QUEUE_PATH = new QueuePath(QUEUEC_FULL);
+ private static final QueuePath D_QUEUE_PATH = new QueuePath(QUEUED_FULL);
+
private static final Resource QUEUE_A_MINRES =
Resource.newInstance(100 * GB, 10);
private static final Resource QUEUE_A_MAXRES =
@@ -104,15 +111,15 @@ public class TestAbsoluteResourceWithAutoQueue
private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
CapacitySchedulerConfiguration csConf) {
// Update min/max resource to queueA/B/C
- csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MINRES);
- csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MINRES);
- csConf.setMinimumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MINRES);
- csConf.setMinimumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MINRES);
+ csConf.setMinimumResourceRequirement("", A_QUEUE_PATH, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", B_QUEUE_PATH, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", C_QUEUE_PATH, QUEUE_C_MINRES);
+ csConf.setMinimumResourceRequirement("", D_QUEUE_PATH, QUEUE_D_MINRES);
- csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEA_FULL), QUEUE_A_MAXRES);
- csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEB_FULL), QUEUE_B_MAXRES);
- csConf.setMaximumResourceRequirement("", new QueuePath(QUEUEC_FULL), QUEUE_C_MAXRES);
- csConf.setMaximumResourceRequirement("", new QueuePath(QUEUED_FULL), QUEUE_D_MAXRES);
+ csConf.setMaximumResourceRequirement("", A_QUEUE_PATH, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", B_QUEUE_PATH, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", C_QUEUE_PATH, QUEUE_C_MAXRES);
+ csConf.setMaximumResourceRequirement("", D_QUEUE_PATH, QUEUE_D_MAXRES);
return csConf;
}
@@ -127,30 +134,30 @@ public class TestAbsoluteResourceWithAutoQueue
boolean isCapacityNeeded) {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT_QUEUE_PATH,
new String[] { QUEUEA, QUEUEB, QUEUEC, QUEUED });
// Set default capacities like normal configuration.
if (isCapacityNeeded) {
- csConf.setCapacity(QUEUEA_FULL, 50f);
- csConf.setCapacity(QUEUEB_FULL, 25f);
- csConf.setCapacity(QUEUEC_FULL, 25f);
- csConf.setCapacity(QUEUED_FULL, 25f);
+ csConf.setCapacity(A_QUEUE_PATH, 50f);
+ csConf.setCapacity(B_QUEUE_PATH, 25f);
+ csConf.setCapacity(C_QUEUE_PATH, 25f);
+ csConf.setCapacity(D_QUEUE_PATH, 25f);
}
- csConf.setAutoCreateChildQueueEnabled(QUEUEC_FULL, true);
+ csConf.setAutoCreateChildQueueEnabled(C_QUEUE_PATH, true);
- csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUEC_FULL, "",
+ csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(C_QUEUE_PATH, "",
QUEUE_C_MINRES);
- csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUEC_FULL, "",
+ csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(C_QUEUE_PATH, "",
QUEUE_C_MAXRES);
- csConf.setAutoCreateChildQueueEnabled(QUEUED_FULL, true);
+ csConf.setAutoCreateChildQueueEnabled(D_QUEUE_PATH, true);
// Setup leaf queue template configs
- csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(QUEUED_FULL, "",
+ csConf.setAutoCreatedLeafQueueTemplateCapacityByLabel(D_QUEUE_PATH, "",
Resource.newInstance(10 * GB, 2));
- csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(QUEUED_FULL, "",
+ csConf.setAutoCreatedLeafQueueTemplateMaxCapacity(D_QUEUE_PATH, "",
QUEUE_D_MAXRES);
return csConf;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 0df032062580..ed5ff571e028 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -145,22 +145,33 @@ public class TestApplicationLimits {
doReturn(100).when(queue).getMaxApplications();
doReturn(25).when(queue).getMaxApplicationsPerUser();
}
-
+
private static final String A = "a";
private static final String B = "b";
+ private static final String C = "c";
+ private static final String D = "d";
+ private static final String AA1 = "a1";
+ private static final String AA2 = "a2";
+ private static final String AA3 = "a3";
+ private static final QueuePath ROOT_QUEUE_PATH =
+ new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(A);
+ private static final QueuePath B_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(B);
+ private static final QueuePath C_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(C);
+ private static final QueuePath D_QUEUE_PATH = ROOT_QUEUE_PATH.createNewLeaf(D);
+ private static final QueuePath AA1_QUEUE_PATH = A_QUEUE_PATH.createNewLeaf(AA1);
+ private static final QueuePath AA2_QUEUE_PATH = A_QUEUE_PATH.createNewLeaf(AA2);
+ private static final QueuePath AA3_QUEUE_PATH = A_QUEUE_PATH.createNewLeaf(AA3);
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
+ conf.setQueues(ROOT_QUEUE_PATH, new String[] {A, B});
- final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
- conf.setCapacity(Q_A, 10);
-
- final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
- conf.setCapacity(Q_B, 90);
+ conf.setCapacity(A_QUEUE_PATH, 10);
+ conf.setCapacity(B_QUEUE_PATH, 90);
- conf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 50);
- conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + "." + A, 5.0f);
+ conf.setUserLimit(A_QUEUE_PATH, 50);
+ conf.setUserLimitFactor(A_QUEUE_PATH, 5.0f);
LOG.info("Setup top-level queues a and b");
}
@@ -336,7 +347,7 @@ public class TestApplicationLimits {
// should return -1 if per queue setting not set
assertEquals(
(int)CapacitySchedulerConfiguration.UNDEFINED,
- csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+ csConf.getMaximumApplicationsPerQueue(queue.getQueuePathObject()));
int expectedMaxApps =
(int)
(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS *
@@ -351,7 +362,7 @@ public class TestApplicationLimits {
// should default to global setting if per queue setting not set
assertEquals(CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT,
csConf.getMaximumApplicationMasterResourcePerQueuePercent(
- queue.getQueuePath()), epsilon);
+ queue.getQueuePathObject()), epsilon);
// Change the per-queue max AM resources percentage.
csConf.setFloat(PREFIX + queue.getQueuePath()
@@ -371,7 +382,7 @@ public class TestApplicationLimits {
assertEquals(0.5f,
csConf.getMaximumApplicationMasterResourcePerQueuePercent(
- queue.getQueuePath()), epsilon);
+ queue.getQueuePathObject()), epsilon);
assertThat(queue.calculateAndGetAMResourceLimit()).isEqualTo(
Resource.newInstance(800 * GB, 1));
@@ -391,7 +402,8 @@ public class TestApplicationLimits {
clusterResource));
queue = (LeafQueue)queues.get(A);
- assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
+ assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(
+ queue.getQueuePathObject()));
assertEquals(9999, queue.getMaxApplications());
expectedMaxAppsPerUser = Math.min(9999, (int)(9999 *
@@ -576,7 +588,7 @@ public class TestApplicationLimits {
public void testHeadroom() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 25);
+ csConf.setUserLimit(A_QUEUE_PATH, 25);
setupQueueConfiguration(csConf);
// Say cluster has 100 nodes of 16G each
@@ -721,53 +733,46 @@ public class TestApplicationLimits {
CapacitySchedulerConfiguration conf =
new CapacitySchedulerConfiguration(config);
// Define top-level
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT_QUEUE_PATH,
new String[]{"a", "b", "c", "d"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ conf.setCapacityByLabel(ROOT_QUEUE_PATH, "x", 100);
+ conf.setCapacityByLabel(ROOT_QUEUE_PATH, "y", 100);
+ conf.setCapacityByLabel(ROOT_QUEUE_PATH, "z", 100);
conf.setInt(CapacitySchedulerConfiguration.QUEUE_GLOBAL_MAX_APPLICATION,
20);
conf.setInt("yarn.scheduler.capacity.root.a.a1.maximum-applications", 1);
conf.setFloat("yarn.scheduler.capacity.root.d.user-limit-factor", 0.1f);
conf.setInt("yarn.scheduler.capacity.maximum-applications", 4);
- final String a = CapacitySchedulerConfiguration.ROOT + ".a";
- final String b = CapacitySchedulerConfiguration.ROOT + ".b";
- final String c = CapacitySchedulerConfiguration.ROOT + ".c";
- final String d = CapacitySchedulerConfiguration.ROOT + ".d";
- final String aa1 = a + ".a1";
- final String aa2 = a + ".a2";
- final String aa3 = a + ".a3";
-
- conf.setQueues(a, new String[]{"a1", "a2", "a3"});
- conf.setCapacity(a, 50);
- conf.setCapacity(b, 50);
- conf.setCapacity(c, 0);
- conf.setCapacity(d, 0);
- conf.setCapacity(aa1, 50);
- conf.setCapacity(aa2, 50);
- conf.setCapacity(aa3, 0);
-
- conf.setCapacityByLabel(a, "y", 25);
- conf.setCapacityByLabel(b, "y", 50);
- conf.setCapacityByLabel(c, "y", 25);
- conf.setCapacityByLabel(d, "y", 0);
-
- conf.setCapacityByLabel(a, "x", 50);
- conf.setCapacityByLabel(b, "x", 50);
-
- conf.setCapacityByLabel(a, "z", 50);
- conf.setCapacityByLabel(b, "z", 50);
-
- conf.setCapacityByLabel(aa1, "x", 100);
- conf.setCapacityByLabel(aa2, "x", 0);
-
- conf.setCapacityByLabel(aa1, "y", 25);
- conf.setCapacityByLabel(aa2, "y", 75);
-
- conf.setCapacityByLabel(aa2, "z", 75);
- conf.setCapacityByLabel(aa3, "z", 25);
+
+ conf.setQueues(A_QUEUE_PATH, new String[]{"a1", "a2", "a3"});
+ conf.setCapacity(A_QUEUE_PATH, 50);
+ conf.setCapacity(B_QUEUE_PATH, 50);
+ conf.setCapacity(C_QUEUE_PATH, 0);
+ conf.setCapacity(D_QUEUE_PATH, 0);
+ conf.setCapacity(AA1_QUEUE_PATH, 50);
+ conf.setCapacity(AA2_QUEUE_PATH, 50);
+ conf.setCapacity(AA3_QUEUE_PATH, 0);
+
+ conf.setCapacityByLabel(A_QUEUE_PATH, "y", 25);
+ conf.setCapacityByLabel(B_QUEUE_PATH, "y", 50);
+ conf.setCapacityByLabel(C_QUEUE_PATH, "y", 25);
+ conf.setCapacityByLabel(D_QUEUE_PATH, "y", 0);
+
+ conf.setCapacityByLabel(A_QUEUE_PATH, "x", 50);
+ conf.setCapacityByLabel(B_QUEUE_PATH, "x", 50);
+
+ conf.setCapacityByLabel(A_QUEUE_PATH, "z", 50);
+ conf.setCapacityByLabel(B_QUEUE_PATH, "z", 50);
+
+ conf.setCapacityByLabel(AA1_QUEUE_PATH, "x", 100);
+ conf.setCapacityByLabel(AA2_QUEUE_PATH, "x", 0);
+
+ conf.setCapacityByLabel(AA1_QUEUE_PATH, "y", 25);
+ conf.setCapacityByLabel(AA2_QUEUE_PATH, "y", 75);
+
+ conf.setCapacityByLabel(AA2_QUEUE_PATH, "z", 75);
+ conf.setCapacityByLabel(AA3_QUEUE_PATH, "z", 25);
return conf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index ef50e5271e71..6e8c0b02f947 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -77,6 +77,15 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
public class TestApplicationLimitsByPartition {
final static int GB = 1024;
+ final static String A1_PATH = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
+ final static String B1_PATH = CapacitySchedulerConfiguration.ROOT + ".b" + ".b1";
+ final static String B2_PATH = CapacitySchedulerConfiguration.ROOT + ".b" + ".b2";
+ final static String C1_PATH = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1";
+ final static QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ final static QueuePath A1 = new QueuePath(A1_PATH);
+ final static QueuePath B1 = new QueuePath(B1_PATH);
+ final static QueuePath B2 = new QueuePath(B2_PATH);
+ final static QueuePath C1 = new QueuePath(C1_PATH);
LeafQueue queue;
RMNodeLabelsManager mgr;
@@ -136,8 +145,6 @@ public class TestApplicationLimitsByPartition {
// After getting queue conf, configure AM resource percent for Queue A1
// as 0.2 (Label X) and for Queue C1 as 0.2 (Empty Label)
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
- final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1";
config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.2f);
config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.2f);
@@ -292,8 +299,6 @@ public class TestApplicationLimitsByPartition {
// After getting queue conf, configure AM resource percent for Queue A1
// as 0.15 (Label X) and for Queue C1 as 0.15 (Empty Label)
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
- final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1";
config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.15f);
config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.15f);
// inject node label manager
@@ -401,7 +406,6 @@ public class TestApplicationLimitsByPartition {
// After getting queue conf, configure AM resource percent for Queue A1
// as 0.2 (not for partition, rather in queue level)
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
config.setMaximumApplicationMasterResourcePerQueuePercent(A1, 0.2f);
// inject node label manager
MockRM rm1 = new MockRM(config) {
@@ -483,7 +487,6 @@ public class TestApplicationLimitsByPartition {
// as 0.4 (Label X). Also set userlimit as 50% for this queue. So when we
// have two users submitting applications, each user will get 50% of AM
// resource which is available in this partition.
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.4f);
config.setUserLimit(A1, 50);
@@ -625,8 +628,6 @@ public class TestApplicationLimitsByPartition {
* A1 : 0.25
* B2 : 0.15
*/
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
- final String B1 = CapacitySchedulerConfiguration.ROOT + ".b" + ".b1";
config.setMaximumAMResourcePercentPerPartition(A1, "y", 0.25f);
config.setMaximumApplicationMasterResourcePerQueuePercent(B1, 0.15f);
@@ -756,8 +757,6 @@ public class TestApplicationLimitsByPartition {
CapacitySchedulerConfiguration csConf =
(CapacitySchedulerConfiguration) TestUtils
.getComplexConfigurationWithQueueLabels(conf);
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
- final String B2 = CapacitySchedulerConfiguration.ROOT + ".b" + ".b2";
csConf.setUserLimit(A1, 25);
csConf.setUserLimit(B2, 25);
@@ -941,9 +940,9 @@ public class TestApplicationLimitsByPartition {
new CapacitySchedulerConfiguration();
csconf.setResourceComparator(DominantResourceCalculator.class);
String queueName = "a1";
- csconf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csconf.setQueues(ROOT,
new String[] {queueName});
- csconf.setCapacity("root." + queueName, 100);
+ csconf.setCapacity(new QueuePath("root.a1"), 100);
ResourceInformation res0 = ResourceInformation.newInstance("memory-mb",
ResourceInformation.MEMORY_MB.getUnits(), GB, Long.MAX_VALUE);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java
index 598bd49b0e7b..e2011ff9cdb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLConfiguration.java
@@ -40,28 +40,36 @@ public class TestApplicationPriorityACLConfiguration {
private static final String QUEUEB = "queueB";
private static final String QUEUEC = "queueC";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A_QUEUE_PATH = new QueuePath(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEA);
+ private static final QueuePath B_QUEUE_PATH = new QueuePath(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEB);
+ private static final QueuePath C_QUEUE_PATH = new QueuePath(
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEC);
+
+
@Test
public void testSimpleACLConfiguration() throws Exception {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[]{QUEUEA, QUEUEB, QUEUEC});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f);
+ csConf.setCapacity(A_QUEUE_PATH, 50f);
+ csConf.setCapacity(B_QUEUE_PATH, 25f);
+ csConf.setCapacity(C_QUEUE_PATH, 25f);
// Success case: Configure one user/group level priority acl for queue A.
String[] aclsForA = new String[2];
aclsForA[0] = QUEUE_A_USER;
aclsForA[1] = QUEUE_A_GROUP;
- csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ csConf.setPriorityAcls(A_QUEUE_PATH,
Priority.newInstance(maxPriorityQueueA),
Priority.newInstance(defaultPriorityQueueA), aclsForA);
// Try to get the ACL configs and make sure there are errors/exceptions
List<AppPriorityACLGroup> pGroupA = csConf.getPriorityAcls(
- CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
- Priority.newInstance(clusterMaxPriority));
+ A_QUEUE_PATH, Priority.newInstance(clusterMaxPriority));
// Validate!
verifyACLs(pGroupA, QUEUE_A_USER, QUEUE_A_GROUP, maxPriorityQueueA,
@@ -71,34 +79,32 @@ public class TestApplicationPriorityACLConfiguration {
@Test
public void testACLConfigurationForInvalidCases() throws Exception {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[]{QUEUEA, QUEUEB, QUEUEC});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f);
+ csConf.setCapacity(A_QUEUE_PATH, 50f);
+ csConf.setCapacity(B_QUEUE_PATH, 25f);
+ csConf.setCapacity(C_QUEUE_PATH, 25f);
// Success case: Configure one user/group level priority acl for queue A.
String[] aclsForA = new String[2];
aclsForA[0] = QUEUE_A_USER;
aclsForA[1] = QUEUE_A_GROUP;
- csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ csConf.setPriorityAcls(A_QUEUE_PATH,
Priority.newInstance(maxPriorityQueueA),
Priority.newInstance(defaultPriorityQueueA), aclsForA);
String[] aclsForB = new String[1];
aclsForB[0] = QUEUE_B_USER;
- csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
+ csConf.setPriorityAcls(B_QUEUE_PATH,
Priority.newInstance(maxPriorityQueueB),
Priority.newInstance(defaultPriorityQueueB), aclsForB);
// Try to get the ACL configs and make sure there are errors/exceptions
List<AppPriorityACLGroup> pGroupA = csConf.getPriorityAcls(
- CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
- Priority.newInstance(clusterMaxPriority));
+ A_QUEUE_PATH, Priority.newInstance(clusterMaxPriority));
List<AppPriorityACLGroup> pGroupB = csConf.getPriorityAcls(
- CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
- Priority.newInstance(clusterMaxPriority));
+ B_QUEUE_PATH, Priority.newInstance(clusterMaxPriority));
// Validate stored ACL values with configured ones.
verifyACLs(pGroupA, QUEUE_A_USER, QUEUE_A_GROUP, maxPriorityQueueA,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java
index d3193537046a..674bb5c8c1eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriorityACLs.java
@@ -177,24 +177,24 @@ public class TestApplicationPriorityACLs extends ACLsTestBase {
@Override
protected Configuration createConfiguration() {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[]{QUEUEA, QUEUEB, QUEUEC});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 25f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEC, 25f);
+ csConf.setCapacity(A_QUEUE_PATH, 50f);
+ csConf.setCapacity(B_QUEUE_PATH, 25f);
+ csConf.setCapacity(C_QUEUE_PATH, 25f);
String[] aclsForA = new String[2];
aclsForA[0] = QUEUE_A_USER;
aclsForA[1] = QUEUE_A_GROUP;
- csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
+ csConf.setPriorityAcls(A_QUEUE_PATH,
Priority.newInstance(maxPriorityQueueA),
Priority.newInstance(defaultPriorityQueueA), aclsForA);
String[] aclsForB = new String[2];
aclsForB[0] = QUEUE_B_USER;
aclsForB[1] = QUEUE_B_GROUP;
- csConf.setPriorityAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
+ csConf.setPriorityAcls(B_QUEUE_PATH,
Priority.newInstance(maxPriorityQueueB),
Priority.newInstance(defaultPriorityQueueB), aclsForB);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java
index 502b6a40ef0a..6ed76b6341c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueDeletionPolicy.java
@@ -48,6 +48,11 @@ public class TestAutoCreatedQueueDeletionPolicy
private CapacitySchedulerQueueManager autoQueueHandler;
+ public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ public static final QueuePath ROOT_A = new QueuePath("root", "a");
+ public static final QueuePath ROOT_A_A1 = QueuePath.createFromQueues("root", "a", "a1");
+ public static final QueuePath ROOT_B = new QueuePath("root", "b");
+
/*
Create the following structure:
root
@@ -63,15 +68,15 @@ public class TestAutoCreatedQueueDeletionPolicy
ResourceScheduler.class);
// By default, set 3 queues, a/b, and a.a1
- csConf.setQueues("root", new String[]{"a", "b"});
- csConf.setNonLabeledQueueWeight("root", 1f);
- csConf.setNonLabeledQueueWeight("root.a", 1f);
- csConf.setNonLabeledQueueWeight("root.b", 1f);
- csConf.setQueues("root.a", new String[]{"a1"});
- csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
- csConf.setAutoQueueCreationV2Enabled("root", true);
- csConf.setAutoQueueCreationV2Enabled("root.a", true);
- csConf.setAutoQueueCreationV2Enabled(PARENT_QUEUE, true);
+ csConf.setQueues(ROOT, new String[]{"a", "b"});
+ csConf.setNonLabeledQueueWeight(ROOT, 1f);
+ csConf.setNonLabeledQueueWeight(ROOT_A, 1f);
+ csConf.setNonLabeledQueueWeight(ROOT_B, 1f);
+ csConf.setQueues(ROOT_A, new String[]{"a1"});
+ csConf.setNonLabeledQueueWeight(ROOT_A_A1, 1f);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(ROOT_A, true);
+ csConf.setAutoQueueCreationV2Enabled(PARENT_QUEUE_PATH, true);
// Test for auto deletion when expired
csConf.setAutoExpiredDeletionTime(1);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
index 201214ec2674..5b3acf11c4fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAutoCreatedQueueTemplate.java
@@ -29,75 +29,79 @@ public class TestAutoCreatedQueueTemplate {
private static final QueuePath TEST_QUEUE_AB = new QueuePath("root.a.b");
private static final QueuePath TEST_QUEUE_A = new QueuePath("root.a");
private static final QueuePath TEST_QUEUE_B = new QueuePath("root.b");
- private static final String ROOT = "root";
+ private static final QueuePath TEST_QUEUE_WILDCARD = new QueuePath("*");
+ private static final QueuePath TEST_QUEUE_ROOT_WILDCARD = new QueuePath("root.*");
+ private static final QueuePath TEST_QUEUE_TWO_LEVEL_WILDCARDS = new QueuePath("root.*.*");
+ private static final QueuePath TEST_QUEUE_A_WILDCARD = new QueuePath("root.a.*");
+ private static final QueuePath ROOT = new QueuePath("root");
private CapacitySchedulerConfiguration conf;
@Before
public void setUp() throws Exception {
conf = new CapacitySchedulerConfiguration();
- conf.setQueues("root", new String[]{"a"});
- conf.setQueues("a", new String[]{"b"});
- conf.setQueues("b", new String[]{"c"});
+ conf.setQueues(ROOT, new String[]{"a"});
+ conf.setQueues(TEST_QUEUE_A, new String[]{"b"});
+ conf.setQueues(TEST_QUEUE_B, new String[]{"c"});
}
@Test
public void testNonWildCardTemplate() {
- conf.set(getTemplateKey(TEST_QUEUE_AB.getFullPath(), "capacity"), "6w");
+ conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w");
AutoCreatedQueueTemplate template =
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
Assert.assertEquals("weight is not set", 6f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
}
@Test
public void testOneLevelWildcardTemplate() {
- conf.set(getTemplateKey("root.a.*", "capacity"), "6w");
+ conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "6w");
AutoCreatedQueueTemplate template =
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
Assert.assertEquals("weight is not set", 6f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
}
@Test
public void testTwoLevelWildcardTemplate() {
- conf.set(getTemplateKey("root.*", "capacity"), "6w");
- conf.set(getTemplateKey("root.*.*", "capacity"), "5w");
+ conf.set(getTemplateKey(TEST_QUEUE_ROOT_WILDCARD, "capacity"), "6w");
+ conf.set(getTemplateKey(TEST_QUEUE_TWO_LEVEL_WILDCARDS, "capacity"), "5w");
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A)
- .setTemplateEntriesForChild(conf, TEST_QUEUE_AB.getFullPath());
+ .setTemplateEntriesForChild(conf, TEST_QUEUE_AB);
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB)
- .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
+ .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
Assert.assertEquals("weight is not set", 6f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_AB.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_AB), 10e-6);
Assert.assertEquals("weight is not set", 5f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
}
@Test
public void testIgnoredWhenRootWildcarded() {
- conf.set(getTemplateKey("*", "capacity"), "6w");
+ conf.set(getTemplateKey(TEST_QUEUE_WILDCARD, "capacity"), "6w");
AutoCreatedQueueTemplate template =
- new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath());
+ new AutoCreatedQueueTemplate(conf, ROOT);
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
Assert.assertEquals("weight is set", -1f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
}
@Test
public void testIgnoredWhenNoParent() {
- conf.set(getTemplateKey("root", "capacity"), "6w");
+ conf.set(getTemplateKey(ROOT, "capacity"), "6w");
AutoCreatedQueueTemplate template =
- new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
+ new AutoCreatedQueueTemplate(conf, ROOT);
template.setTemplateEntriesForChild(conf, ROOT);
Assert.assertEquals("weight is set", -1f,
@@ -106,10 +110,10 @@ public class TestAutoCreatedQueueTemplate {
@Test
public void testWildcardAfterRoot() {
- conf.set(getTemplateKey("root.*", "acl_submit_applications"), "user");
+ conf.set(getTemplateKey(TEST_QUEUE_ROOT_WILDCARD, "acl_submit_applications"), "user");
AutoCreatedQueueTemplate template =
- new AutoCreatedQueueTemplate(conf, new QueuePath("root.a"));
- template.setTemplateEntriesForChild(conf, "root.a");
+ new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A);
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
Assert.assertEquals("acl_submit_applications is set", "user",
template.getTemplateProperties().get("acl_submit_applications"));
@@ -117,80 +121,122 @@ public class TestAutoCreatedQueueTemplate {
@Test
public void testTemplatePrecedence() {
- conf.set(getTemplateKey("root.a.b", "capacity"), "6w");
- conf.set(getTemplateKey("root.a.*", "capacity"), "4w");
- conf.set(getTemplateKey("root.*.*", "capacity"), "2w");
+ conf.set(getTemplateKey(TEST_QUEUE_AB, "capacity"), "6w");
+ conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "4w");
+ conf.set(getTemplateKey(TEST_QUEUE_TWO_LEVEL_WILDCARDS, "capacity"), "2w");
AutoCreatedQueueTemplate template =
new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB);
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC.getFullPath());
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
Assert.assertEquals(
"explicit template does not have the highest precedence", 6f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
CapacitySchedulerConfiguration newConf =
new CapacitySchedulerConfiguration();
- newConf.set(getTemplateKey("root.a.*", "capacity"), "4w");
+ newConf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "4w");
template =
new AutoCreatedQueueTemplate(newConf, TEST_QUEUE_AB);
- template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC.getFullPath());
+ template.setTemplateEntriesForChild(newConf, TEST_QUEUE_ABC);
Assert.assertEquals("precedence is invalid", 4f,
- newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC.getFullPath()), 10e-6);
+ newConf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
}
@Test
public void testRootTemplate() {
- conf.set(getTemplateKey("root", "capacity"), "2w");
+ conf.set(getTemplateKey(ROOT, "capacity"), "2w");
AutoCreatedQueueTemplate template =
- new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath());
+ new AutoCreatedQueueTemplate(conf, ROOT);
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
Assert.assertEquals("root property is not set", 2f,
- conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6);
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
}
@Test
public void testQueueSpecificTemplates() {
- conf.set(getTemplateKey("root", "capacity"), "2w");
- conf.set(getLeafTemplateKey("root",
+ conf.set(getTemplateKey(ROOT, "capacity"), "2w");
+ conf.set(getLeafTemplateKey(ROOT,
"default-node-label-expression"), "test");
- conf.set(getLeafTemplateKey("root", "capacity"), "10w");
+ conf.set(getLeafTemplateKey(ROOT, "capacity"), "10w");
conf.setBoolean(getParentTemplateKey(
- "root", AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false);
+ ROOT, AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE), false);
AutoCreatedQueueTemplate template =
- new AutoCreatedQueueTemplate(conf, new QueuePath(ROOT));
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_A.getFullPath());
- template.setTemplateEntriesForChild(conf, TEST_QUEUE_B.getFullPath(), true);
+ new AutoCreatedQueueTemplate(conf, ROOT);
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_A);
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_B, true);
Assert.assertNull("default-node-label-expression is set for parent",
- conf.getDefaultNodeLabelExpression(TEST_QUEUE_A.getFullPath()));
+ conf.getDefaultNodeLabelExpression(TEST_QUEUE_A));
Assert.assertEquals("default-node-label-expression is not set for leaf",
- "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B.getFullPath()));
+ "test", conf.getDefaultNodeLabelExpression(TEST_QUEUE_B));
Assert.assertFalse("auto queue removal is not disabled for parent",
- conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A.getFullPath()));
+ conf.isAutoExpiredDeletionEnabled(TEST_QUEUE_A));
Assert.assertEquals("weight should not be overridden when set by " +
"queue type specific template",
- 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B.getFullPath()), 10e-6);
+ 10f, conf.getNonLabeledQueueWeight(TEST_QUEUE_B), 10e-6);
Assert.assertEquals("weight should be set by common template",
- 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A.getFullPath()), 10e-6);
+ 2f, conf.getNonLabeledQueueWeight(TEST_QUEUE_A), 10e-6);
}
- private String getTemplateKey(String queuePath, String entryKey) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ @Test
+ public void testWildcardTemplateWithLimitedAutoCreatedQueueDepth() {
+ conf.set(getTemplateKey(TEST_QUEUE_ROOT_WILDCARD, "capacity"), "6w");
+ conf.set(getTemplateKey(TEST_QUEUE_A_WILDCARD, "capacity"), "5w");
+ conf.setMaximumAutoCreatedQueueDepth(TEST_QUEUE_A, 1);
+ conf.setMaximumAutoCreatedQueueDepth(TEST_QUEUE_AB, 1);
+
+ new AutoCreatedQueueTemplate(conf, TEST_QUEUE_A)
+ .setTemplateEntriesForChild(conf, TEST_QUEUE_AB);
+ new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB)
+ .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals("weight is not set", 6f,
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_AB), 10e-6);
+ Assert.assertEquals("weight is not set", 5f,
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+ }
+
+ @Test
+ public void testIgnoredTemplateWithLimitedAutoCreatedQueueDepth() {
+ conf.set(getTemplateKey(TEST_QUEUE_TWO_LEVEL_WILDCARDS, "capacity"), "5w");
+ conf.setMaximumAutoCreatedQueueDepth(TEST_QUEUE_AB, 1);
+
+ new AutoCreatedQueueTemplate(conf, TEST_QUEUE_AB)
+ .setTemplateEntriesForChild(conf, TEST_QUEUE_ABC);
+
+ Assert.assertEquals("weight is set incorrectly", -1f,
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_ABC), 10e-6);
+ }
+
+ @Test
+ public void testIgnoredTemplateWhenQueuePathIsInvalid() {
+ QueuePath invalidPath = new QueuePath("a");
+ conf.set(getTemplateKey(invalidPath, "capacity"), "6w");
+ AutoCreatedQueueTemplate template =
+ new AutoCreatedQueueTemplate(conf, invalidPath);
+ template.setTemplateEntriesForChild(conf, TEST_QUEUE_AB);
+
+ Assert.assertEquals("weight is set using invalid queue path", -1f,
+ conf.getNonLabeledQueueWeight(TEST_QUEUE_AB), 10e-6);
+ }
+
+ private String getTemplateKey(QueuePath queuePath, String entryKey) {
+ return QueuePrefixes.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey;
}
- private String getParentTemplateKey(String queuePath, String entryKey) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ private String getParentTemplateKey(QueuePath queuePath, String entryKey) {
+ return QueuePrefixes.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + entryKey;
}
- private String getLeafTemplateKey(String queuePath, String entryKey) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ private String getLeafTemplateKey(QueuePath queuePath, String entryKey) {
+ return QueuePrefixes.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + entryKey;
}
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
index 04e2f8e298ca..5c0f5d1d318f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSAllocateCustomResource.java
@@ -71,6 +71,9 @@ import static org.junit.Assert.assertEquals;
* */
public class TestCSAllocateCustomResource {
+ private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A = new QueuePath(A_PATH);
private YarnConfiguration conf;
private RMNodeLabelsManager mgr;
@@ -117,7 +120,7 @@ public class TestCSAllocateCustomResource {
.getConfigurationWithMultipleQueues(conf);
newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class, ResourceCalculator.class);
- newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+ newConf.set(QueuePrefixes.getQueuePrefix(A)
+ MAXIMUM_ALLOCATION_MB, "4096");
// We must set this to false to avoid MockRM init configuration with
// resource-types.xml by ResourceUtils.resetResourceTypes(conf);
@@ -298,7 +301,7 @@ public class TestCSAllocateCustomResource {
Resource.newInstance(1000, 10, valuesMax);
// Define top-level queues
- newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ newConf.setQueues(ROOT,
new String[] {"a", "b", "c"});
newConf.setMinimumResourceRequirement("", new QueuePath("root", "a"),
aMINRES);
@@ -316,10 +319,10 @@ public class TestCSAllocateCustomResource {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
Assert.assertEquals(aMINRES,
cs.getConfiguration().
- getMinimumResourceRequirement("", "root.a", resourceTypes));
+ getMinimumResourceRequirement("", A, resourceTypes));
Assert.assertEquals(aMAXRES,
cs.getConfiguration().
- getMaximumResourceRequirement("", "root.a", resourceTypes));
+ getMaximumResourceRequirement("", A, resourceTypes));
// Check the gpu resource of queue is right.
Assert.assertEquals(aMINRES, cs.getQueue("root.a").
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java
index f7460de7aafd..768768e95dbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSMaxRunningAppsEnforcer.java
@@ -103,11 +103,15 @@ public class TestCSMaxRunningAppsEnforcer {
}
private void setupQueues(CapacitySchedulerConfiguration config) {
- config.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {"queue1", "queue2"});
- config.setQueues("root.queue1", new String[] {"subqueue1", "subqueue2"});
- config.setQueues("root.queue1.subqueue1", new String[] {"leaf1"});
- config.setQueues("root.queue1.subqueue2", new String[] {"leaf2"});
+ QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ QueuePath queue1 = new QueuePath("root.queue1");
+ QueuePath subqueue1 = new QueuePath("root.queue1.subqueue1");
+ QueuePath subqueue2 = new QueuePath("root.queue1.subqueue2");
+
+ config.setQueues(root, new String[] {"queue1", "queue2"});
+ config.setQueues(queue1, new String[] {"subqueue1", "subqueue2"});
+ config.setQueues(subqueue1, new String[] {"leaf1"});
+ config.setQueues(subqueue2, new String[] {"leaf2"});
config.setFloat(PREFIX + "root.capacity", 100.0f);
config.setFloat(PREFIX + "root.queue1.capacity", 50.0f);
config.setFloat(PREFIX + "root.queue2.capacity", 50.0f);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index ede33b6f3868..e46d6b07f759 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -31,12 +31,17 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A3;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_PATH;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_PATH;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.DEFAULT;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.checkApplicationResourceUsage;
@@ -587,15 +592,14 @@ public class TestCapacityScheduler {
@Test
public void testMaximumCapacitySetup() {
float delta = 0.0000001f;
- QueuePath queuePathA = new QueuePath(A);
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,
- conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta);
+ conf.getNonLabeledQueueMaximumCapacity(A), delta);
conf.setMaximumCapacity(A, 50.0f);
- assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta);
+ assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A), delta);
conf.setMaximumCapacity(A, -1);
assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,
- conf.getNonLabeledQueueMaximumCapacity(queuePathA), delta);
+ conf.getNonLabeledQueueMaximumCapacity(A), delta);
}
@Test
@@ -636,18 +640,19 @@ public class TestCapacityScheduler {
public void testParseQueueWithAbsoluteResource() {
String childQueue = "testQueue";
String labelName = "testLabel";
+ QueuePath childQueuePath = new QueuePath("root." + childQueue);
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
cs.setRMContext(resourceManager.getRMContext());
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- conf.setQueues("root", new String[] {childQueue});
- conf.setCapacity("root." + childQueue, "[memory=20480,vcores=200]");
- conf.setAccessibleNodeLabels("root." + childQueue,
+ conf.setQueues(ROOT, new String[] {childQueue});
+ conf.setCapacity(childQueuePath, "[memory=20480,vcores=200]");
+ conf.setAccessibleNodeLabels(childQueuePath,
Sets.newHashSet(labelName));
- conf.setCapacityByLabel("root", labelName, "[memory=10240,vcores=100]");
- conf.setCapacityByLabel("root." + childQueue, labelName,
+ conf.setCapacityByLabel(ROOT, labelName, "[memory=10240,vcores=100]");
+ conf.setCapacityByLabel(childQueuePath, labelName,
"[memory=4096,vcores=10]");
cs.init(conf);
@@ -1238,8 +1243,8 @@ public class TestCapacityScheduler {
cs.reinitialize(conf, rmContext);
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueB = findQueue(rootQueue, B);
- CSQueue queueB2 = findQueue(queueB, B2);
+ CSQueue queueB = findQueue(rootQueue, B_PATH);
+ CSQueue queueB2 = findQueue(queueB, B2_PATH);
// When preemption turned on for the whole system
// (yarn.resourcemanager.scheduler.monitor.enable=true), and with no other
@@ -1250,7 +1255,7 @@ public class TestCapacityScheduler {
// Disable preemption at the root queue level.
// The preemption property should be inherited from root all the
// way down so that root.b.b2 should NOT be preemptable.
- conf.setPreemptionDisabled(rootQueue.getQueuePath(), true);
+ conf.setPreemptionDisabled(rootQueue.getQueuePathObject(), true);
cs.reinitialize(conf, rmContext);
assertTrue(
"queue " + B2 + " should have inherited non-preemptability from root",
@@ -1258,8 +1263,8 @@ public class TestCapacityScheduler {
// Enable preemption for root (grandparent) but disable for root.b (parent).
// root.b.b2 should inherit property from parent and NOT be preemptable
- conf.setPreemptionDisabled(rootQueue.getQueuePath(), false);
- conf.setPreemptionDisabled(queueB.getQueuePath(), true);
+ conf.setPreemptionDisabled(rootQueue.getQueuePathObject(), false);
+ conf.setPreemptionDisabled(queueB.getQueuePathObject(), true);
cs.reinitialize(conf, rmContext);
assertTrue(
"queue " + B2 + " should have inherited non-preemptability from parent",
@@ -1267,7 +1272,7 @@ public class TestCapacityScheduler {
// When preemption is turned on for root.b.b2, it should be preemptable
// even though preemption is disabled on root.b (parent).
- conf.setPreemptionDisabled(queueB2.getQueuePath(), false);
+ conf.setPreemptionDisabled(queueB2.getQueuePathObject(), false);
cs.reinitialize(conf, rmContext);
assertFalse("queue " + B2 + " should have been preemptable",
queueB2.getPreemptionDisabled());
@@ -1550,7 +1555,7 @@ public class TestCapacityScheduler {
* A1 A2
*/
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
csConf.setCapacity(A, 50);
csConf.setMaximumCapacity(A, 50);
csConf.setCapacity(B, 50);
@@ -2554,13 +2559,11 @@ public class TestCapacityScheduler {
long defaultLifetime) {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {"default"});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".default", 100);
- csConf.setMaximumLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + ".default", maxLifetime);
- csConf.setDefaultLifetimePerQueue(
- CapacitySchedulerConfiguration.ROOT + ".default", defaultLifetime);
+ csConf.setCapacity(DEFAULT, 100);
+ csConf.setMaximumLifetimePerQueue(DEFAULT, maxLifetime);
+ csConf.setDefaultLifetimePerQueue(DEFAULT, defaultLifetime);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler cs = new CapacityScheduler();
@@ -2896,11 +2899,11 @@ public class TestCapacityScheduler {
.getMaxAbsoluteCapacity(), DELTA);
// Add child queue to a, and reinitialize. Metrics should be updated
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a",
+ csConf.setQueues(A,
new String[] {"a1", "a2", "a3"});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 29.5f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.5f);
- csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3",
+ csConf.setCapacity(A2, 29.5f);
+ csConf.setCapacity(A3, 40.5f);
+ csConf.setMaximumCapacity(A3,
50.0f);
cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null,
@@ -2945,14 +2948,14 @@ public class TestCapacityScheduler {
public void testReservedContainerLeakWhenMoveApplication() throws Exception {
CapacitySchedulerConfiguration csConf
= new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {"a", "b"});
- csConf.setCapacity("root.a", 50);
- csConf.setMaximumCapacity("root.a", 100);
- csConf.setUserLimitFactor("root.a", 100);
- csConf.setCapacity("root.b", 50);
- csConf.setMaximumCapacity("root.b", 100);
- csConf.setUserLimitFactor("root.b", 100);
+ csConf.setCapacity(A, 50);
+ csConf.setMaximumCapacity(A, 100);
+ csConf.setUserLimitFactor(A, 100);
+ csConf.setCapacity(B, 50);
+ csConf.setMaximumCapacity(B, 100);
+ csConf.setUserLimitFactor(B, 100);
YarnConfiguration conf=new YarnConfiguration(csConf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java
index 88e6aff2537d..a3500943f59e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAmbiguousLeafs.java
@@ -29,14 +29,26 @@ import org.junit.Test;
import java.io.IOException;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
-
public class TestCapacitySchedulerAmbiguousLeafs {
/**
* Internal counter for incremental application id generation
*/
int appId = 0;
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".default");
+ private static final QueuePath A = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".a");
+ private static final QueuePath B = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".b");
+ private static final QueuePath A_UNIQUE = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".a.unique");
+ private static final QueuePath A_AMBI = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".a.ambi");
+ private static final QueuePath B_AMBI = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".b.ambi");
+
/**
* Helper method to submit applications via RMClientService, to make sure
* all submissions go through RMAppManager.
@@ -79,19 +91,19 @@ public class TestCapacitySchedulerAmbiguousLeafs {
schedulerConf.setAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
schedulerConf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, "forbidden forbidden");
- schedulerConf.setQueues(ROOT + ".a", new String[] {"unique", "ambi"});
- schedulerConf.setAcl(ROOT + ".a", QueueACL.SUBMIT_APPLICATIONS, "forbidden forbidden");
- schedulerConf.setCapacity(ROOT + ".a", 45);
+ schedulerConf.setQueues(A, new String[] {"unique", "ambi"});
+ schedulerConf.setAcl(A, QueueACL.SUBMIT_APPLICATIONS, "forbidden forbidden");
+ schedulerConf.setCapacity(A, 45);
- schedulerConf.setQueues(ROOT + ".b", new String[] {"ambi"});
- schedulerConf.setCapacity(ROOT + ".b", 45);
- schedulerConf.setCapacity(ROOT + ".default", 10);
+ schedulerConf.setQueues(B, new String[] {"ambi"});
+ schedulerConf.setCapacity(B, 45);
+ schedulerConf.setCapacity(DEFAULT, 10);
- schedulerConf.setCapacity(ROOT + ".a.unique", 50);
- schedulerConf.setAcl(ROOT + ".a.unique", QueueACL.SUBMIT_APPLICATIONS, "* *");
- schedulerConf.setCapacity(ROOT + ".a.ambi", 50);
- schedulerConf.setAcl(ROOT + ".a.ambi", QueueACL.SUBMIT_APPLICATIONS, "* *");
- schedulerConf.setCapacity(ROOT + ".b.ambi", 100);
+ schedulerConf.setCapacity(A_UNIQUE, 50);
+ schedulerConf.setAcl(A_UNIQUE, QueueACL.SUBMIT_APPLICATIONS, "* *");
+ schedulerConf.setCapacity(A_AMBI, 50);
+ schedulerConf.setAcl(A_AMBI, QueueACL.SUBMIT_APPLICATIONS, "* *");
+ schedulerConf.setCapacity(B_AMBI, 100);
schedulerConf.set(CapacitySchedulerConfiguration.MAPPING_RULE_FORMAT, "json");
//Simple %specified mapping rule for all submissions with skip fallback
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
index d192e7dcc693..e881c3327b67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerApps.java
@@ -97,6 +97,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.setupQueueConfiguration;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.appHelper;
@@ -1042,7 +1043,7 @@ public class TestCapacitySchedulerApps {
MockRM rm = setUpMove();
ResourceScheduler scheduler = rm.getResourceScheduler();
CapacityScheduler cs = (CapacityScheduler) scheduler;
- cs.getQueueContext().getConfiguration().setInt(CapacitySchedulerConfiguration.getQueuePrefix(A1)
+ cs.getQueueContext().getConfiguration().setInt(QueuePrefixes.getQueuePrefix(A1)
+ CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS, MAX_PARALLEL_APPS);
cs.reinitialize(cs.getQueueContext().getConfiguration(), mockContext);
List<ApplicationAttemptId> attemptIds = new ArrayList<>();
@@ -1173,7 +1174,7 @@ public class TestCapacitySchedulerApps {
new CapacitySchedulerConfiguration(conf);
// Define top-level queues
- newConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ newConf.setQueues(ROOT,
new String[]{"a", "b"});
newConf.setCapacity(A, 50);
@@ -1469,7 +1470,7 @@ public class TestCapacitySchedulerApps {
ResourceScheduler.class);
conf.setQueues(A, new String[]{"a1", "a2", "b"});
conf.setCapacity(A1, 20);
- conf.setCapacity("root.a.b", 10);
+ conf.setCapacity(new QueuePath("root.a.b"), 10);
MockRM rm = new MockRM(conf);
rm.start();
@@ -1544,7 +1545,7 @@ public class TestCapacitySchedulerApps {
new CapacitySchedulerConfiguration(config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[]{"a", "b"});
conf.setCapacity(A, 50);
conf.setCapacity(B, 50);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
index 48431b6eaf0f..e03e59d2512c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueueBase.java
@@ -96,8 +96,6 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
.capacity.CapacitySchedulerConfiguration.FAIR_APP_ORDERING_POLICY;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
- .capacity.CapacitySchedulerConfiguration.ROOT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -111,23 +109,44 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
public static final ContainerUpdates NULL_UPDATE_REQUESTS =
new ContainerUpdates();
- public static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
- public static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
- public static final String C = CapacitySchedulerConfiguration.ROOT + ".c";
- public static final String D = CapacitySchedulerConfiguration.ROOT + ".d";
- public static final String E = CapacitySchedulerConfiguration.ROOT + ".e";
- public static final String ESUBGROUP1 =
+ public static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default";
+ public static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ public static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ public static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c";
+ public static final String D_PATH = CapacitySchedulerConfiguration.ROOT + ".d";
+ public static final String E_PATH = CapacitySchedulerConfiguration.ROOT + ".e";
+
+ public static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ public static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH);
+ public static final QueuePath A = new QueuePath(A_PATH);
+ public static final QueuePath B = new QueuePath(B_PATH);
+ public static final QueuePath C = new QueuePath(C_PATH);
+ public static final QueuePath D = new QueuePath(D_PATH);
+ public static final QueuePath E = new QueuePath(E_PATH);
+ public static final String ESUBGROUP1_PATH =
CapacitySchedulerConfiguration.ROOT + ".esubgroup1";
- public static final String FGROUP =
+ public static final String FGROUP_PATH =
CapacitySchedulerConfiguration.ROOT + ".fgroup";
- public static final String A1 = A + ".a1";
- public static final String A2 = A + ".a2";
- public static final String B1 = B + ".b1";
- public static final String B2 = B + ".b2";
- public static final String B3 = B + ".b3";
- public static final String B4 = B + ".b4subgroup1";
- public static final String ESUBGROUP1_A = ESUBGROUP1 + ".e";
- public static final String FGROUP_F = FGROUP + ".f";
+ public static final String A1_PATH = A_PATH + ".a1";
+ public static final String A2_PATH = A_PATH + ".a2";
+ public static final String B1_PATH = B_PATH + ".b1";
+ public static final String B2_PATH = B_PATH + ".b2";
+ public static final String B3_PATH = B_PATH + ".b3";
+ public static final String B4_PATH = B_PATH + ".b4subgroup1";
+ public static final String ESUBGROUP1_A_PATH = ESUBGROUP1_PATH + ".e";
+ public static final String FGROUP_F_PATH = FGROUP_PATH + ".f";
+
+ public static final QueuePath A1 = new QueuePath(A1_PATH);
+ public static final QueuePath A2 = new QueuePath(A2_PATH);
+ public static final QueuePath B1 = new QueuePath(B1_PATH);
+ public static final QueuePath B2 = new QueuePath(B2_PATH);
+ public static final QueuePath B3 = new QueuePath(B3_PATH);
+ public static final QueuePath B4 = new QueuePath(B4_PATH);
+ public static final QueuePath E_GROUP = new QueuePath(ESUBGROUP1_PATH);
+ public static final QueuePath F_GROUP = new QueuePath(FGROUP_PATH);
+ public static final QueuePath E_SG = new QueuePath(ESUBGROUP1_A_PATH);
+ public static final QueuePath F_SG = new QueuePath(FGROUP_F_PATH);
+
public static final float A_CAPACITY = 20f;
public static final float B_CAPACITY = 20f;
public static final float C_CAPACITY = 20f;
@@ -159,6 +178,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
public static final String USER2 = USER + 2;
public static final String USER3 = USER + 3;
public static final String PARENT_QUEUE = "c";
+ public static final QueuePath PARENT_QUEUE_PATH = new QueuePath(PARENT_QUEUE);
public static final Set<String> accessibleNodeLabelsOnC = new HashSet<>();
@@ -375,8 +395,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
conf.setCapacity(B, B_CAPACITY);
conf.setCapacity(C, C_CAPACITY);
conf.setCapacity(D, D_CAPACITY);
- conf.setCapacity(ESUBGROUP1, ESUBGROUP1_CAPACITY);
- conf.setCapacity(FGROUP, FGROUP_CAPACITY);
+ conf.setCapacity(E_GROUP, ESUBGROUP1_CAPACITY);
+ conf.setCapacity(F_GROUP, FGROUP_CAPACITY);
// Define 2nd-level queues
conf.setQueues(A, new String[] { "a1", "a2" });
@@ -395,12 +415,12 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
conf.setCapacity(B4, B4_CAPACITY);
conf.setUserLimitFactor(B4, 100.0f);
- conf.setQueues(ESUBGROUP1, new String[] {"e"});
- conf.setCapacity(ESUBGROUP1_A, 100f);
- conf.setUserLimitFactor(ESUBGROUP1_A, 100.0f);
- conf.setQueues(FGROUP, new String[] {"f"});
- conf.setCapacity(FGROUP_F, 100f);
- conf.setUserLimitFactor(FGROUP_F, 100.0f);
+ conf.setQueues(E_GROUP, new String[] {"e"});
+ conf.setCapacity(E_SG, 100f);
+ conf.setUserLimitFactor(E_SG, 100.0f);
+ conf.setQueues(F_GROUP, new String[] {"f"});
+ conf.setCapacity(F_SG, 100f);
+ conf.setUserLimitFactor(F_SG, 100.0f);
conf.setUserLimitFactor(C, 1.0f);
conf.setAutoCreateChildQueueEnabled(C, true);
@@ -471,7 +491,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
//setup new queues with one of them auto enabled
// Define top-level queues
// Set childQueue for root
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] {"c"});
conf.setCapacity(C, 100f);
@@ -493,7 +513,7 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
//setup new queues with one of them auto enabled
// Define top-level queues
// Set childQueue for root
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] {"c"});
conf.setCapacity(C, 100f);
@@ -796,8 +816,8 @@ public class TestCapacitySchedulerAutoCreatedQueueBase {
if (expectedQueueEntitlements.get(label).getCapacity() > EPSILON) {
if (leafQueue.getCapacityConfigType().equals(ABSOLUTE_RESOURCE)) {
- String templatePrefix = cs.getConfiguration().getAutoCreatedQueueTemplateConfPrefix(
- parentQueue.getQueuePath());
+ QueuePath templatePrefix = QueuePrefixes.getAutoCreatedQueueObjectTemplateConfPrefix(
+ parentQueue.getQueuePathObject());
Resource resourceTemplate = parentQueue.getLeafQueueTemplate().getLeafQueueConfigs()
.getMinimumResourceRequirement(label, templatePrefix, RESOURCE_TYPES);
assertEquals(resourceTemplate, leafQueue.getEffectiveCapacity(label));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java
index a025f1ed7de0..09724b7d8d6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoCreatedQueuePreemption.java
@@ -26,6 +26,8 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
.capacity.TestCapacitySchedulerAutoCreatedQueueBase.D;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
.capacity.TestCapacitySchedulerAutoCreatedQueueBase.E;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .capacity.TestCapacitySchedulerAutoCreatedQueueBase.ROOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
.capacity.TestCapacitySchedulerAutoCreatedQueueBase.USER0;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler
@@ -60,7 +62,7 @@ public class TestCapacitySchedulerAutoCreatedQueuePreemption
//setup new queues with one of them auto enabled
// Define top-level queues
// Set childQueue for root
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "c" });
+ conf.setQueues(ROOT, new String[] {"c"});
conf.setCapacity(C, 100f);
conf.setUserLimitFactor(C, 1.0f);
@@ -91,7 +93,7 @@ public class TestCapacitySchedulerAutoCreatedQueuePreemption
//setup new queues with one of them auto enabled
// Define top-level queues
// Set childQueue for root
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] { "c", "d", "e" });
conf.setCapacity(C, 45f);
conf.setCapacity(D, 45f);
@@ -120,8 +122,8 @@ public class TestCapacitySchedulerAutoCreatedQueuePreemption
conf.setAutoCreatedLeafQueueConfigUserLimit(E, 100);
conf.setAutoCreatedLeafQueueConfigUserLimitFactor(E, 3.0f);
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1);
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".d", 2);
+ conf.setQueuePriority(C, 1);
+ conf.setQueuePriority(D, 2);
return conf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index 590b0a4b7668..cd00d31bcd03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -584,7 +584,7 @@ public class TestCapacitySchedulerAutoQueueCreation
//And weight mode, to allow dynamic auto queue creation for root
CapacitySchedulerConfiguration conf = setupSchedulerConfiguration();
conf.setAutoQueueCreationV2Enabled(ROOT, true);
- conf.setCapacity("root.default", "1w");
+ conf.setCapacity(DEFAULT, "1w");
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
@@ -1006,15 +1006,14 @@ public class TestCapacitySchedulerAutoQueueCreation
try {
CapacitySchedulerConfiguration csConf
= new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {"a", "b"});
- csConf.setCapacity("root.a", 90);
- csConf.setCapacity("root.b", 10);
- csConf.setAutoCreateChildQueueEnabled("root.a", true);
- csConf.setAutoCreatedLeafQueueConfigCapacity("root.a", 50);
- csConf.setAutoCreatedLeafQueueConfigMaxCapacity("root.a", 100);
- csConf.setAcl("root.a", QueueACL.ADMINISTER_QUEUE, "*");
- csConf.setAcl("root.a", QueueACL.SUBMIT_APPLICATIONS, "*");
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
+ csConf.setCapacity(A, 90);
+ csConf.setCapacity(B, 10);
+ csConf.setAutoCreateChildQueueEnabled(A, true);
+ csConf.setAutoCreatedLeafQueueConfigCapacity(A, 50);
+ csConf.setAutoCreatedLeafQueueConfigMaxCapacity(A, 100);
+ csConf.setAcl(A, QueueACL.ADMINISTER_QUEUE, "*");
+ csConf.setAcl(A, QueueACL.SUBMIT_APPLICATIONS, "*");
csConf.setBoolean(YarnConfiguration
.APPLICATION_TAG_BASED_PLACEMENT_ENABLED, true);
csConf.setStrings(YarnConfiguration
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
index 7303a1407236..8c9f518fcfbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
@@ -699,10 +699,10 @@ public class TestCapacitySchedulerConfigValidator {
csConf.set(YarnConfiguration.RESOURCE_TYPES, ResourceInformation.GPU_URI);
}
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(new QueuePath(CapacitySchedulerConfiguration.ROOT),
new String[]{PARENT_A, PARENT_B});
- csConf.setQueues(PARENT_A_FULL_PATH.getFullPath(), new String[]{LEAF_A});
- csConf.setQueues(PARENT_B_FULL_PATH.getFullPath(), new String[]{LEAF_B});
+ csConf.setQueues(PARENT_A_FULL_PATH, new String[]{LEAF_A});
+ csConf.setQueues(PARENT_B_FULL_PATH, new String[]{LEAF_B});
if (useDominantRC) {
setupGpuResourceValues();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java
index 2cbb11fc9bdb..85e0ff494eb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfiguration.java
@@ -24,15 +24,15 @@ import org.junit.Test;
import java.util.Set;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ROOT;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.getQueuePrefix;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TestCapacitySchedulerConfiguration {
- private static final String ROOT_TEST = ROOT + ".test";
+ private static final String ROOT_TEST_PATH = CapacitySchedulerConfiguration.ROOT + ".test";
+ private static final QueuePath ROOT_TEST = new QueuePath(ROOT_TEST_PATH);
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
private static final String EMPTY_ACL = "";
private static final String SPACE_ACL = " ";
private static final String USER1 = "user1";
@@ -47,29 +47,29 @@ public class TestCapacitySchedulerConfiguration {
return new CapacitySchedulerConfiguration(new Configuration(false), false);
}
- private AccessControlList getSubmitAcl(CapacitySchedulerConfiguration csConf, String queue) {
+ private AccessControlList getSubmitAcl(CapacitySchedulerConfiguration csConf, QueuePath queue) {
return csConf.getAcl(queue, QueueACL.SUBMIT_APPLICATIONS);
}
- private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, String queue,
+ private void setSubmitAppsConfig(CapacitySchedulerConfiguration csConf, QueuePath queue,
String value) {
csConf.set(getSubmitAppsConfigKey(queue), value);
}
- private String getSubmitAppsConfigKey(String queue) {
- return getQueuePrefix(queue) + "acl_submit_applications";
+ private String getSubmitAppsConfigKey(QueuePath queue) {
+ return QueuePrefixes.getQueuePrefix(queue) + "acl_submit_applications";
}
- private void testWithGivenAclNoOneHasAccess(String queue, String aclValue) {
+ private void testWithGivenAclNoOneHasAccess(QueuePath queue, String aclValue) {
testWithGivenAclNoOneHasAccessInternal(queue, queue, aclValue);
}
- private void testWithGivenAclNoOneHasAccess(String queueToSet, String queueToVerify,
+ private void testWithGivenAclNoOneHasAccess(QueuePath queueToSet, QueuePath queueToVerify,
String aclValue) {
testWithGivenAclNoOneHasAccessInternal(queueToSet, queueToVerify, aclValue);
}
- private void testWithGivenAclNoOneHasAccessInternal(String queueToSet, String queueToVerify,
+ private void testWithGivenAclNoOneHasAccessInternal(QueuePath queueToSet, QueuePath queueToVerify,
String aclValue) {
CapacitySchedulerConfiguration csConf = createDefaultCsConf();
setSubmitAppsConfig(csConf, queueToSet, aclValue);
@@ -79,14 +79,14 @@ public class TestCapacitySchedulerConfiguration {
assertFalse(acl.isAllAllowed());
}
- private void testWithGivenAclCorrectUserAndGroupHasAccess(String queue, String aclValue,
+ private void testWithGivenAclCorrectUserAndGroupHasAccess(QueuePath queue, String aclValue,
Set<String> expectedUsers, Set<String> expectedGroups) {
testWithGivenAclCorrectUserAndGroupHasAccessInternal(queue, queue, aclValue, expectedUsers,
expectedGroups);
}
- private void testWithGivenAclCorrectUserAndGroupHasAccessInternal(String queueToSet,
- String queueToVerify, String aclValue, Set<String> expectedUsers,
+ private void testWithGivenAclCorrectUserAndGroupHasAccessInternal(QueuePath queueToSet,
+ QueuePath queueToVerify, String aclValue, Set<String> expectedUsers,
Set<String> expectedGroups) {
CapacitySchedulerConfiguration csConf = createDefaultCsConf();
setSubmitAppsConfig(csConf, queueToSet, aclValue);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index e8bbaec17c6f..e8577fba2c38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2_CAPACITY;
@@ -30,6 +31,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.getDefaultCapacities;
import static org.junit.Assert.assertEquals;
@@ -292,7 +294,7 @@ public class TestCapacitySchedulerDynamicBehavior {
private void setupPlanQueueConfiguration(CapacitySchedulerConfiguration conf) {
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] { "a", "b" });
conf.setCapacity(A, A_CAPACITY);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
index e5c6710126d4..8f07668192df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerLazyPreemption.java
@@ -651,7 +651,8 @@ public class TestCapacitySchedulerLazyPreemption
* from app1, and app2 will receive the preempted container
*/
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
- csConf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".c", 0.1f);
+ QueuePath cQueuePath = new QueuePath(CapacitySchedulerConfiguration.ROOT + ".c");
+ csConf.setUserLimitFactor(cQueuePath, 0.1f);
MockRM rm1 = new MockRM(csConf);
rm1.getRMContext().setNodeLabelManager(mgr);
rm1.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
index b08903893ffe..484308442ff3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerMultiNodes.java
@@ -64,6 +64,7 @@ public class TestCapacitySchedulerMultiNodes {
private static final Logger LOG = LoggerFactory
.getLogger(TestCapacitySchedulerMultiNodes.class);
+ private static final QueuePath DEFAULT = new QueuePath("root.default");
private CapacitySchedulerConfiguration conf;
private static final String POLICY_CLASS_NAME =
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.ResourceUsageMultiNodeLookupPolicy";
@@ -204,7 +205,7 @@ public class TestCapacitySchedulerMultiNodes {
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME
+ ".resource-based.sorting-interval.ms", 0);
- newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT,
1.0f);
MockRM rm1 = new MockRM(newConf);
@@ -290,7 +291,7 @@ public class TestCapacitySchedulerMultiNodes {
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME
+ ".resource-based.sorting-interval.ms", 0);
- newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT,
1.0f);
MockRM rm1 = new MockRM(newConf);
@@ -361,7 +362,7 @@ public class TestCapacitySchedulerMultiNodes {
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME
+ ".resource-based.sorting-interval.ms", 0);
- newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT,
1.0f);
MockRM rm1 = new MockRM(newConf);
@@ -490,7 +491,7 @@ public class TestCapacitySchedulerMultiNodes {
YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
newConf.setInt(CapacitySchedulerConfiguration.MULTI_NODE_SORTING_POLICY_NAME
+ ".resource-based.sorting-interval.ms", 0);
- newConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default", 1.0f);
+ newConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 1.0f);
newConf.set(CapacitySchedulerConfiguration.SKIP_ALLOCATE_ON_NODES_WITH_RESERVED_CONTAINERS,
"true");
MockRM rm1 = new MockRM(newConf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index d2891379c47e..9e2134143b8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -59,6 +59,12 @@ public class TestCapacitySchedulerNewQueueAutoCreation
private static final Logger LOG = LoggerFactory.getLogger(
org.apache.hadoop.yarn.server.resourcemanager
.scheduler.capacity.TestCapacitySchedulerAutoCreatedQueueBase.class);
+ private static final QueuePath EMPTY_AUTO_PARENT = new QueuePath("root.empty-auto-parent");
+ private static final QueuePath A_A2_AUTO = new QueuePath("root.a.a2-auto");
+ private static final QueuePath E_AUTO = new QueuePath("root.e-auto");
+ private static final QueuePath E_E1 = new QueuePath("root.e.e1");
+ private static final QueuePath A_A_AUTO_A2 = new QueuePath("root.a.a-auto.a2");
+ private static final QueuePath A_A1_AUTO_A2_AUTO = new QueuePath("root.a.a1-auto.a2-auto");
public static final int GB = 1024;
public static final int MAX_MEMORY = 1200;
private MockRM mockRM = null;
@@ -91,16 +97,16 @@ public class TestCapacitySchedulerNewQueueAutoCreation
ResourceScheduler.class);
// By default, set 3 queues, a/b, and a.a1
- csConf.setQueues("root", new String[]{"a", "b"});
- csConf.setNonLabeledQueueWeight("root", 1f);
- csConf.setNonLabeledQueueWeight("root.a", 1f);
- csConf.setNonLabeledQueueWeight("root.b", 1f);
- csConf.setQueues("root.a", new String[]{"a1"});
- csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
- csConf.setAutoQueueCreationV2Enabled("root", true);
- csConf.setAutoQueueCreationV2Enabled("root.a", true);
- csConf.setAutoQueueCreationV2Enabled("root.e", true);
- csConf.setAutoQueueCreationV2Enabled(PARENT_QUEUE, true);
+ csConf.setQueues(ROOT, new String[]{"a", "b"});
+ csConf.setNonLabeledQueueWeight(ROOT, 1f);
+ csConf.setNonLabeledQueueWeight(A, 1f);
+ csConf.setNonLabeledQueueWeight(B, 1f);
+ csConf.setQueues(A, new String[]{"a1"});
+ csConf.setNonLabeledQueueWeight(A1, 1f);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(A, true);
+ csConf.setAutoQueueCreationV2Enabled(E, true);
+ csConf.setAutoQueueCreationV2Enabled(new QueuePath(PARENT_QUEUE), true);
// Test for auto deletion when expired
csConf.setAutoExpiredDeletionTime(1);
}
@@ -288,7 +294,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
public void testAutoCreateQueueAutoParentTwoLevelsWithSiblings()
throws Exception {
startScheduler();
- csConf.setAutoQueueCreationV2Enabled("root.a.a2-auto", true);
+ csConf.setAutoQueueCreationV2Enabled(A_A2_AUTO, true);
// root.a has 1/2 abs resource -> a1 and a2-auto same weight 1/4
// -> a3-auto is alone with weight 1/4
@@ -326,9 +332,9 @@ public class TestCapacitySchedulerNewQueueAutoCreation
// create a new dynamic queue when the sibling is
// not in weight mode
assumeThat(csConf.isLegacyQueueMode(), is(true));
- csConf.setCapacity("root.a", 50f);
- csConf.setCapacity("root.b", 50f);
- csConf.setCapacity("root.a.a1", 100f);
+ csConf.setCapacity(A, 50f);
+ csConf.setCapacity(B, 50f);
+ csConf.setCapacity(A1, 100f);
cs.reinitialize(csConf, mockRM.getRMContext());
createQueue("root.a.a2-auto");
}
@@ -342,7 +348,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
() -> createQueue("root.a.a3-auto.a4-auto.a5-auto"));
// Set depth 3 for root.a, making it a valid scenario
- csConf.setMaximumAutoCreatedQueueDepth("root.a", 3);
+ csConf.setMaximumAutoCreatedQueueDepth(A, 3);
cs.reinitialize(csConf, mockRM.getRMContext());
try {
createQueue("root.a.a3-auto.a4-auto.a5-auto");
@@ -353,7 +359,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
// Set global depth to 3
csConf.setMaximumAutoCreatedQueueDepth(3);
- csConf.unset(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+ csConf.unset(QueuePrefixes.getQueuePrefix(A)
+ CapacitySchedulerConfiguration.MAXIMUM_QUEUE_DEPTH);
cs.reinitialize(csConf, mockRM.getRMContext());
try {
@@ -364,7 +370,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
}
// Set depth on a dynamic queue, which has no effect on auto queue creation validation
- csConf.setMaximumAutoCreatedQueueDepth("root.a.a6-auto.a7-auto.a8-auto", 10);
+ csConf.setMaximumAutoCreatedQueueDepth(new QueuePath("root.a.a6-auto.a7-auto.a8-auto"), 10);
Assert.assertThrows(SchedulerDynamicEditException.class,
() -> createQueue("root.a.a6-auto.a7-auto.a8-auto.a9-auto.a10-auto.a11-auto"));
}
@@ -373,7 +379,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
public void testAutoCreateQueueShouldFailIfNotEnabledForParent()
throws Exception {
startScheduler();
- csConf.setAutoQueueCreationV2Enabled("root", false);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, false);
cs.reinitialize(csConf, mockRM.getRMContext());
createQueue("root.c-auto");
}
@@ -401,7 +407,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
createBasicQueueStructureAndValidate();
// Now, update root.a's weight to 6
- csConf.setNonLabeledQueueWeight("root.a", 6f);
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.a"), 6f);
cs.reinitialize(csConf, mockRM.getRMContext());
// Double confirm, after refresh, we should still see root queue has 5
@@ -419,8 +425,8 @@ public class TestCapacitySchedulerNewQueueAutoCreation
Assert.assertEquals(6f, a.getQueueCapacities().getWeight(), 1e-6);
// Set queue c-auto's weight to 6, and mark c-auto to be static queue
- csConf.setQueues("root", new String[]{"a", "b", "c-auto"});
- csConf.setNonLabeledQueueWeight("root.c-auto", 6f);
+ csConf.setQueues(ROOT, new String[]{"a", "b", "c-auto"});
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.c-auto"), 6f);
cs.reinitialize(csConf, mockRM.getRMContext());
// Get queue c
@@ -437,10 +443,10 @@ public class TestCapacitySchedulerNewQueueAutoCreation
createQueue("root.e-auto.e2-auto");
// Do change 2nd level queue from dynamic to static
- csConf.setQueues("root", new String[]{"a", "b", "c-auto", "e-auto"});
- csConf.setNonLabeledQueueWeight("root.e-auto", 6f);
- csConf.setQueues("root.e-auto", new String[]{"e1-auto"});
- csConf.setNonLabeledQueueWeight("root.e-auto.e1-auto", 6f);
+ csConf.setQueues(ROOT, new String[]{"a", "b", "c-auto", "e-auto"});
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.e-auto"), 6f);
+ csConf.setQueues(E_AUTO, new String[]{"e1-auto"});
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.e-auto.e1-auto"), 6f);
cs.reinitialize(csConf, mockRM.getRMContext());
// Get queue e1
@@ -467,9 +473,9 @@ public class TestCapacitySchedulerNewQueueAutoCreation
public void testConvertDynamicParentToStaticParent() throws Exception {
startScheduler();
createQueue("root.d-auto.d1-auto");
- csConf.setQueues("root", new String[]{"a", "b", "d-auto"});
- csConf.setNonLabeledQueueWeight("root.a", 6f);
- csConf.setNonLabeledQueueWeight("root.d-auto", 1f);
+ csConf.setQueues(ROOT, new String[]{"a", "b", "d-auto"});
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.a"), 6f);
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.d-auto"), 1f);
cs.reinitialize(csConf, mockRM.getRMContext());
CSQueue d = cs.getQueue("root.d-auto");
@@ -504,14 +510,14 @@ public class TestCapacitySchedulerNewQueueAutoCreation
public void testChildlessParentQueueWhenAutoQueueCreationEnabled()
throws Exception {
startScheduler();
- csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
- csConf.setNonLabeledQueueWeight("root", 1f);
- csConf.setNonLabeledQueueWeight("root.a", 1f);
- csConf.setNonLabeledQueueWeight("root.b", 1f);
- csConf.setQueues("root.a", new String[]{"a1"});
- csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
- csConf.setAutoQueueCreationV2Enabled("root", true);
- csConf.setAutoQueueCreationV2Enabled("root.a", true);
+ csConf.setQueues(ROOT, new String[]{"a", "b", "empty-auto-parent"});
+ csConf.setNonLabeledQueueWeight(ROOT, 1f);
+ csConf.setNonLabeledQueueWeight(A, 1f);
+ csConf.setNonLabeledQueueWeight(B, 1f);
+ csConf.setQueues(A, new String[]{"a1"});
+ csConf.setNonLabeledQueueWeight(A1, 1f);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(A, true);
cs.reinitialize(csConf, mockRM.getRMContext());
CSQueue empty = cs.getQueue("root.empty-auto-parent");
@@ -519,15 +525,15 @@ public class TestCapacitySchedulerNewQueueAutoCreation
empty instanceof LeafQueue);
empty.stopQueue();
- csConf.setQueues("root", new String[]{"a", "b", "empty-auto-parent"});
- csConf.setNonLabeledQueueWeight("root", 1f);
- csConf.setNonLabeledQueueWeight("root.a", 1f);
- csConf.setNonLabeledQueueWeight("root.b", 1f);
- csConf.setQueues("root.a", new String[]{"a1"});
- csConf.setNonLabeledQueueWeight("root.a.a1", 1f);
- csConf.setAutoQueueCreationV2Enabled("root", true);
- csConf.setAutoQueueCreationV2Enabled("root.a", true);
- csConf.setAutoQueueCreationV2Enabled("root.empty-auto-parent", true);
+ csConf.setQueues(ROOT, new String[]{"a", "b", "empty-auto-parent"});
+ csConf.setNonLabeledQueueWeight(ROOT, 1f);
+ csConf.setNonLabeledQueueWeight(A, 1f);
+ csConf.setNonLabeledQueueWeight(B, 1f);
+ csConf.setQueues(A, new String[]{"a1"});
+ csConf.setNonLabeledQueueWeight(A1, 1f);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(A, true);
+ csConf.setAutoQueueCreationV2Enabled(EMPTY_AUTO_PARENT, true);
cs.reinitialize(csConf, mockRM.getRMContext());
empty = cs.getQueue("root.empty-auto-parent");
@@ -644,7 +650,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
Assert.assertEquals(1000, e.getMaxApplications());
// when set some queue for max apps
- csConf.setMaximumApplicationsPerQueue("root.e1", 50);
+ csConf.setMaximumApplicationsPerQueue(new QueuePath("root.e1"), 50);
createQueue("root.e1");
LeafQueue e1 = (LeafQueue)cs.
getQueue("root.e1");
@@ -705,7 +711,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
public void testAutoCreateQueueMaxQueuesLimit() throws Exception {
startScheduler();
- csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit("root.e", 5);
+ csConf.setAutoCreatedQueuesV2MaxChildQueuesLimit(E, 5);
cs.reinitialize(csConf, mockRM.getRMContext());
for (int i = 0; i < 5; ++i) {
@@ -725,8 +731,13 @@ public class TestCapacitySchedulerNewQueueAutoCreation
@Test
public void testAutoCreatedQueueTemplateConfig() throws Exception {
startScheduler();
+
+ QueuePath childQueuesOfA = new QueuePath("root.a.*");
+ QueuePath aQueuePath = new QueuePath("root.a");
+ QueuePath cQueuePath = new QueuePath("root.c");
+
csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
- "root.a.*") + "capacity", "6w");
+ childQueuesOfA) + "capacity", "6w");
cs.reinitialize(csConf, mockRM.getRMContext());
AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2");
@@ -740,9 +751,9 @@ public class TestCapacitySchedulerNewQueueAutoCreation
// Set the user-limit-factor and maximum-am-resource-percent via templates to ensure their
// modified defaults are indeed overridden
csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
- "root.a.*") + "user-limit-factor", "10");
+ childQueuesOfA) + "user-limit-factor", "10");
csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
- "root.a.*") + "maximum-am-resource-percent", "0.8");
+ childQueuesOfA) + "maximum-am-resource-percent", "0.8");
cs.reinitialize(csConf, mockRM.getRMContext());
a2 = (LeafQueue) cs.getQueue("root.a.a-auto.a2");
@@ -754,13 +765,13 @@ public class TestCapacitySchedulerNewQueueAutoCreation
0.8f, a2.getMaxAMResourcePerQueuePercent(), 1e-6);
- csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
+ csConf.setNonLabeledQueueWeight(new QueuePath("root.a.a-auto.a2"), 4f);
cs.reinitialize(csConf, mockRM.getRMContext());
Assert.assertEquals("weight is not explicitly set", 4f,
a2.getQueueCapacities().getWeight(), 1e-6);
csConf.setBoolean(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
- "root.a") + CapacitySchedulerConfiguration
+ aQueuePath) + CapacitySchedulerConfiguration
.AUTO_CREATE_CHILD_QUEUE_AUTO_REMOVAL_ENABLE, false);
cs.reinitialize(csConf, mockRM.getRMContext());
AbstractLeafQueue a3 = createQueue("root.a.a3");
@@ -769,9 +780,9 @@ public class TestCapacitySchedulerNewQueueAutoCreation
// Set the capacity of label TEST
csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
- "root.c") + "accessible-node-labels.TEST.capacity", "6w");
- csConf.setQueues("root", new String[]{"a", "b", "c"});
- csConf.setAutoQueueCreationV2Enabled("root.c", true);
+ cQueuePath) + "accessible-node-labels.TEST.capacity", "6w");
+ csConf.setQueues(ROOT, new String[]{"a", "b", "c"});
+ csConf.setAutoQueueCreationV2Enabled(C, true);
cs.reinitialize(csConf, mockRM.getRMContext());
AbstractLeafQueue c1 = createQueue("root.c.c1");
Assert.assertEquals("weight is not set for label TEST", 6f,
@@ -786,19 +797,19 @@ public class TestCapacitySchedulerNewQueueAutoCreation
public void testAutoCreatedQueueConfigChange() throws Exception {
startScheduler();
AbstractLeafQueue a2 = createQueue("root.a.a-auto.a2");
- csConf.setNonLabeledQueueWeight("root.a.a-auto.a2", 4f);
+ csConf.setNonLabeledQueueWeight(a2.getQueuePathObject(), 4f);
cs.reinitialize(csConf, mockRM.getRMContext());
Assert.assertEquals("weight is not explicitly set", 4f,
a2.getQueueCapacities().getWeight(), 1e-6);
a2 = (AbstractLeafQueue) cs.getQueue("root.a.a-auto.a2");
- csConf.setState("root.a.a-auto.a2", QueueState.STOPPED);
+ csConf.setState(A_A_AUTO_A2, QueueState.STOPPED);
cs.reinitialize(csConf, mockRM.getRMContext());
Assert.assertEquals("root.a.a-auto.a2 has not been stopped",
QueueState.STOPPED, a2.getState());
- csConf.setState("root.a.a-auto.a2", QueueState.RUNNING);
+ csConf.setState(A_A_AUTO_A2, QueueState.RUNNING);
cs.reinitialize(csConf, mockRM.getRMContext());
Assert.assertEquals("root.a.a-auto.a2 is not running",
QueueState.RUNNING, a2.getState());
@@ -809,9 +820,9 @@ public class TestCapacitySchedulerNewQueueAutoCreation
startScheduler();
createQueue("root.e.e1");
- csConf.setState("root.e", QueueState.STOPPED);
- csConf.setState("root.e.e1", QueueState.STOPPED);
- csConf.setState("root.a", QueueState.STOPPED);
+ csConf.setState(E, QueueState.STOPPED);
+ csConf.setState(E_E1, QueueState.STOPPED);
+ csConf.setState(A, QueueState.STOPPED);
cs.reinitialize(csConf, mockRM.getRMContext());
// Make sure the static queue is stopped
@@ -827,13 +838,13 @@ public class TestCapacitySchedulerNewQueueAutoCreation
QueueState.STOPPED);
// Make root.e state to RUNNING
- csConf.setState("root.e", QueueState.RUNNING);
+ csConf.setState(E, QueueState.RUNNING);
cs.reinitialize(csConf, mockRM.getRMContext());
Assert.assertEquals(cs.getQueue("root.e.e1").getState(),
QueueState.STOPPED);
// Make root.e.e1 state to RUNNING
- csConf.setState("root.e.e1", QueueState.RUNNING);
+ csConf.setState(E_E1, QueueState.RUNNING);
cs.reinitialize(csConf, mockRM.getRMContext());
Assert.assertEquals(cs.getQueue("root.e.e1").getState(),
QueueState.RUNNING);
@@ -959,7 +970,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
startScheduler();
// Test for disabled auto deletion
csConf.setAutoExpiredDeletionEnabled(
- "root.a.a1-auto.a2-auto", false);
+ A_A1_AUTO_A2_AUTO, false);
csConf.setBoolean(
YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
csConf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
@@ -1026,7 +1037,7 @@ public class TestCapacitySchedulerNewQueueAutoCreation
// Enabled now
// The auto deletion will work.
csConf.setAutoExpiredDeletionEnabled(
- "root.a.a1-auto.a2-auto", true);
+ A_A1_AUTO_A2_AUTO, true);
cs.reinitialize(csConf, mockRM.getRMContext());
// Wait the a2 deleted.
@@ -1171,12 +1182,12 @@ public class TestCapacitySchedulerNewQueueAutoCreation
getQueue("root.a.a-auto");
Assert.assertTrue(aAuto.isDynamicQueue());
- csConf.setState("root.a", QueueState.STOPPED);
+ csConf.setState(A, QueueState.STOPPED);
cs.reinitialize(csConf, mockRM.getRMContext());
aAuto = (AbstractCSQueue) cs.
getQueue("root.a.a-auto");
Assert.assertEquals("root.a.a-auto is not in STOPPED state", QueueState.STOPPED, aAuto.getState());
- csConf.setQueues("root", new String[]{"b"});
+ csConf.setQueues(ROOT, new String[]{"b"});
cs.reinitialize(csConf, mockRM.getRMContext());
CSQueue aAutoNew = cs.getQueue("root.a.a-auto");
Assert.assertNull(aAutoNew);
@@ -1186,24 +1197,24 @@ public class TestCapacitySchedulerNewQueueAutoCreation
Assert.assertNotNull(aAutoNew);
// Validate static grandparent deletion
- csConf.setQueues("root", new String[]{"a", "b"});
- csConf.setQueues("root.a", new String[]{"a1"});
- csConf.setAutoQueueCreationV2Enabled("root.a.a1", true);
+ csConf.setQueues(ROOT, new String[]{"a", "b"});
+ csConf.setQueues(A, new String[]{"a1"});
+ csConf.setAutoQueueCreationV2Enabled(A1, true);
cs.reinitialize(csConf, mockRM.getRMContext());
createQueue("root.a.a1.a1-auto");
CSQueue a1Auto = cs.getQueue("root.a.a1.a1-auto");
Assert.assertNotNull("a1-auto should exist", a1Auto);
- csConf.setQueues("root", new String[]{"b"});
+ csConf.setQueues(ROOT, new String[]{"b"});
cs.reinitialize(csConf, mockRM.getRMContext());
a1Auto = cs.getQueue("root.a.a1.a1-auto");
Assert.assertNull("a1-auto has no parent and should not exist", a1Auto);
// Validate dynamic parent deletion
- csConf.setState("root.b", QueueState.STOPPED);
+ csConf.setState(B, QueueState.STOPPED);
cs.reinitialize(csConf, mockRM.getRMContext());
- csConf.setAutoQueueCreationV2Enabled("root.b", true);
+ csConf.setAutoQueueCreationV2Enabled(B, true);
cs.reinitialize(csConf, mockRM.getRMContext());
createQueue("root.b.b-auto-parent.b-auto-leaf");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java
index ccbea250f5f1..d7083b3ba0c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors.java
@@ -90,14 +90,14 @@ public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors
// root
// - a 25%
// a and root has AQCv2 enabled
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"});
+ csConf.setQueues(ROOT, new String[]{"a"});
csConf.setCapacity(A, 25f);
- csConf.setAutoQueueCreationV2Enabled(CapacitySchedulerConfiguration.ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
csConf.setAutoQueueCreationV2Enabled(A, true);
// Set up dynamic queue templates
- csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"), "6.25");
+ csConf.set(getTemplateKey(ROOT, "capacity"), "6.25");
csConf.set(getLeafTemplateKey(A, "capacity"), "[memory=25%, vcores=50%]");
}
@@ -105,14 +105,14 @@ public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors
// root
// - a [memory=8000, vcores=8]
// a and root has AQCv2 enabled
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a"});
+ csConf.setQueues(ROOT, new String[]{"a"});
csConf.setCapacity(A, "[memory=8000, vcores=8]");
- csConf.setAutoQueueCreationV2Enabled(CapacitySchedulerConfiguration.ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
csConf.setAutoQueueCreationV2Enabled(A, true);
// Set up dynamic queue templates
- csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"),
+ csConf.set(getTemplateKey(ROOT, "capacity"),
"[memory=2000mb, vcores=2]");
csConf.set(getLeafTemplateKey(A, "capacity"),
"[memory=2000, vcores=4]");
@@ -131,23 +131,23 @@ public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors
// - e1-auto [memory=2048, vcores=2]
// Set up static queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{"a", "b", "d"});
+ csConf.setQueues(ROOT, new String[]{"a", "b", "d"});
csConf.setCapacityVector(A, NL, "[memory=10%, vcores=2]");
csConf.setCapacityVector(B, NL, "[memory=2000, vcores=10%]");
csConf.setCapacityVector(D, NL, "[memory=10%, vcores=10%]");
csConf.setQueues(A, new String[]{"a1"});
csConf.setCapacityVector(A1, NL, "[memory=100%, vcores=100%]");
- csConf.setAutoQueueCreationV2Enabled(CapacitySchedulerConfiguration.ROOT, true);
+ csConf.setAutoQueueCreationV2Enabled(ROOT, true);
csConf.setAutoQueueCreationV2Enabled(A, true);
csConf.setAutoQueueCreationV2Enabled(D, true);
// Set up dynamic queue templates
- csConf.set(getTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"),
+ csConf.set(getTemplateKey(ROOT, "capacity"),
"[memory=2w, vcores=5w]");
- csConf.set(getParentTemplateKey(CapacitySchedulerConfiguration.ROOT, "capacity"),
+ csConf.set(getParentTemplateKey(ROOT, "capacity"),
"[memory=2w, vcores=10]");
- csConf.set(getLeafTemplateKey(CapacitySchedulerConfiguration.ROOT + ".*", "capacity"),
- "[memory=2000, vcores=2]");
+ csConf.set(getLeafTemplateKey(new QueuePath(CapacitySchedulerConfiguration.ROOT + ".*"),
+ "capacity"), "[memory=2000, vcores=2]");
csConf.set(getLeafTemplateKey(D, "capacity"), "[memory=1000, vcores=1]");
}
@@ -185,7 +185,7 @@ public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors
}
private void validateBasicConfiguration() throws Exception {
- CSQueue a = cs.getQueue(A);
+ CSQueue a = cs.getQueue(A_PATH);
Assert.assertEquals(8 / 32f, a.getAbsoluteCapacity(), EPSILON);
Assert.assertEquals(-1f, a.getQueueCapacities().getWeight(), EPSILON);
Assert.assertEquals(8000,
@@ -316,18 +316,18 @@ public class TestCapacitySchedulerNewQueueAutoCreationWithCapacityVectors
return autoQueueHandler.createQueue(new QueuePath(queuePath));
}
- private String getTemplateKey(String queuePath, String entryKey) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ private String getTemplateKey(QueuePath queuePath, String entryKey) {
+ return QueuePrefixes.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX + entryKey;
}
- private String getParentTemplateKey(String queuePath, String entryKey) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ private String getParentTemplateKey(QueuePath queuePath, String entryKey) {
+ return QueuePrefixes.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_PARENT_TEMPLATE_PREFIX + entryKey;
}
- private String getLeafTemplateKey(String queuePath, String entryKey) {
- return CapacitySchedulerConfiguration.getQueuePrefix(queuePath)
+ private String getLeafTemplateKey(QueuePath queuePath, String entryKey) {
+ return QueuePrefixes.getQueuePrefix(queuePath)
+ AutoCreatedQueueTemplate.AUTO_QUEUE_LEAF_TEMPLATE_PREFIX + entryKey;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index 5776fea61915..90a563245a18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -62,6 +62,7 @@ public class TestCapacitySchedulerNodeLabelUpdate {
private final int GB = 1024;
private YarnConfiguration conf;
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
RMNodeLabelsManager mgr;
@@ -79,17 +80,18 @@ public class TestCapacitySchedulerNodeLabelUpdate {
new CapacitySchedulerConfiguration(config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
-
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
- conf.setCapacity(A, 100);
- conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z"));
- conf.setCapacityByLabel(A, "x", 100);
- conf.setCapacityByLabel(A, "y", 100);
- conf.setCapacityByLabel(A, "z", 100);
+ conf.setQueues(ROOT, new String[] {"a"});
+ conf.setCapacityByLabel(ROOT, "x", 100);
+ conf.setCapacityByLabel(ROOT, "y", 100);
+ conf.setCapacityByLabel(ROOT, "z", 100);
+
+ final String aPath = CapacitySchedulerConfiguration.ROOT + ".a";
+ final QueuePath a = new QueuePath(aPath);
+ conf.setCapacity(a, 100);
+ conf.setAccessibleNodeLabels(a, ImmutableSet.of("x", "y", "z"));
+ conf.setCapacityByLabel(a, "x", 100);
+ conf.setCapacityByLabel(a, "y", 100);
+ conf.setCapacityByLabel(a, "z", 100);
return conf;
}
@@ -100,17 +102,25 @@ public class TestCapacitySchedulerNodeLabelUpdate {
new CapacitySchedulerConfiguration(config);
// Define top-level queues
- conf2.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf2.setQueues(ROOT,
new String[] {"a", "b"});
- conf2.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf2.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
-
- final String a = CapacitySchedulerConfiguration.ROOT + ".a";
- final String b = CapacitySchedulerConfiguration.ROOT + ".b";
- final String aa1 = a + ".a1";
- final String aa2 = a + ".a2";
- final String aa3 = a + ".a3";
- final String aa4 = a + ".a4";
+ conf2.setCapacityByLabel(ROOT, "x", 100);
+ conf2.setCapacityByLabel(ROOT, "y", 100);
+
+ final String aPath = CapacitySchedulerConfiguration.ROOT + ".a";
+ final String bPath = CapacitySchedulerConfiguration.ROOT + ".b";
+ final String aa1Path = aPath + ".a1";
+ final String aa2Path = aPath + ".a2";
+ final String aa3Path = aPath + ".a3";
+ final String aa4Path = aPath + ".a4";
+
+ final QueuePath a = new QueuePath(aPath);
+ final QueuePath b = new QueuePath(bPath);
+ final QueuePath aa1 = new QueuePath(aa1Path);
+ final QueuePath aa2 = new QueuePath(aa2Path);
+ final QueuePath aa3= new QueuePath(aa3Path);
+ final QueuePath aa4 = new QueuePath(aa4Path);
+
conf2.setQueues(a, new String[] {"a1", "a2", "a3", "a4"});
conf2.setCapacity(a, 50);
conf2.setCapacity(b, 50);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java
index e1cb44599ea2..be1645f27f44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodes.java
@@ -194,8 +194,8 @@ public class TestCapacitySchedulerNodes {
CapacityScheduler cs = new CapacityScheduler();
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
- conf.setDefaultNodeLabelExpression("root.a", " x");
- conf.setDefaultNodeLabelExpression("root.b", " y ");
+ conf.setDefaultNodeLabelExpression(new QueuePath("root.a"), " x");
+ conf.setDefaultNodeLabelExpression(new QueuePath("root.b"), " y ");
cs.setConf(new YarnConfiguration());
cs.setRMContext(resourceManager.getRMContext());
cs.init(conf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java
index 27eb3ac7bfd5..4a2a10f6032e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerOvercommit.java
@@ -29,6 +29,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerOver
*/
public class TestCapacitySchedulerOvercommit extends TestSchedulerOvercommit {
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT
+ + ".default");
@Override
protected Configuration getConfiguration() {
Configuration conf = super.getConfiguration();
@@ -38,14 +41,10 @@ public class TestCapacitySchedulerOvercommit extends TestSchedulerOvercommit {
// Remove limits on AMs to allow multiple applications running
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration(conf);
- csConf.setMaximumApplicationMasterResourcePerQueuePercent(
- CapacitySchedulerConfiguration.ROOT, 100.0f);
- csConf.setMaximumAMResourcePercentPerPartition(
- CapacitySchedulerConfiguration.ROOT, "", 100.0f);
- csConf.setMaximumApplicationMasterResourcePerQueuePercent(
- CapacitySchedulerConfiguration.ROOT + ".default", 100.0f);
- csConf.setMaximumAMResourcePercentPerPartition(
- CapacitySchedulerConfiguration.ROOT + ".default", "", 100.0f);
+ csConf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f);
+ csConf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f);
+ csConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT, 100.0f);
+ csConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 100.0f);
return csConf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java
index b8209a54952e..24038ebeb8f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerPerf.java
@@ -69,6 +69,9 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class TestCapacitySchedulerPerf {
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath DEFAULT = new QueuePath(CapacitySchedulerConfiguration.ROOT
+ + ".default");
private final int GB = 1024;
private String getResourceName(int idx) {
@@ -448,16 +451,16 @@ public class TestCapacitySchedulerPerf {
CapacitySchedulerConfiguration csconf =
new CapacitySchedulerConfiguration();
csconf.setResourceComparator(DominantResourceCalculator.class);
- csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f);
- csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f);
- csconf.setCapacity("root.default", 0.0f);
+ csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f);
+ csconf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f);
+ csconf.setCapacity(DEFAULT, 0.0f);
csconf.setOffSwitchPerHeartbeatLimit(numQueues);
float capacity = 100.0f / numQueues;
String[] subQueues = new String[numQueues];
for (int i = 0; i < numQueues; i++) {
String queueName = String.format("%03d", i);
- String queuePath = "root." + queueName;
+ QueuePath queuePath = new QueuePath("root." + queueName);
subQueues[i] = queueName;
csconf.setMaximumApplicationMasterResourcePerQueuePercent(
queuePath, 100.0f);
@@ -467,7 +470,7 @@ public class TestCapacitySchedulerPerf {
csconf.setMaximumCapacity(queuePath, 100.0f);
}
- csconf.setQueues("root", subQueues);
+ csconf.setQueues(ROOT, subQueues);
return csconf;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
index bb5b790fe2fc..841b8f9193fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
@@ -39,13 +39,11 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
protected Configuration createConfiguration() {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {
+ csConf.setQueues(ROOT, new String[] {
QUEUEA, QUEUEB });
- setQueueCapacity(csConf, 50,
- CapacitySchedulerConfiguration.ROOT + "." + QUEUEA);
- setQueueCapacity(csConf, 50,
- CapacitySchedulerConfiguration.ROOT + "." + QUEUEB);
+ setQueueCapacity(csConf, 50, A_QUEUE_PATH);
+ setQueueCapacity(csConf, 50, B_QUEUE_PATH);
Map<QueueACL, AccessControlList> aclsOnQueueA =
new HashMap<QueueACL, AccessControlList>();
@@ -54,8 +52,7 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
AccessControlList adminACLonQueueA = new AccessControlList(QUEUE_A_ADMIN);
aclsOnQueueA.put(QueueACL.SUBMIT_APPLICATIONS, submitACLonQueueA);
aclsOnQueueA.put(QueueACL.ADMINISTER_QUEUE, adminACLonQueueA);
- csConf.setAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA,
- aclsOnQueueA);
+ csConf.setAcls(A_QUEUE_PATH, aclsOnQueueA);
Map<QueueACL, AccessControlList> aclsOnQueueB =
new HashMap<QueueACL, AccessControlList>();
@@ -64,8 +61,7 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
AccessControlList adminACLonQueueB = new AccessControlList(QUEUE_B_ADMIN);
aclsOnQueueB.put(QueueACL.SUBMIT_APPLICATIONS, submitACLonQueueB);
aclsOnQueueB.put(QueueACL.ADMINISTER_QUEUE, adminACLonQueueB);
- csConf.setAcls(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB,
- aclsOnQueueB);
+ csConf.setAcls(B_QUEUE_PATH, aclsOnQueueB);
Map<QueueACL, AccessControlList> aclsOnRootQueue =
new HashMap<QueueACL, AccessControlList>();
@@ -73,7 +69,7 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
AccessControlList adminACLonRoot = new AccessControlList(ROOT_ADMIN);
aclsOnRootQueue.put(QueueACL.SUBMIT_APPLICATIONS, submitACLonRoot);
aclsOnRootQueue.put(QueueACL.ADMINISTER_QUEUE, adminACLonRoot);
- csConf.setAcls(CapacitySchedulerConfiguration.ROOT, aclsOnRootQueue);
+ csConf.setAcls(ROOT, aclsOnRootQueue);
csConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
csConf.set(YarnConfiguration.RM_SCHEDULER,
@@ -110,30 +106,30 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
CapacitySchedulerConfiguration csConf =
(CapacitySchedulerConfiguration) getConf();
csConf.clear();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {QUEUED, QUEUEA, QUEUEB});
String dPath = CapacitySchedulerConfiguration.ROOT + "." + QUEUED;
String d1Path = dPath + "." + QUEUED1;
- csConf.setQueues(dPath, new String[] {QUEUED1});
- setQueueCapacity(csConf, 100, d1Path);
- setQueueCapacity(csConf, 30, CapacitySchedulerConfiguration.ROOT
- + "." + QUEUEA);
- setQueueCapacity(csConf, 50, CapacitySchedulerConfiguration.ROOT
- + "." + QUEUEB);
- setQueueCapacity(csConf, 20, dPath);
+ QueuePath dQueuePath = new QueuePath(dPath);
+ QueuePath d1QueuePath = new QueuePath(d1Path);
+
+ csConf.setQueues(dQueuePath, new String[] {QUEUED1});
+ setQueueCapacity(csConf, 100, d1QueuePath);
+ setQueueCapacity(csConf, 30, A_QUEUE_PATH);
+ setQueueCapacity(csConf, 50, B_QUEUE_PATH);
+ setQueueCapacity(csConf, 20, dQueuePath);
if (rootAcl != null) {
- setAdminAndSubmitACL(csConf, rootAcl,
- CapacitySchedulerConfiguration.ROOT);
+ setAdminAndSubmitACL(csConf, rootAcl, ROOT);
}
if (queueDAcl != null) {
- setAdminAndSubmitACL(csConf, queueDAcl, dPath);
+ setAdminAndSubmitACL(csConf, queueDAcl, dQueuePath);
}
if (queueD1Acl != null) {
- setAdminAndSubmitACL(csConf, queueD1Acl, d1Path);
+ setAdminAndSubmitACL(csConf, queueD1Acl, d1QueuePath);
}
resourceManager.getResourceScheduler()
.reinitialize(csConf, resourceManager.getRMContext());
@@ -141,12 +137,12 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
private void setQueueCapacity(CapacitySchedulerConfiguration csConf,
- float capacity, String queuePath) {
+ float capacity, QueuePath queuePath) {
csConf.setCapacity(queuePath, capacity);
}
private void setAdminAndSubmitACL(CapacitySchedulerConfiguration csConf,
- String queueAcl, String queuePath) {
+ String queueAcl, QueuePath queuePath) {
csConf.setAcl(queuePath, QueueACL.ADMINISTER_QUEUE, queueAcl);
csConf.setAcl(queuePath, QueueACL.SUBMIT_APPLICATIONS, queueAcl);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java
index 77c37d78478c..2f445e636c8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueues.java
@@ -50,6 +50,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A2;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.A1_B1;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B1_CAPACITY;
@@ -57,6 +58,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B2_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.checkQueueStructureCapacities;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ExpectedCapacities;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.findQueue;
@@ -129,9 +131,9 @@ public class TestCapacitySchedulerQueues {
cs.init(conf);
cs.start();
- conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[]{"b1"});
- conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
- conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
+ conf.setQueues(A1, new String[]{"b1"});
+ conf.setCapacity(A1_B1, 100.0f);
+ conf.setUserLimitFactor(A1_B1, 100.0f);
cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf),
@@ -169,7 +171,8 @@ public class TestCapacitySchedulerQueues {
checkQueueStructureCapacities(cs);
// Add a new queue b4
- final String b4 = B + ".b4";
+ final String b4Path = B + ".b4";
+ final QueuePath b4 = new QueuePath(b4Path);
final float b4Capacity = 10;
final float modifiedB3Capacity = B3_CAPACITY - b4Capacity;
@@ -187,15 +190,15 @@ public class TestCapacitySchedulerQueues {
final float capB = 20f / 100.0f;
Map<String, ExpectedCapacities> expectedCapacities =
getDefaultCapacities(capA, capB);
- expectedCapacities.put(B3,
+ expectedCapacities.put(B3.getFullPath(),
new ExpectedCapacities(modifiedB3Capacity / 100.0f, capB));
- expectedCapacities.put(b4, new ExpectedCapacities(b4Capacity / 100.0f, capB));
+ expectedCapacities.put(b4Path, new ExpectedCapacities(b4Capacity / 100.0f, capB));
checkQueueStructureCapacities(cs, expectedCapacities);
// Verify parent for B4
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueB = findQueue(rootQueue, B);
- CSQueue queueB4 = findQueue(queueB, b4);
+ CSQueue queueB = findQueue(rootQueue, B.getFullPath());
+ CSQueue queueB4 = findQueue(queueB, b4Path);
assertEquals(queueB, queueB4.getParent());
} finally {
@@ -226,8 +229,8 @@ public class TestCapacitySchedulerQueues {
ResourceUtils.fetchMaximumAllocationFromConfig(conf).getMemorySize());
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueA = findQueue(rootQueue, A);
- CSQueue queueA1 = findQueue(queueA, A1);
+ CSQueue queueA = findQueue(rootQueue, A.getFullPath());
+ CSQueue queueA1 = findQueue(queueA, A1.getFullPath());
assertEquals("queue max allocation", ((LeafQueue) queueA1)
.getMaximumAllocation().getMemorySize(), 8192);
@@ -309,8 +312,8 @@ public class TestCapacitySchedulerQueues {
checkQueueStructureCapacities(cs);
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueA = findQueue(rootQueue, A);
- CSQueue queueA1 = findQueue(queueA, A1);
+ CSQueue queueA = findQueue(rootQueue, A.getFullPath());
+ CSQueue queueA1 = findQueue(queueA, A1.getFullPath());
assertEquals("max capability MB in CS",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
@@ -430,11 +433,11 @@ public class TestCapacitySchedulerQueues {
cs.getMaximumResourceCapability().getVirtualCores());
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueA = findQueue(rootQueue, A);
- CSQueue queueB = findQueue(rootQueue, B);
- CSQueue queueA1 = findQueue(queueA, A1);
- CSQueue queueA2 = findQueue(queueA, A2);
- CSQueue queueB2 = findQueue(queueB, B2);
+ CSQueue queueA = findQueue(rootQueue, A.getFullPath());
+ CSQueue queueB = findQueue(rootQueue, B.getFullPath());
+ CSQueue queueA1 = findQueue(queueA, A1.getFullPath());
+ CSQueue queueA2 = findQueue(queueA, A2.getFullPath());
+ CSQueue queueB2 = findQueue(queueB, B2.getFullPath());
assertEquals("queue A1 max allocation MB", 4096,
queueA1.getMaximumAllocation().getMemorySize());
@@ -520,8 +523,8 @@ public class TestCapacitySchedulerQueues {
+ " a queue WITHOUT running apps");
}
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueB = findQueue(rootQueue, B);
- CSQueue queueB3 = findQueue(queueB, B1);
+ CSQueue queueB = findQueue(rootQueue, B.getFullPath());
+ CSQueue queueB3 = findQueue(queueB, B1.getFullPath());
assertNull("Refresh needs to support delete of leaf queue ", queueB3);
// reset back to default configuration for testing parent queue delete
@@ -571,7 +574,7 @@ public class TestCapacitySchedulerQueues {
+ " a queue without running apps");
}
rootQueue = cs.getRootQueue();
- queueB = findQueue(rootQueue, B);
+ queueB = findQueue(rootQueue, B.getFullPath());
String message =
"Refresh needs to support delete of Parent queue and its children.";
assertNull(message, queueB);
@@ -640,7 +643,7 @@ public class TestCapacitySchedulerQueues {
+ " all children of a parent queue(without running apps).");
}
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueB = findQueue(rootQueue, B);
+ CSQueue queueB = findQueue(rootQueue, B.getFullPath());
assertNotNull("Parent Queue B should not be deleted", queueB);
Assert.assertTrue("As Queue'B children are not deleted",
queueB instanceof LeafQueue);
@@ -724,7 +727,7 @@ public class TestCapacitySchedulerQueues {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
// Test the child queue overrides
- setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+ setMaxAllocation(conf, ROOT,
"memory-mb=4096,vcores=2");
setMaxAllocation(conf, A1, "memory-mb=6144,vcores=2");
setMaxAllocation(conf, B, "memory-mb=5120, vcores=2");
@@ -736,12 +739,12 @@ public class TestCapacitySchedulerQueues {
checkQueueStructureCapacities(cs);
CSQueue rootQueue = cs.getRootQueue();
- CSQueue queueA = findQueue(rootQueue, A);
- CSQueue queueB = findQueue(rootQueue, B);
- CSQueue queueA1 = findQueue(queueA, A1);
- CSQueue queueA2 = findQueue(queueA, A2);
- CSQueue queueB1 = findQueue(queueB, B1);
- CSQueue queueB2 = findQueue(queueB, B2);
+ CSQueue queueA = findQueue(rootQueue, A.getFullPath());
+ CSQueue queueB = findQueue(rootQueue, B.getFullPath());
+ CSQueue queueA1 = findQueue(queueA, A1.getFullPath());
+ CSQueue queueA2 = findQueue(queueA, A2.getFullPath());
+ CSQueue queueB1 = findQueue(queueB, B1.getFullPath());
+ CSQueue queueB2 = findQueue(queueB, B2.getFullPath());
assertEquals("max capability MB in CS",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
@@ -771,7 +774,7 @@ public class TestCapacitySchedulerQueues {
unsetMaxAllocation(conf, A1);
unsetMaxAllocation(conf, B);
unsetMaxAllocation(conf, B1);
- setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+ setMaxAllocation(conf, ROOT,
"memory-mb=6144,vcores=2");
setMaxAllocation(conf, A, "memory-mb=8192,vcores=2");
@@ -797,7 +800,7 @@ public class TestCapacitySchedulerQueues {
queueB1.getMaximumAllocation().getVirtualCores());
// Test the default
- unsetMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT);
+ unsetMaxAllocation(conf, ROOT);
unsetMaxAllocation(conf, A);
unsetMaxAllocation(conf, A1);
cs.reinitialize(conf, rm.getRMContext());
@@ -845,7 +848,7 @@ public class TestCapacitySchedulerQueues {
cs.reinitialize(conf, rm.getRMContext());
checkQueueStructureCapacities(cs);
- setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+ setMaxAllocation(conf, ROOT,
"memory-mb=" + largerMem + ",vcores=2");
try {
cs.reinitialize(conf, rm.getRMContext());
@@ -855,7 +858,7 @@ public class TestCapacitySchedulerQueues {
e.getCause().getMessage().contains("maximum allocation"));
}
- setMaxAllocation(conf, CapacitySchedulerConfiguration.ROOT,
+ setMaxAllocation(conf, ROOT,
"memory-mb=4096,vcores=2");
setMaxAllocation(conf, A, "memory-mb=6144,vcores=2");
setMaxAllocation(conf, A1, "memory-mb=" + largerMem + ",vcores=2");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index ce79528c2599..322b8f5b6cb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -62,6 +62,17 @@ public class TestCapacitySchedulerSurgicalPreemption
extends CapacitySchedulerPreemptionTestBase {
private static final int NUM_NM = 5;
+
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".a");
+ private static final QueuePath B = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".b");
+ private static final QueuePath C = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".c");
+ private static final QueuePath D = new QueuePath(CapacitySchedulerConfiguration.ROOT +
+ ".d");
+
@Override
@Before
public void setUp() throws Exception {
@@ -350,11 +361,11 @@ public class TestCapacitySchedulerSurgicalPreemption
*/
conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true);
conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000);
- conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueueOrderingPolicy(ROOT,
CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
// Queue c has higher priority than a/b
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1);
+ conf.setQueuePriority(C, 1);
MockRM rm1 = new MockRM(conf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -470,12 +481,12 @@ public class TestCapacitySchedulerSurgicalPreemption
*/
conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true);
conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000);
- conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueueOrderingPolicy(ROOT,
CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
conf.setPUOrderingPolicyUnderUtilizedPreemptionMoveReservation(true);
// Queue c has higher priority than a/b
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1);
+ conf.setQueuePriority(C, 1);
MockRM rm1 = new MockRM(conf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -610,11 +621,11 @@ public class TestCapacitySchedulerSurgicalPreemption
*/
conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true);
conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000);
- conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueueOrderingPolicy(ROOT,
CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
// Queue c has higher priority than a/b
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1);
+ conf.setQueuePriority(C, 1);
MockRM rm1 = new MockRM(conf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -750,11 +761,11 @@ public class TestCapacitySchedulerSurgicalPreemption
*/
// A/B has higher priority
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a" , 1);
- conf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2);
- conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 45f);
- conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b", 45f);
- conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c", 10f);
+ conf.setQueuePriority(A, 1);
+ conf.setQueuePriority(B, 2);
+ conf.setCapacity(A, 45f);
+ conf.setCapacity(B, 45f);
+ conf.setCapacity(C, 10f);
testPriorityPreemptionFromHighestPriorityQueueAndOldestContainer(new
String[] {"a", "b", "c"}, new String[] {"user", "user", "user"});
@@ -770,7 +781,7 @@ public class TestCapacitySchedulerSurgicalPreemption
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
conf.setPUOrderingPolicyUnderUtilizedPreemptionEnabled(true);
conf.setPUOrderingPolicyUnderUtilizedPreemptionDelay(1000);
- conf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueueOrderingPolicy(ROOT,
CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
MockRM rm1 = new MockRM(conf) {
@@ -987,12 +998,14 @@ public class TestCapacitySchedulerSurgicalPreemption
private void initializeConfProperties(CapacitySchedulerConfiguration conf)
throws IOException {
+ QueuePath aQueuePath = new QueuePath("root.A");
+ QueuePath bQueuePath = new QueuePath("root.B");
- conf.setQueues("root", new String[] {"A", "B"});
- conf.setCapacity("root.A", 50);
- conf.setCapacity("root.B", 50);
- conf.setQueuePriority("root.A", 1);
- conf.setQueuePriority("root.B", 2);
+ conf.setQueues(ROOT, new String[] {"A", "B"});
+ conf.setCapacity(aQueuePath, 50);
+ conf.setCapacity(bQueuePath, 50);
+ conf.setQueuePriority(aQueuePath, 1);
+ conf.setQueuePriority(bQueuePath, 2);
conf.set(PREFIX + "root.ordering-policy", "priority-utilization");
conf.set(PREFIX + "ordering-policy.priority-utilization.underutilized-preemption.enabled", "true");
@@ -1171,11 +1184,11 @@ public class TestCapacitySchedulerSurgicalPreemption
this.conf);
conf.setLong(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
1024 * 21);
- conf.setQueues("root", new String[] { "a", "b" });
- conf.setCapacity("root.a", 50);
- conf.setUserLimitFactor("root.a", 100);
- conf.setCapacity("root.b", 50);
- conf.setUserLimitFactor("root.b", 100);
+ conf.setQueues(ROOT, new String[] {"a", "b"});
+ conf.setCapacity(A, 50);
+ conf.setUserLimitFactor(A, 100);
+ conf.setCapacity(B, 50);
+ conf.setUserLimitFactor(B, 100);
MockRM rm1 = new MockRM(conf);
rm1.getRMContext().setNodeLabelManager(mgr);
rm1.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
index 115813a01717..216ec0f9b926 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWeightMode.java
@@ -52,11 +52,19 @@ import java.util.ArrayList;
import java.util.Set;
public class TestCapacitySchedulerWeightMode {
- private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
- private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
- private static final String A1 = A + ".a1";
- private static final String B1 = B + ".b1";
- private static final String B2 = B + ".b2";
+ private static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default";
+ private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String A1_PATH = A_PATH + ".a1";
+ private static final String B1_PATH = B_PATH + ".b1";
+ private static final String B2_PATH = B_PATH + ".b2";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH);
+ private static final QueuePath A = new QueuePath(A_PATH);
+ private static final QueuePath B = new QueuePath(B_PATH);
+ private static final QueuePath A1 = new QueuePath(A1_PATH);
+ private static final QueuePath B1 = new QueuePath(B1_PATH);
+ private static final QueuePath B2 = new QueuePath(B2_PATH);
private YarnConfiguration conf;
@@ -82,7 +90,7 @@ public class TestCapacitySchedulerWeightMode {
config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] { "a"});
conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 100f);
@@ -114,11 +122,11 @@ public class TestCapacitySchedulerWeightMode {
config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] { "a", "b" });
- conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
- conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ conf.setLabeledQueueWeight(ROOT, "x", 100);
+ conf.setLabeledQueueWeight(ROOT, "y", 100);
+ conf.setLabeledQueueWeight(ROOT, "z", 100);
conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A, 10);
@@ -173,11 +181,11 @@ public class TestCapacitySchedulerWeightMode {
config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] { "a", "b" });
- conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
- conf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ conf.setLabeledQueueWeight(ROOT, "x", 100);
+ conf.setLabeledQueueWeight(ROOT, "y", 100);
+ conf.setLabeledQueueWeight(ROOT, "z", 100);
conf.setLabeledQueueWeight(A, RMNodeLabelsManager.NO_LABEL, 1);
conf.setMaximumCapacity(A, 10);
@@ -232,11 +240,11 @@ public class TestCapacitySchedulerWeightMode {
config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ conf.setQueues(ROOT,
new String[] { "a", "b" });
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ conf.setCapacityByLabel(ROOT, "x", 100);
+ conf.setCapacityByLabel(ROOT, "y", 100);
+ conf.setCapacityByLabel(ROOT, "z", 100);
conf.setCapacityByLabel(A, RMNodeLabelsManager.NO_LABEL, 10);
conf.setMaximumCapacity(A, 10);
@@ -322,15 +330,15 @@ public class TestCapacitySchedulerWeightMode {
rm.start();
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String capacityOrWeightString = ((ParentQueue) cs.getQueue(A))
+ String capacityOrWeightString = ((ParentQueue) cs.getQueue(A.getFullPath()))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
- capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath()))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
- capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath()))
.getExtendedCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
}
@@ -348,15 +356,15 @@ public class TestCapacitySchedulerWeightMode {
rm.start();
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- String capacityOrWeightString = ((ParentQueue) cs.getQueue(A))
+ String capacityOrWeightString = ((ParentQueue) cs.getQueue(A.getFullPath()))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, true);
- capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath()))
.getCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, false);
- capacityOrWeightString = ((LeafQueue) cs.getQueue(A1))
+ capacityOrWeightString = ((LeafQueue) cs.getQueue(A1.getFullPath()))
.getExtendedCapacityOrWeightString();
validateCapacityOrWeightString(capacityOrWeightString, false);
}
@@ -379,7 +387,7 @@ public class TestCapacitySchedulerWeightMode {
CapacityScheduler cs =
(CapacityScheduler) rm.getRMContext().getScheduler();
- LeafQueue b1 = (LeafQueue) cs.getQueue(B1);
+ LeafQueue b1 = (LeafQueue) cs.getQueue(B1.getFullPath());
Assert.assertNotNull(b1);
Assert.assertTrue(b1.getAccessibleNodeLabels().isEmpty());
@@ -422,11 +430,11 @@ public class TestCapacitySchedulerWeightMode {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(
conf);
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {"a", "b", "default"});
- csConf.setNonLabeledQueueWeight("root.a", 1);
- csConf.setNonLabeledQueueWeight("root.b", 2);
- csConf.setNonLabeledQueueWeight("root.default", 3);
+ csConf.setNonLabeledQueueWeight(A, 1);
+ csConf.setNonLabeledQueueWeight(B, 2);
+ csConf.setNonLabeledQueueWeight(DEFAULT, 3);
// Check queue info capacity
CapacityScheduler cs =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index 98cffda51cc8..deebe3d59be4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Assert;
+
import static org.junit.Assert.assertEquals;
import org.junit.Test;
@@ -75,9 +76,13 @@ import java.util.Map;
*/
public class TestCapacitySchedulerWithMultiResourceTypes {
private static String RESOURCE_1 = "res1";
-
- private static final String A_QUEUE = CapacitySchedulerConfiguration.ROOT + ".a";
- private static final String B_QUEUE = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default";
+ private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH);
+ private static final QueuePath A = new QueuePath(A_PATH);
+ private static final QueuePath B = new QueuePath(B_PATH);
private static float A_CAPACITY = 50.0f;
private static float B_CAPACITY = 50.0f;
@@ -116,11 +121,11 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
CapacitySchedulerConfiguration csconf =
new CapacitySchedulerConfiguration();
- csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f);
- csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f);
- csconf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ csconf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f);
+ csconf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f);
+ csconf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT,
100.0f);
- csconf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f);
+ csconf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 100.0f);
csconf.setResourceComparator(DominantResourceCalculator.class);
csconf.set(YarnConfiguration.RESOURCE_TYPES, RESOURCE_1);
csconf.setInt(YarnConfiguration.RESOURCE_TYPES + "." + RESOURCE_1
@@ -271,23 +276,22 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f);
- csConf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f);
- csConf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
+ csConf.setMaximumApplicationMasterResourcePerQueuePercent(ROOT, 100.0f);
+ csConf.setMaximumAMResourcePercentPerPartition(ROOT, "", 100.0f);
+ csConf.setMaximumApplicationMasterResourcePerQueuePercent(DEFAULT,
100.0f);
- csConf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f);
+ csConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 100.0f);
csConf.setResourceComparator(DominantResourceCalculator.class);
csConf.set(YarnConfiguration.RESOURCE_TYPES, ResourceInformation.GPU_URI);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {"a", "b"});
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
// Set each queue to consider 50% each.
- csConf.setCapacity(A_QUEUE, A_CAPACITY);
- csConf.setCapacity(B_QUEUE, B_CAPACITY);
- csConf.setMaximumCapacity(A_QUEUE, 100.0f);
- csConf.setUserLimitFactor(A_QUEUE, 2);
+ csConf.setCapacity(A, A_CAPACITY);
+ csConf.setCapacity(B, B_CAPACITY);
+ csConf.setMaximumCapacity(A, 100.0f);
+ csConf.setUserLimitFactor(A, 2);
YarnConfiguration conf = new YarnConfiguration(csConf);
// Don't reset resource types since we have already configured resource
@@ -411,14 +415,13 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
csconf.setResourceComparator(DominantResourceCalculator.class);
// Define top-level queues
- csconf.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {"a", "b"});
+ csconf.setQueues(ROOT, new String[] {"a", "b"});
// Set each queue to contain 50% each.
- csconf.setCapacity(A_QUEUE, A_CAPACITY);
- csconf.setCapacity(B_QUEUE, B_CAPACITY);
- csconf.setMaximumCapacity(A_QUEUE, 100.0f);
- csconf.setUserLimitFactor(A_QUEUE, 2);
+ csconf.setCapacity(A, A_CAPACITY);
+ csconf.setCapacity(B, B_CAPACITY);
+ csconf.setMaximumCapacity(A, 100.0f);
+ csconf.setUserLimitFactor(A, 2);
YarnConfiguration yarnConf = new YarnConfiguration(csconf);
// Don't reset resource types since we have already configured resource
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java
index 07b2562a0a84..1108e76a3ec1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWorkflowPriorityMapping.java
@@ -32,6 +32,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B3_CAPACITY;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.B_CAPACITY;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueHelpers.ROOT;
import static org.junit.Assert.assertEquals;
@@ -67,8 +68,7 @@ public class TestCapacitySchedulerWorkflowPriorityMapping {
private static void setWorkFlowPriorityMappings(
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(
- CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+ conf.setQueues(ROOT, new String[] {"a", "b"});
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);
@@ -84,9 +84,9 @@ public class TestCapacitySchedulerWorkflowPriorityMapping {
conf.setCapacity(B3, B3_CAPACITY);
List<WorkflowPriorityMapping> mappings = Arrays.asList(
- new WorkflowPriorityMapping("workflow1", B, Priority.newInstance(2)),
- new WorkflowPriorityMapping("workflow2", A1, Priority.newInstance(3)),
- new WorkflowPriorityMapping("Workflow3", A, Priority.newInstance(4)));
+ new WorkflowPriorityMapping("workflow1", B.getFullPath(), Priority.newInstance(2)),
+ new WorkflowPriorityMapping("workflow2", A1.getFullPath(), Priority.newInstance(3)),
+ new WorkflowPriorityMapping("Workflow3", A.getFullPath(), Priority.newInstance(4)));
conf.setWorkflowPriorityMappings(mappings);
}
@@ -108,9 +108,9 @@ public class TestCapacitySchedulerWorkflowPriorityMapping {
cs.start();
Map<String, Object> expected = ImmutableMap.of(
- A, ImmutableMap.of("workflow3", Priority.newInstance(4)),
- B, ImmutableMap.of("workflow1", Priority.newInstance(2)),
- A1, ImmutableMap.of("workflow2", Priority.newInstance(3)));
+ A.getFullPath(), ImmutableMap.of("workflow3", Priority.newInstance(4)),
+ B.getFullPath(), ImmutableMap.of("workflow1", Priority.newInstance(2)),
+ A1.getFullPath(), ImmutableMap.of("workflow2", Priority.newInstance(3)));
assertEquals(expected, cs.getWorkflowPriorityMappingsManager()
.getWorkflowPriorityMappings());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 52bafa792ef9..c03392da1101 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -204,19 +204,20 @@ public class TestChildQueueOrder {
private void setupSortedQueues(CapacitySchedulerConfiguration conf) {
// Define queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
+ final QueuePath root = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ csConf.setQueues(root, new String[] {A, B, C, D});
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
- conf.setCapacity(Q_A, 25);
+ conf.setCapacity(new QueuePath(Q_A), 25);
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
- conf.setCapacity(Q_B, 25);
+ conf.setCapacity(new QueuePath(Q_B), 25);
final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
- conf.setCapacity(Q_C, 25);
+ conf.setCapacity(new QueuePath(Q_C), 25);
final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
- conf.setCapacity(Q_D, 25);
+ conf.setCapacity(new QueuePath(Q_D), 25);
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 7e00c218aa1f..df1db68f7d73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -78,7 +78,19 @@ public class TestContainerAllocation {
private static final Logger LOG = LoggerFactory
.getLogger(TestContainerAllocation.class);
-
+ public static final String DEFAULT_PATH = CapacitySchedulerConfiguration.ROOT + ".default";
+ public static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ public static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ public static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c";
+ public static final String C1_PATH = C_PATH + ".c1";
+ public static final String C2_PATH = C_PATH + ".c2";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath DEFAULT = new QueuePath(DEFAULT_PATH);
+ private static final QueuePath A = new QueuePath(A_PATH);
+ private static final QueuePath B = new QueuePath(B_PATH);
+ private static final QueuePath C = new QueuePath(C_PATH);
+ private static final QueuePath C1 = new QueuePath(C1_PATH);
+ private static final QueuePath C2 = new QueuePath(C2_PATH);
private final int GB = 1024;
private YarnConfiguration conf;
@@ -724,7 +736,7 @@ public class TestContainerAllocation {
(CapacitySchedulerConfiguration) TestUtils
.getConfigurationWithMultipleQueues(conf);
// Set maximum capacity of A to 10
- newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 10);
+ newConf.setMaximumCapacity(A, 10);
MockRM rm1 = new MockRM(newConf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -789,10 +801,8 @@ public class TestContainerAllocation {
// Set maximum capacity of A to 10
CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration(
conf);
- newConf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".default",
- 0.5f);
- newConf.setMaximumAMResourcePercentPerPartition(
- CapacitySchedulerConfiguration.ROOT + ".default", "", 1.0f);
+ newConf.setUserLimitFactor(DEFAULT, 0.5f);
+ newConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 1.0f);
MockRM rm1 = new MockRM(newConf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -892,14 +902,14 @@ public class TestContainerAllocation {
.getConfigurationWithMultipleQueues(conf);
// Set ordering policy
- newConf.setQueueOrderingPolicy(CapacitySchedulerConfiguration.ROOT,
+ newConf.setQueueOrderingPolicy(ROOT,
CapacitySchedulerConfiguration.QUEUE_PRIORITY_UTILIZATION_ORDERING_POLICY);
// Set maximum capacity of A to 20
- newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 20);
- newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".c", 1);
- newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".b", 2);
- newConf.setQueuePriority(CapacitySchedulerConfiguration.ROOT + ".a", 3);
+ newConf.setMaximumCapacity(A, 20);
+ newConf.setQueuePriority(C, 1);
+ newConf.setQueuePriority(B, 2);
+ newConf.setQueuePriority(A, 3);
MockRM rm1 = new MockRM(newConf);
@@ -1029,7 +1039,7 @@ public class TestContainerAllocation {
// make sure an unlimited number of containers can be assigned,
// overriding the default of 100 after YARN-8896
newConf.set(MAX_ASSIGN_PER_HEARTBEAT, "-1");
- newConf.setUserLimit("root.c", 50);
+ newConf.setUserLimit(C, 50);
MockRM rm1 = new MockRM(newConf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -1092,8 +1102,7 @@ public class TestContainerAllocation {
CapacitySchedulerConfiguration newConf =
new CapacitySchedulerConfiguration(conf);
- newConf.setMaximumAMResourcePercentPerPartition(
- CapacitySchedulerConfiguration.ROOT + ".default", "", 0.2f);
+ newConf.setMaximumAMResourcePercentPerPartition(DEFAULT, "", 0.2f);
MockRM rm1 = new MockRM(newConf);
rm1.getRMContext().setNodeLabelManager(mgr);
@@ -1187,7 +1196,7 @@ public class TestContainerAllocation {
newConf
.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
// Set maximum capacity of queue "a" to 50
- newConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a", 50);
+ newConf.setMaximumCapacity(A, 50);
MockRM rm1 = new MockRM(newConf);
RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
@@ -1276,12 +1285,12 @@ public class TestContainerAllocation {
CapacitySchedulerConfiguration newConf =
(CapacitySchedulerConfiguration) TestUtils
.getConfigurationWithMultipleQueues(conf);
- newConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".c",
+ newConf.setQueues(C,
new String[] { "c1", "c2" });
- newConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c1", 10);
+ newConf.setCapacity(C1, 10);
newConf
- .setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c1", 10);
- newConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 90);
+ .setMaximumCapacity(C1, 10);
+ newConf.setCapacity(C2, 90);
newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class, ResourceCalculator.class);
@@ -1345,7 +1354,7 @@ public class TestContainerAllocation {
.getConfigurationWithMultipleQueues(conf);
newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
DominantResourceCalculator.class, ResourceCalculator.class);
- newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a")
+ newConf.set(QueuePrefixes.getQueuePrefix(A)
+ MAXIMUM_ALLOCATION_MB, "4096");
MockRM rm1 = new MockRM(newConf);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index c92e94c6cfb7..870d61539b02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -240,7 +240,7 @@ public class TestLeafQueue {
root =
CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
- ROOT,
+ ROOT.getFullPath(),
queues, queues,
TestUtils.spyHook);
queueManager.setRootQueue(root);
@@ -268,66 +268,69 @@ public class TestLeafQueue {
private static final String C1 = "c1";
private static final String D = "d";
private static final String E = "e";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A_QUEUE_PATH = ROOT.createNewLeaf(A);
private void setupQueueConfiguration(
CapacitySchedulerConfiguration conf,
- final String newRoot, boolean withNodeLabels) {
+ final String newRootName, boolean withNodeLabels) {
// Define top-level queues
- conf.setQueues(ROOT, new String[] {newRoot});
+ conf.setQueues(ROOT, new String[] {newRootName});
conf.setMaximumCapacity(ROOT, 100);
conf.setAcl(ROOT,
QueueACL.SUBMIT_APPLICATIONS, " ");
if (withNodeLabels) {
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, LABEL, 100);
- conf.setMaximumCapacityByLabel(CapacitySchedulerConfiguration.ROOT,
- LABEL, 100);
+ conf.setCapacityByLabel(ROOT, LABEL, 100);
+ conf.setMaximumCapacityByLabel(ROOT, LABEL, 100);
}
- final String Q_newRoot = ROOT + "." + newRoot;
- conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E});
- conf.setCapacity(Q_newRoot, 100);
- conf.setMaximumCapacity(Q_newRoot, 100);
- conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " ");
+ final String newRootPath = CapacitySchedulerConfiguration.ROOT + "." + newRootName;
+ final QueuePath newRoot = new QueuePath(newRootPath);
+ conf.setQueues(newRoot, new String[] {A, B, C, D, E});
+ conf.setCapacity(newRoot, 100);
+ conf.setMaximumCapacity(newRoot, 100);
+ conf.setAcl(newRoot, QueueACL.SUBMIT_APPLICATIONS, " ");
if (withNodeLabels) {
- conf.setAccessibleNodeLabels(Q_newRoot, Collections.singleton(LABEL));
- conf.setCapacityByLabel(Q_newRoot, LABEL, 100);
- conf.setMaximumCapacityByLabel(Q_newRoot, LABEL, 100);
+ conf.setAccessibleNodeLabels(newRoot, Collections.singleton(LABEL));
+ conf.setCapacityByLabel(newRoot, LABEL, 100);
+ conf.setMaximumCapacityByLabel(newRoot, LABEL, 100);
}
- final String Q_A = Q_newRoot + "." + A;
- conf.setCapacity(Q_A, 8.5f);
- conf.setMaximumCapacity(Q_A, 20);
- conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
+ final QueuePath a = new QueuePath(newRootPath, A);
+ conf.setCapacity(a, 8.5f);
+ conf.setMaximumCapacity(a, 20);
+ conf.setAcl(a, QueueACL.SUBMIT_APPLICATIONS, "*");
if (withNodeLabels) {
- conf.setAccessibleNodeLabels(Q_A, Collections.singleton(LABEL));
- conf.setCapacityByLabel(Q_A, LABEL, 100);
- conf.setMaximumCapacityByLabel(Q_A, LABEL, 100);
+ conf.setAccessibleNodeLabels(a, Collections.singleton(LABEL));
+ conf.setCapacityByLabel(a, LABEL, 100);
+ conf.setMaximumCapacityByLabel(a, LABEL, 100);
}
-
- final String Q_B = Q_newRoot + "." + B;
- conf.setCapacity(Q_B, 80);
- conf.setMaximumCapacity(Q_B, 99);
- conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
-
- final String Q_C = Q_newRoot + "." + C;
- conf.setCapacity(Q_C, 1.5f);
- conf.setMaximumCapacity(Q_C, 10);
- conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
-
- conf.setQueues(Q_C, new String[] {C1});
- final String Q_C1 = Q_C + "." + C1;
- conf.setCapacity(Q_C1, 100);
+ final QueuePath b = new QueuePath(newRootPath, B);
+ conf.setCapacity(b, 80);
+ conf.setMaximumCapacity(b, 99);
+ conf.setAcl(b, QueueACL.SUBMIT_APPLICATIONS, "*");
- final String Q_D = Q_newRoot + "." + D;
- conf.setCapacity(Q_D, 9);
- conf.setMaximumCapacity(Q_D, 11);
- conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d");
+ final String cPath = newRootPath + "." + C;
+ final QueuePath c = new QueuePath(cPath);
+ conf.setCapacity(c, 1.5f);
+ conf.setMaximumCapacity(c, 10);
+ conf.setAcl(c, QueueACL.SUBMIT_APPLICATIONS, " ");
- final String Q_E = Q_newRoot + "." + E;
- conf.setCapacity(Q_E, 1);
- conf.setMaximumCapacity(Q_E, 1);
- conf.setAcl(Q_E, QueueACL.SUBMIT_APPLICATIONS, "user_e");
+ conf.setQueues(c, new String[] {C1});
+
+ final QueuePath c1 = new QueuePath(cPath, C1);
+ conf.setCapacity(c1, 100);
+
+ final QueuePath d = new QueuePath(newRootPath, D);
+ conf.setCapacity(d, 9);
+ conf.setMaximumCapacity(d, 11);
+ conf.setAcl(d, QueueACL.SUBMIT_APPLICATIONS, "user_d");
+
+ final QueuePath e = new QueuePath(newRootPath, E);
+ conf.setCapacity(e, 1);
+ conf.setMaximumCapacity(e, 1);
+ conf.setAcl(e, QueueACL.SUBMIT_APPLICATIONS, "user_e");
}
@@ -480,7 +483,7 @@ public class TestLeafQueue {
"testPolicyRoot" + System.currentTimeMillis();
OrderingPolicy<FiCaSchedulerApp> comPol =
- testConf.<FiCaSchedulerApp>getAppOrderingPolicy(tproot);
+ testConf.<FiCaSchedulerApp>getAppOrderingPolicy(new QueuePath(tproot));
}
@@ -627,8 +630,9 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration testConf =
new CapacitySchedulerConfiguration();
- String tproot = ROOT + "." +
+ String tprootPath = ROOT + "." +
"testPolicyRoot" + System.currentTimeMillis();
+ QueuePath tproot = new QueuePath(tprootPath);
OrderingPolicy<FiCaSchedulerApp> schedOrder =
testConf.<FiCaSchedulerApp>getAppOrderingPolicy(tproot);
@@ -1093,9 +1097,9 @@ public class TestLeafQueue {
when(csContext.getClusterResource()).thenReturn(clusterResource);
// working with just one queue
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100);
- csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A,
+ csConf.setQueues(ROOT, new String[]{A});
+ csConf.setCapacity(A_QUEUE_PATH, 100);
+ csConf.setMaximumCapacity(A_QUEUE_PATH,
100);
queueContext.reinitialize();
@@ -1312,9 +1316,9 @@ public class TestLeafQueue {
when(csContext.getClusterResource()).thenReturn(clusterResource);
// working with just one queue
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 100);
- csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + "." + A,
+ csConf.setQueues(ROOT, new String[] {A});
+ csConf.setCapacity(A_QUEUE_PATH, 100);
+ csConf.setMaximumCapacity(A_QUEUE_PATH,
100);
queueContext.reinitialize();
@@ -1919,7 +1923,7 @@ public class TestLeafQueue {
// Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
// Set minimum-user-limit-percent for queue "a" in the configs.
- csConf.setUserLimit(a.getQueuePath(), 50);
+ csConf.setUserLimit(a.getQueuePathObject(), 50);
// Set weight for "user_0" to be 1.5f for the a queue in the configs.
csConf.setFloat("yarn.scheduler.capacity." + a.getQueuePath()
+ ".user-settings.user_0." + CapacitySchedulerConfiguration.USER_WEIGHT,
@@ -3232,7 +3236,7 @@ public class TestLeafQueue {
queueContext.reinitialize();
CSQueueStore newQueues = new CSQueueStore();
CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(queueContext,
- csConf, null, ROOT, newQueues, queues,
+ csConf, null, ROOT.getFullPath(), newQueues, queues,
TestUtils.spyHook);
csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot);
root.reinitialize(newRoot, cs.getClusterResource());
@@ -3671,7 +3675,7 @@ public class TestLeafQueue {
CSQueueStore newQueues = new CSQueueStore();
CSQueue newRoot =
CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
- ROOT,
+ ROOT.getFullPath(),
newQueues, queues,
TestUtils.spyHook);
queues = newQueues;
@@ -3703,7 +3707,7 @@ public class TestLeafQueue {
CSQueueStore newQueues = new CSQueueStore();
CSQueue newRoot =
CapacitySchedulerQueueManager.parseQueue(queueContext, csConf, null,
- ROOT,
+ ROOT.getFullPath(),
newQueues, queues,
TestUtils.spyHook);
csContext.getCapacitySchedulerQueueManager().setRootQueue(newRoot);
@@ -5134,19 +5138,19 @@ public class TestLeafQueue {
final String leafQueueName =
"testSetupQueueConfigsWithSpecifiedConfiguration";
+ final QueuePath leafQueuePath = new QueuePath(ROOT.getFullPath(), leafQueueName);
+ final QueuePath bQueuePath = new QueuePath(ROOT + DOT + B);
assertEquals(0, conf.size());
conf.setNodeLocalityDelay(60);
csConf.setQueues(ROOT, new String[] {leafQueueName, B});
- csConf.setCapacity(ROOT + DOT + leafQueueName, 10);
- csConf.setMaximumCapacity(ROOT + DOT + leafQueueName,
- 100);
- csConf.setUserLimitFactor(ROOT + DOT + leafQueueName, 0.1f);
+ csConf.setCapacity(leafQueuePath, 10);
+ csConf.setMaximumCapacity(leafQueuePath, 100);
+ csConf.setUserLimitFactor(leafQueuePath, 0.1f);
- csConf.setCapacity(ROOT + DOT + B, 90);
- csConf.setMaximumCapacity(ROOT + DOT + B,
- 100);
+ csConf.setCapacity(bQueuePath, 90);
+ csConf.setMaximumCapacity(bQueuePath, 100);
csConf.setNodeLocalityDelay(30);
csConf.setGlobalMaximumApplicationsPerQueue(20);
@@ -5172,9 +5176,9 @@ public class TestLeafQueue {
assertEquals(2, leafQueue.getMaxApplicationsPerUser());
//check queue configs
- conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueuePath(),
+ conf.setMaximumAMResourcePercentPerPartition(leafQueue.getQueuePathObject(),
NO_LABEL, 10);
- conf.setMaximumCapacity(leafQueue.getQueuePath(), 10);
+ conf.setMaximumCapacity(leafQueue.getQueuePathObject(), 10);
assertEquals(0.1, leafQueue.getMaxAMResourcePerQueuePercent(),
EPSILON);
@@ -5244,13 +5248,13 @@ public class TestLeafQueue {
Resources.createResource(2 * 16 * GB, 2 * 32));
conf.setCapacityByLabel(ROOT, "test", 100);
- conf.setCapacityByLabel(rootChild, "test", 100);
- conf.setCapacityByLabel(rootChild + "." + A, "test", 20);
- conf.setCapacityByLabel(rootChild + "." + B, "test", 40);
- conf.setCapacityByLabel(rootChild + "." + C, "test", 10);
- conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test", 100);
- conf.setCapacityByLabel(rootChild + "." + D, "test", 30);
- conf.setCapacityByLabel(rootChild + "." + E, "test", 0);
+ conf.setCapacityByLabel(new QueuePath(rootChild), "test", 100);
+ conf.setCapacityByLabel(new QueuePath(rootChild, A), "test", 20);
+ conf.setCapacityByLabel(new QueuePath(rootChild, B), "test", 40);
+ conf.setCapacityByLabel(new QueuePath(rootChild, C), "test", 10);
+ conf.setCapacityByLabel(QueuePath.createFromQueues(rootChild, C, C1), "test", 100);
+ conf.setCapacityByLabel(new QueuePath(rootChild, D), "test", 30);
+ conf.setCapacityByLabel(new QueuePath(rootChild, E), "test", 0);
cs.getCapacitySchedulerQueueManager().reinitConfiguredNodeLabels(conf);
cs.setMaxRunningAppsEnforcer(new CSMaxRunningAppsEnforcer(cs));
cs.reinitialize(conf, cs.getRMContext());
@@ -5261,11 +5265,11 @@ public class TestLeafQueue {
(int)(conf.getMaximumSystemApplications()
* e.getAbsoluteCapacity()), e.getMaxApplications());
- conf.setCapacityByLabel(rootChild + "." + A, "test", 10);
- conf.setCapacityByLabel(rootChild + "." + B, "test", 10);
- conf.setCapacityByLabel(rootChild + "." + C, "test", 10);
- conf.setCapacityByLabel(rootChild + "." + D, "test", 10);
- conf.setCapacityByLabel(rootChild + "." + E, "test", 60);
+ conf.setCapacityByLabel(new QueuePath(rootChild, A), "test", 10);
+ conf.setCapacityByLabel(new QueuePath(rootChild, B), "test", 10);
+ conf.setCapacityByLabel(new QueuePath(rootChild, C), "test", 10);
+ conf.setCapacityByLabel(new QueuePath(rootChild, D), "test", 10);
+ conf.setCapacityByLabel(new QueuePath(rootChild, E), "test", 60);
cs.reinitialize(conf, cs.getRMContext());
e = (LeafQueue) cs.getQueue("e");
@@ -5282,21 +5286,21 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration conf = csConf;
String rootChild = root.getChildQueues().get(0).getQueuePath();
- conf.setCapacityByLabel(rootChild, "test", 100);
- conf.setCapacityByLabel(rootChild + "." + A, "test", 20);
- conf.setCapacityByLabel(rootChild + "." + B, "test", 40);
- conf.setCapacityByLabel(rootChild + "." + C, "test", 10);
- conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test", 100);
- conf.setCapacityByLabel(rootChild + "." + D, "test", 30);
- conf.setCapacityByLabel(rootChild + "." + E, "test", 0);
-
- conf.setCapacityByLabel(rootChild, "test2", 100);
- conf.setCapacityByLabel(rootChild + "." + A, "test2", 20);
- conf.setCapacityByLabel(rootChild + "." + B, "test2", 40);
- conf.setCapacityByLabel(rootChild + "." + C, "test2", 10);
- conf.setCapacityByLabel(rootChild + "." + C + "." + C1, "test2", 100);
- conf.setCapacityByLabel(rootChild + "." + D, "test2", 30);
- conf.setCapacityByLabel(rootChild + "." + E, "test2", 0);
+ conf.setCapacityByLabel(new QueuePath(rootChild), "test", 100);
+ conf.setCapacityByLabel(new QueuePath(rootChild, A), "test", 20);
+ conf.setCapacityByLabel(new QueuePath(rootChild, B), "test", 40);
+ conf.setCapacityByLabel(new QueuePath(rootChild, C), "test", 10);
+ conf.setCapacityByLabel(QueuePath.createFromQueues(rootChild, C, C1), "test", 100);
+ conf.setCapacityByLabel(new QueuePath(rootChild, D), "test", 30);
+ conf.setCapacityByLabel(new QueuePath(rootChild, E), "test", 0);
+
+ conf.setCapacityByLabel(new QueuePath(rootChild), "test2", 100);
+ conf.setCapacityByLabel(new QueuePath(rootChild, A), "test2", 20);
+ conf.setCapacityByLabel(new QueuePath(rootChild, B), "test2", 40);
+ conf.setCapacityByLabel(new QueuePath(rootChild, C), "test2", 10);
+ conf.setCapacityByLabel(QueuePath.createFromQueues(rootChild, C, C1), "test2", 100);
+ conf.setCapacityByLabel(new QueuePath(rootChild, D), "test2", 30);
+ conf.setCapacityByLabel(new QueuePath(rootChild, E), "test2", 0);
cs.getCapacitySchedulerQueueManager().reinitConfiguredNodeLabels(conf);
cs.setMaxRunningAppsEnforcer(new CSMaxRunningAppsEnforcer(cs));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java
index fcfd1bbb0e82..d3084503e930 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestMixedQueueResourceCalculation.java
@@ -522,15 +522,15 @@ public class TestMixedQueueResourceCalculation extends CapacitySchedulerQueueCal
}
private void setQueues() {
- csConf.setQueues("root", new String[]{"a", "b", "c"});
+ csConf.setQueues(ROOT, new String[]{"a", "b", "c"});
csConf.setQueues(A, new String[]{"a1", "a2"});
csConf.setQueues(B, new String[]{"b1"});
}
private Optional<QueueUpdateWarning> getSpecificWarning(
Collection<QueueUpdateWarning> warnings, QueueUpdateWarningType warningTypeToSelect,
- String queue) {
+ QueuePath queue) {
return warnings.stream().filter((w) -> w.getWarningType().equals(warningTypeToSelect)
- && w.getQueue().equals(queue)).findFirst();
+ && w.getQueue().equals(queue.getFullPath())).findFirst();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index db739377aaac..f960acd95611 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -76,6 +76,28 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
public class TestNodeLabelContainerAllocation {
+ private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c";
+ private static final String D_PATH = CapacitySchedulerConfiguration.ROOT + ".d";
+ private static final String A1_PATH = A_PATH + ".a1";
+ private static final String A2_PATH = A_PATH + ".a2";
+ private static final String B1_PATH = B_PATH + ".b1";
+ private static final String B2_PATH = B_PATH + ".b2";
+ private static final String C1_PATH = C_PATH + ".c1";
+ private static final String C2_PATH = C_PATH + ".c2";
+
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A = new QueuePath(A_PATH);
+ private static final QueuePath B = new QueuePath(B_PATH);
+ private static final QueuePath C = new QueuePath(C_PATH);
+ private static final QueuePath D = new QueuePath(D_PATH);
+ private static final QueuePath A1 = new QueuePath(A1_PATH);
+ private static final QueuePath A2 = new QueuePath(A2_PATH);
+ private static final QueuePath B1 = new QueuePath(B1_PATH);
+ private static final QueuePath B2 = new QueuePath(B2_PATH);
+ private static final QueuePath C1 = new QueuePath(C1_PATH);
+ private static final QueuePath C2 = new QueuePath(C2_PATH);
private final int GB = 1024;
private YarnConfiguration conf;
@@ -96,43 +118,37 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(config);
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ conf.setQueues(ROOT, new String[] {"a", "b", "c"});
+ conf.setCapacityByLabel(ROOT, "x", 100);
+ conf.setCapacityByLabel(ROOT, "y", 100);
+ conf.setCapacityByLabel(ROOT, "z", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
conf.setAccessibleNodeLabels(A, toSet("x"));
conf.setCapacityByLabel(A, "x", 100);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
conf.setAccessibleNodeLabels(B, toSet("y", "z"));
conf.setCapacityByLabel(B, "y", 100);
conf.setCapacityByLabel(B, "z", 100);
- final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
conf.setMaximumCapacity(C, 70);
conf.setAccessibleNodeLabels(C, RMNodeLabelsManager.EMPTY_STRING_SET);
-
+
// Define 2nd-level queues
- final String A1 = A + ".a1";
conf.setQueues(A, new String[] {"a1"});
conf.setCapacity(A1, 100);
conf.setMaximumCapacity(A1, 100);
conf.setCapacityByLabel(A1, "x", 100);
-
- final String B1 = B + ".b1";
+
conf.setQueues(B, new String[] {"b1"});
conf.setCapacity(B1, 100);
conf.setMaximumCapacity(B1, 100);
conf.setCapacityByLabel(B1, "y", 100);
conf.setCapacityByLabel(B1, "z", 100);
- final String C1 = C + ".c1";
conf.setQueues(C, new String[] {"c1"});
conf.setCapacity(C1, 100);
conf.setMaximumCapacity(C1, 100);
@@ -1373,52 +1389,43 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "b", "c"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 33);
csConf.setAccessibleNodeLabels(A, toSet("x"));
csConf.setCapacityByLabel(A, "x", 33);
csConf.setQueues(A, new String[] {"a1", "a2"});
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+
csConf.setCapacity(B, 33);
csConf.setAccessibleNodeLabels(B, toSet("x"));
csConf.setCapacityByLabel(B, "x", 33);
csConf.setQueues(B, new String[] {"b1", "b2"});
-
- final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+
csConf.setCapacity(C, 34);
csConf.setAccessibleNodeLabels(C, toSet("x"));
csConf.setCapacityByLabel(C, "x", 34);
csConf.setQueues(C, new String[] {"c1", "c2"});
// Define 2nd-level queues
- final String A1 = A + ".a1";
csConf.setCapacity(A1, 50);
csConf.setCapacityByLabel(A1, "x", 100);
csConf.setDefaultNodeLabelExpression(A1, "x");
-
- final String A2 = A + ".a2";
+
csConf.setCapacity(A2, 50);
csConf.setCapacityByLabel(A2, "x", 0);
-
- final String B1 = B + ".b1";
+
csConf.setCapacity(B1, 50);
csConf.setCapacityByLabel(B1, "x", 100);
csConf.setDefaultNodeLabelExpression(B1, "x");
-
- final String B2 = B + ".b2";
+
csConf.setCapacity(B2, 50);
csConf.setCapacityByLabel(B2, "x", 0);
-
- final String C1 = C + ".c1";
+
csConf.setCapacity(C1, 50);
csConf.setCapacityByLabel(C1, "x", 100);
csConf.setDefaultNodeLabelExpression(C1, "x");
-
- final String C2 = C + ".c2";
+
csConf.setCapacity(C2, 50);
csConf.setCapacityByLabel(C2, "x", 0);
@@ -1566,15 +1573,13 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 50);
csConf.setAccessibleNodeLabels(A, toSet("x"));
csConf.setCapacityByLabel(A, "x", 100);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+
csConf.setCapacity(B, 50);
csConf.setAccessibleNodeLabels(B, new HashSet<String>());
csConf.setUserLimitFactor(B, 5);
@@ -1730,18 +1735,15 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a",
- "b" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 50);
csConf.setAccessibleNodeLabels(A, toSet("x"));
csConf.setCapacityByLabel(A, "x", 50);
csConf.setMaximumCapacityByLabel(A, "x", 50);
csConf.setUserLimit(A, 200);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
csConf.setCapacity(B, 50);
csConf.setAccessibleNodeLabels(B, toSet("x"));
csConf.setCapacityByLabel(B, "x", 50);
@@ -1882,31 +1884,26 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a",
- "b" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
/**
* Initially, we set A/B's resource 50:50
*/
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 50);
csConf.setAccessibleNodeLabels(A, toSet("x"));
csConf.setCapacityByLabel(A, "x", 50);
csConf.setQueues(A, new String[] { "a1", "a2" });
-
- final String A1 = A + ".a1";
+
csConf.setCapacity(A1, 50);
csConf.setAccessibleNodeLabels(A1, toSet("x"));
csConf.setCapacityByLabel(A1, "x", 50);
-
- final String A2 = A + ".a2";
+
csConf.setCapacity(A2, 50);
csConf.setAccessibleNodeLabels(A2, toSet("x"));
csConf.setCapacityByLabel(A2, "x", 50);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
csConf.setCapacity(B, 50);
csConf.setAccessibleNodeLabels(B, toSet("x"));
csConf.setCapacityByLabel(B, "x", 50);
@@ -2143,25 +2140,20 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a",
- "b", "c", "d" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "b", "c", "d"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 25);
csConf.setAccessibleNodeLabels(A, toSet("x"));
csConf.setCapacityByLabel(A, "x", 30);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
csConf.setCapacity(B, 25);
csConf.setAccessibleNodeLabels(B, toSet("x"));
csConf.setCapacityByLabel(B, "x", 70);
-
- final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+
csConf.setAccessibleNodeLabels(C, Collections.<String> emptySet());
csConf.setCapacity(C, 25);
-
- final String D = CapacitySchedulerConfiguration.ROOT + ".d";
+
csConf.setAccessibleNodeLabels(D, Collections.<String> emptySet());
csConf.setCapacity(D, 25);
@@ -2346,16 +2338,14 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] { "a", "b" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 25);
csConf.setAccessibleNodeLabels(A, toSet("*"));
csConf.setCapacityByLabel(A, "x", 60);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
csConf.setCapacity(B, 75);
csConf.setAccessibleNodeLabels(B, toSet("x"));
csConf.setCapacityByLabel(B, "x", 40);
@@ -2434,17 +2424,14 @@ public class TestNodeLabelContainerAllocation {
new CapacitySchedulerConfiguration(this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a",
- "b"});
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 10);
csConf.setAccessibleNodeLabels(A, toSet("x"));
csConf.setCapacityByLabel(A, "x", 50);
csConf.setMaximumCapacityByLabel(A, "x", 50);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
csConf.setCapacity(B, 90);
csConf.setAccessibleNodeLabels(B, toSet("x"));
csConf.setCapacityByLabel(B, "x", 50);
@@ -2454,14 +2441,12 @@ public class TestNodeLabelContainerAllocation {
csConf.setQueues(A, new String[] { "a1",
"a2"});
- final String A1 = A + ".a1";
csConf.setCapacity(A1, 50);
csConf.setAccessibleNodeLabels(A1, toSet("x"));
csConf.setCapacityByLabel(A1, "x", 50);
csConf.setMaximumCapacityByLabel(A1, "x", 100);
csConf.setUserLimitFactor(A1, 100.0f);
- final String A2 = A + ".a2";
csConf.setCapacity(A2, 50);
csConf.setAccessibleNodeLabels(A2, toSet("x"));
csConf.setCapacityByLabel(A2, "x", 50);
@@ -2550,20 +2535,19 @@ public class TestNodeLabelContainerAllocation {
this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] { "a", "b" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
-
- final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
- csConf.setCapacity(queueA, 25);
- csConf.setAccessibleNodeLabels(queueA, toSet("x"));
- csConf.setCapacityByLabel(queueA, "x", 50);
- csConf.setMaximumCapacityByLabel(queueA, "x", 50);
- final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
- csConf.setCapacity(queueB, 75);
- csConf.setAccessibleNodeLabels(queueB, toSet("x"));
- csConf.setCapacityByLabel(queueB, "x", 50);
- csConf.setMaximumCapacityByLabel(queueB, "x", 50);
+ csConf.setCapacityByLabel(ROOT, "x", 100);
+
+ csConf.setCapacity(A, 25);
+ csConf.setAccessibleNodeLabels(A, toSet("x"));
+ csConf.setCapacityByLabel(A, "x", 50);
+ csConf.setMaximumCapacityByLabel(A, "x", 50);
+
+ csConf.setCapacity(B, 75);
+ csConf.setAccessibleNodeLabels(B, toSet("x"));
+ csConf.setCapacityByLabel(B, "x", 50);
+ csConf.setMaximumCapacityByLabel(B, "x", 50);
// set node -> label
mgr.addToCluserNodeLabels(
@@ -2675,20 +2659,19 @@ public class TestNodeLabelContainerAllocation {
this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] { "a", "b" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
-
- final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
- csConf.setCapacity(queueA, 25);
- csConf.setAccessibleNodeLabels(queueA, toSet("x"));
- csConf.setCapacityByLabel(queueA, "x", 50);
- csConf.setMaximumCapacityByLabel(queueA, "x", 50);
- final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
- csConf.setCapacity(queueB, 75);
- csConf.setAccessibleNodeLabels(queueB, toSet("x"));
- csConf.setCapacityByLabel(queueB, "x", 50);
- csConf.setMaximumCapacityByLabel(queueB, "x", 50);
+ csConf.setCapacityByLabel(ROOT, "x", 100);
+
+ csConf.setCapacity(A, 25);
+ csConf.setAccessibleNodeLabels(A, toSet("x"));
+ csConf.setCapacityByLabel(A, "x", 50);
+ csConf.setMaximumCapacityByLabel(A, "x", 50);
+
+ csConf.setCapacity(B, 75);
+ csConf.setAccessibleNodeLabels(B, toSet("x"));
+ csConf.setCapacityByLabel(B, "x", 50);
+ csConf.setMaximumCapacityByLabel(B, "x", 50);
csConf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
@@ -2956,15 +2939,14 @@ public class TestNodeLabelContainerAllocation {
this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {"a"});
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
- csConf.setCapacity(queueA, 100);
- csConf.setAccessibleNodeLabels(queueA, toSet("x"));
- csConf.setCapacityByLabel(queueA, "x", 100);
- csConf.setMaximumCapacityByLabel(queueA, "x", 100);
+ csConf.setCapacity(A, 100);
+ csConf.setAccessibleNodeLabels(A, toSet("x"));
+ csConf.setCapacityByLabel(A, "x", 100);
+ csConf.setMaximumCapacityByLabel(A, "x", 100);
// set node -> label
// label x exclusivity is set to true
@@ -3092,23 +3074,21 @@ public class TestNodeLabelContainerAllocation {
this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] {"a"});
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ csConf.setCapacityByLabel(ROOT, "x", 100);
- final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
- csConf.setCapacity(queueA, 100);
- csConf.setAccessibleNodeLabels(queueA, toSet("x"));
- csConf.setCapacityByLabel(queueA, "x", 100);
- csConf.setMaximumCapacityByLabel(queueA, "x", 100);
+ csConf.setCapacity(A, 100);
+ csConf.setAccessibleNodeLabels(A, toSet("x"));
+ csConf.setCapacityByLabel(A, "x", 100);
+ csConf.setMaximumCapacityByLabel(A, "x", 100);
- csConf.setQueues(queueA, new String[] {"a1"});
- final String queueA1 = queueA + ".a1";
- csConf.setCapacity(queueA1, 100);
+ csConf.setQueues(A, new String[] {"a1"});
- csConf.setAccessibleNodeLabels(queueA1, toSet("x"));
- csConf.setCapacityByLabel(queueA1, "x", 100);
- csConf.setMaximumCapacityByLabel(queueA1, "x", 100);
+ csConf.setCapacity(A1, 100);
+ csConf.setAccessibleNodeLabels(A1, toSet("x"));
+ csConf.setCapacityByLabel(A1, "x", 100);
+ csConf.setMaximumCapacityByLabel(A1, "x", 100);
// set node -> label
// label x exclusivity is set to true
@@ -3312,35 +3292,31 @@ public class TestNodeLabelContainerAllocation {
this.conf);
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ csConf.setQueues(ROOT,
new String[] { "a", "b" });
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
-
- final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
- csConf.setCapacity(queueA, 50);
- csConf.setMaximumCapacity(queueA, 100);
- csConf.setAccessibleNodeLabels(queueA, toSet("x"));
- csConf.setCapacityByLabel(queueA, "x", 50);
- csConf.setMaximumCapacityByLabel(queueA, "x", 100);
- final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
- csConf.setCapacity(queueB, 50);
- csConf.setMaximumCapacity(queueB, 100);
- csConf.setAccessibleNodeLabels(queueB, toSet("x"));
- csConf.setCapacityByLabel(queueB, "x", 50);
- csConf.setMaximumCapacityByLabel(queueB, "x", 100);
+ csConf.setCapacityByLabel(ROOT, "x", 100);
+
+ csConf.setCapacity(A, 50);
+ csConf.setMaximumCapacity(A, 100);
+ csConf.setAccessibleNodeLabels(A, toSet("x"));
+ csConf.setCapacityByLabel(A, "x", 50);
+ csConf.setMaximumCapacityByLabel(A, "x", 100);
+
+ csConf.setCapacity(B, 50);
+ csConf.setMaximumCapacity(B, 100);
+ csConf.setAccessibleNodeLabels(B, toSet("x"));
+ csConf.setCapacityByLabel(B, "x", 50);
+ csConf.setMaximumCapacityByLabel(B, "x", 100);
// Define 2nd-level queues
- csConf.setQueues(queueA, new String[] { "a1",
- "a2"});
+ csConf.setQueues(A, new String[] {"a1", "a2"});
- final String A1 = queueA + ".a1";
csConf.setCapacity(A1, 20);
csConf.setMaximumCapacity(A1, 60);
csConf.setAccessibleNodeLabels(A1, toSet("x"));
csConf.setCapacityByLabel(A1, "x", 60);
csConf.setMaximumCapacityByLabel(A1, "x", 30);
- final String A2 = queueA + ".a2";
csConf.setCapacity(A2, 80);
csConf.setMaximumCapacity(A2, 40);
csConf.setAccessibleNodeLabels(A2, toSet("x"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 85e5515691db..73db7c1c11bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -88,7 +88,35 @@ public class TestParentQueue {
private final ResourceCalculator resourceComparator =
new DefaultResourceCalculator();
-
+
+ private static final String A = "a";
+ private static final String B = "b";
+ private static final String C = "c";
+ private static final String C1 = "c1";
+ private static final String C11 = "c11";
+ private static final String C111 = "c111";
+ private static final String C1111 = "c1111";
+ private static final String D = "d";
+ private static final String A1 = "a1";
+ private static final String A2 = "a2";
+ private static final String B1 = "b1";
+ private static final String B2 = "b2";
+ private static final String B3 = "b3";
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath Q_A = ROOT.createNewLeaf(A);
+ private static final QueuePath Q_B = ROOT.createNewLeaf(B);
+ private static final QueuePath Q_C = ROOT.createNewLeaf(C);
+ private static final QueuePath Q_D = ROOT.createNewLeaf(D);
+ private static final QueuePath Q_A1 = Q_A.createNewLeaf(A1);
+ private static final QueuePath Q_A2 = Q_A.createNewLeaf(A2);
+ private static final QueuePath Q_B1 = Q_B.createNewLeaf(B1);
+ private static final QueuePath Q_B2 = Q_B.createNewLeaf(B2);
+ private static final QueuePath Q_B3 = Q_B.createNewLeaf(B3);
+ private static final QueuePath Q_C1 = Q_C.createNewLeaf(C1);
+ private static final QueuePath Q_C11 = Q_C1.createNewLeaf(C11);
+ private static final QueuePath Q_C111 = Q_C11.createNewLeaf(C111);
+ private static final QueuePath Q_C1111 = Q_C111.createNewLeaf(C1111);
+
@Before
public void setUp() throws Exception {
rmContext = TestUtils.getMockRMContext();
@@ -113,17 +141,11 @@ public class TestParentQueue {
queueContext = new CapacitySchedulerQueueContext(csContext);
}
-
- private static final String A = "a";
- private static final String B = "b";
- private static final String Q_A =
- CapacitySchedulerConfiguration.ROOT + "." + A;
- private static final String Q_B =
- CapacitySchedulerConfiguration.ROOT + "." + B;
+
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
+ conf.setQueues(ROOT, new String[] {A, B});
conf.setCapacity(Q_A, 30);
@@ -138,12 +160,12 @@ public class TestParentQueue {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
+ conf.setQueues(ROOT, new String[]{A, B});
- conf.setMinimumResourceRequirement("", new QueuePath(Q_A),
+ conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
- conf.setMinimumResourceRequirement("", new QueuePath(Q_B),
+ conf.setMinimumResourceRequirement("", Q_B,
QUEUE_B_RESOURCE);
queueContext.reinitialize();
@@ -433,60 +455,40 @@ public class TestParentQueue {
}
}
- private static final String C = "c";
- private static final String C1 = "c1";
- private static final String C11 = "c11";
- private static final String C111 = "c111";
- private static final String C1111 = "c1111";
-
- private static final String D = "d";
- private static final String A1 = "a1";
- private static final String A2 = "a2";
- private static final String B1 = "b1";
- private static final String B2 = "b2";
- private static final String B3 = "b3";
- private static final String B4 = "b4";
-
private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
+ csConf.setQueues(ROOT, new String[] {A, B, C, D});
conf.setCapacity(Q_A, 10);
conf.setCapacity(Q_B, 50);
-
- final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
+
conf.setCapacity(Q_C, 19.5f);
-
- final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
+
conf.setCapacity(Q_D, 20.5f);
// Define 2-nd level queues
conf.setQueues(Q_A, new String[] {A1, A2});
- conf.setCapacity(Q_A + "." + A1, 50);
- conf.setCapacity(Q_A + "." + A2, 50);
+ conf.setCapacity(Q_A1, 50);
+ conf.setCapacity(Q_A2, 50);
conf.setQueues(Q_B, new String[] {B1, B2, B3});
- conf.setCapacity(Q_B + "." + B1, 10);
- conf.setCapacity(Q_B + "." + B2, 20);
- conf.setCapacity(Q_B + "." + B3, 70);
+ conf.setCapacity(Q_B1, 10);
+ conf.setCapacity(Q_B2, 20);
+ conf.setCapacity(Q_B3, 70);
conf.setQueues(Q_C, new String[] {C1});
- final String Q_C1= Q_C + "." + C1;
conf.setCapacity(Q_C1, 100);
conf.setQueues(Q_C1, new String[] {C11});
- final String Q_C11= Q_C1 + "." + C11;
conf.setCapacity(Q_C11, 100);
conf.setQueues(Q_C11, new String[] {C111});
- final String Q_C111= Q_C11 + "." + C111;
conf.setCapacity(Q_C111, 100);
//Leaf Queue
conf.setQueues(Q_C111, new String[] {C1111});
- final String Q_C1111= Q_C111 + "." + C1111;
conf.setCapacity(Q_C1111, 100);
queueContext.reinitialize();
}
@@ -677,9 +679,9 @@ public class TestParentQueue {
assumeThat(csConf.isLegacyQueueMode(), is(true));
// set child queues capacity to 0 when parents not 0
- csConf.setCapacity(Q_B + "." + B1, 0);
- csConf.setCapacity(Q_B + "." + B2, 0);
- csConf.setCapacity(Q_B + "." + B3, 0);
+ csConf.setCapacity(Q_B1, 0);
+ csConf.setCapacity(Q_B2, 0);
+ csConf.setCapacity(Q_B3, 0);
queueContext.reinitialize();
CSQueueStore queues = new CSQueueStore();
@@ -740,9 +742,9 @@ public class TestParentQueue {
// and allow zero capacity sum
csConf.setCapacity(Q_B, 0);
csConf.setCapacity(Q_A, 100);
- csConf.setCapacity(Q_B + "." + B1, 10);
- csConf.setCapacity(Q_B + "." + B2, 20);
- csConf.setCapacity(Q_B + "." + B3, 20);
+ csConf.setCapacity(Q_B1, 10);
+ csConf.setCapacity(Q_B2, 20);
+ csConf.setCapacity(Q_B3, 20);
csConf.setAllowZeroCapacitySum(Q_B, true);
queueContext.reinitialize();
CSQueueStore queues = new CSQueueStore();
@@ -761,9 +763,9 @@ public class TestParentQueue {
// and allow zero capacity sum
csConf.setCapacity(Q_B, 10);
csConf.setCapacity(Q_A, 50);
- csConf.setCapacity(Q_B + "." + B1, 0);
- csConf.setCapacity(Q_B + "." + B2, 0);
- csConf.setCapacity(Q_B + "." + B3, 0);
+ csConf.setCapacity(Q_B1, 0);
+ csConf.setCapacity(Q_B2, 0);
+ csConf.setCapacity(Q_B3, 0);
csConf.setAllowZeroCapacitySum(Q_B, true);
queueContext.reinitialize();
CSQueueStore queues = new CSQueueStore();
@@ -779,9 +781,9 @@ public class TestParentQueue {
// set parent and child capacity to 0
csConf.setCapacity(Q_B, 0);
- csConf.setCapacity(Q_B + "." + B1, 0);
- csConf.setCapacity(Q_B + "." + B2, 0);
- csConf.setCapacity(Q_B + "." + B3, 0);
+ csConf.setCapacity(Q_B1, 0);
+ csConf.setCapacity(Q_B2, 0);
+ csConf.setCapacity(Q_B3, 0);
csConf.setCapacity(Q_A, 60);
queueContext.reinitialize();
@@ -977,12 +979,10 @@ public class TestParentQueue {
public void testQueueAcl() throws Exception {
setupMultiLevelQueues(csConf);
- csConf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
- csConf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.ADMINISTER_QUEUE, " ");
+ csConf.setAcl(ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
+ csConf.setAcl(ROOT, QueueACL.ADMINISTER_QUEUE, " ");
- final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
csConf.setAcl(Q_C, QueueACL.ADMINISTER_QUEUE, "*");
- final String Q_C11= Q_C + "." + C1 + "." + C11;
csConf.setAcl(Q_C11, QueueACL.SUBMIT_APPLICATIONS, "*");
queueContext.reinitialize();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java
index e1e06d72fc85..b30ea50bc9e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueMappings.java
@@ -43,6 +43,9 @@ public class TestQueueMappings {
CapacitySchedulerConfiguration.ROOT + "." + Q1;
private final static String Q2_PATH =
CapacitySchedulerConfiguration.ROOT + "." + Q2;
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath Q1_QUEUE_PATH = new QueuePath(Q1_PATH);
+ private static final QueuePath Q2_QUEUE_PATH = new QueuePath(Q2_PATH);
private CapacityScheduler cs;
private YarnConfiguration conf;
@@ -71,10 +74,10 @@ public class TestQueueMappings {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { Q1, Q2 });
+ conf.setQueues(ROOT, new String[] {Q1, Q2});
- conf.setCapacity(Q1_PATH, 10);
- conf.setCapacity(Q2_PATH, 90);
+ conf.setCapacity(Q1_QUEUE_PATH, 10);
+ conf.setCapacity(Q2_QUEUE_PATH, 90);
LOG.info("Setup top-level queues q1 and q2");
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index 736293c5c433..65bce5c9318a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -54,7 +54,46 @@ public class TestQueueParsing {
private static final Logger LOG =
LoggerFactory.getLogger(TestQueueParsing.class);
-
+ private static final String A_PATH = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B_PATH = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String C_PATH = CapacitySchedulerConfiguration.ROOT + ".c";
+ private static final String A1_PATH = A_PATH + ".a1";
+ private static final String A2_PATH = A_PATH + ".a2";
+ private static final String B1_PATH = B_PATH + ".b1";
+ private static final String B2_PATH = B_PATH + ".b2";
+ private static final String B3_PATH = B_PATH + ".b3";
+ private static final String B4_PATH = B_PATH + ".b4";
+ private static final String C1_PATH = C_PATH + ".c1";
+ private static final String C2_PATH = C_PATH + ".c2";
+ private static final String C3_PATH = C_PATH + ".c3";
+ private static final String C4_PATH = C_PATH + ".c4";
+ private static final String C11_PATH = C1_PATH + ".c11";
+ private static final String C12_PATH = C1_PATH + ".c12";
+ private static final String C13_PATH = C1_PATH + ".c13";
+ private static final String AX_PATH = "root.a.x";
+ private static final String AY_PATH = "root.a.y";
+ private static final String X_PATH = "root.x";
+
+ private static final QueuePath ROOT = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath A = new QueuePath(A_PATH);
+ private static final QueuePath B = new QueuePath(B_PATH);
+ private static final QueuePath C = new QueuePath(C_PATH);
+ private static final QueuePath A1 = new QueuePath(A1_PATH);
+ private static final QueuePath A2 = new QueuePath(A2_PATH);
+ private static final QueuePath B1 = new QueuePath(B1_PATH);
+ private static final QueuePath B2 = new QueuePath(B2_PATH);
+ private static final QueuePath B3 = new QueuePath(B3_PATH);
+ private static final QueuePath B4 = new QueuePath(B4_PATH);
+ private static final QueuePath C1 = new QueuePath(C1_PATH);
+ private static final QueuePath C2 = new QueuePath(C2_PATH);
+ private static final QueuePath C3 = new QueuePath(C3_PATH);
+ private static final QueuePath C4 = new QueuePath(C4_PATH);
+ private static final QueuePath C11 = new QueuePath(C11_PATH);
+ private static final QueuePath C12 = new QueuePath(C12_PATH);
+ private static final QueuePath C13 = new QueuePath(C13_PATH);
+ private static final QueuePath AX = new QueuePath(AX_PATH);
+ private static final QueuePath AY = new QueuePath(AY_PATH);
+ private static final QueuePath X = new QueuePath(X_PATH);
private static final double DELTA = 0.001;
private static final int GB = 1024;
@@ -75,33 +114,23 @@ public class TestQueueParsing {
private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
+ conf.setQueues(ROOT, new String[] {"a", "b", "c"});
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
-
- final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
conf.setMaximumCapacity(C, 70);
LOG.info("Setup top-level queues");
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
conf.setQueues(A, new String[] {"a1", "a2"});
conf.setCapacity(A1, 30);
conf.setMaximumCapacity(A1, 45);
conf.setCapacity(A2, 70);
conf.setMaximumCapacity(A2, 85);
- final String B1 = B + ".b1";
- final String B2 = B + ".b2";
- final String B3 = B + ".b3";
conf.setQueues(B, new String[] {"b1", "b2", "b3"});
conf.setCapacity(B1, 50);
conf.setMaximumCapacity(B1, 85);
@@ -110,10 +139,6 @@ public class TestQueueParsing {
conf.setCapacity(B3, 20);
conf.setMaximumCapacity(B3, 35);
- final String C1 = C + ".c1";
- final String C2 = C + ".c2";
- final String C3 = C + ".c3";
- final String C4 = C + ".c4";
conf.setQueues(C, new String[] {"c1", "c2", "c3", "c4"});
conf.setCapacity(C1, 50);
conf.setMaximumCapacity(C1, 55);
@@ -127,9 +152,6 @@ public class TestQueueParsing {
LOG.info("Setup 2nd-level queues");
// Define 3rd-level queues
- final String C11 = C1 + ".c11";
- final String C12 = C1 + ".c12";
- final String C13 = C1 + ".c13";
conf.setQueues(C1, new String[] {"c11", "c12", "c13"});
conf.setCapacity(C11, 15);
conf.setMaximumCapacity(C11, 30);
@@ -145,18 +167,12 @@ public class TestQueueParsing {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.set(
- CapacitySchedulerConfiguration
- .getQueuePrefix(CapacitySchedulerConfiguration.ROOT)
+ QueuePrefixes.getQueuePrefix(ROOT)
+ CapacitySchedulerConfiguration.QUEUES, " a ,b, c");
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
-
- final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
conf.setMaximumCapacity(C, 70);
}
@@ -165,57 +181,40 @@ public class TestQueueParsing {
CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.set(
- CapacitySchedulerConfiguration
- .getQueuePrefix(CapacitySchedulerConfiguration.ROOT)
+ QueuePrefixes.getQueuePrefix(ROOT)
+ CapacitySchedulerConfiguration.QUEUES, " a ,b, c");
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 20);
-
- final String C = CapacitySchedulerConfiguration.ROOT + ".c";
conf.setCapacity(C, 70);
conf.setMaximumCapacity(C, 70);
// sub queues for A
- conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A)
+ conf.set(QueuePrefixes.getQueuePrefix(A)
+ CapacitySchedulerConfiguration.QUEUES, "a1, a2 ");
- final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1";
conf.setCapacity(A1, 60);
-
- final String A2 = CapacitySchedulerConfiguration.ROOT + ".a.a2";
conf.setCapacity(A2, 40);
}
private void setupQueueConfigurationWithoutLabels(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+ conf.setQueues(ROOT, new String[] {"a", "b"});
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 90);
LOG.info("Setup top-level queues");
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
conf.setQueues(A, new String[] {"a1", "a2"});
conf.setCapacity(A1, 30);
conf.setMaximumCapacity(A1, 45);
conf.setCapacity(A2, 70);
conf.setMaximumCapacity(A2, 85);
- final String B1 = B + ".b1";
- final String B2 = B + ".b2";
- final String B3 = B + ".b3";
conf.setQueues(B, new String[] {"b1", "b2", "b3"});
conf.setCapacity(B1, 50);
conf.setMaximumCapacity(B1, 85);
@@ -227,22 +226,17 @@ public class TestQueueParsing {
private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100);
+ conf.setQueues(ROOT, new String[] {"a", "b"});
+ conf.setCapacityByLabel(ROOT, "red", 100);
+ conf.setCapacityByLabel(ROOT, "blue", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
-
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 90);
LOG.info("Setup top-level queues");
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
conf.setQueues(A, new String[] {"a1", "a2"});
conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue"));
conf.setCapacityByLabel(A, "red", 50);
@@ -260,9 +254,6 @@ public class TestQueueParsing {
conf.setCapacityByLabel(A2, "red", 50);
conf.setMaximumCapacityByLabel(A2, "red", 60);
- final String B1 = B + ".b1";
- final String B2 = B + ".b2";
- final String B3 = B + ".b3";
conf.setQueues(B, new String[] {"b1", "b2", "b3"});
conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue"));
conf.setCapacityByLabel(B, "red", 50);
@@ -287,24 +278,20 @@ public class TestQueueParsing {
private void setupQueueConfigurationWithLabelsAndReleaseCheck(
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100);
+ conf.setQueues(ROOT, new String[] {"a", "b"});
+ conf.setCapacityByLabel(ROOT, "red", 100);
+ conf.setCapacityByLabel(ROOT, "blue", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
// The cap <= max-cap check is not needed
conf.setCapacity(A, 50);
conf.setMaximumCapacity(A, 100);
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 50);
conf.setMaximumCapacity(B, 100);
LOG.info("Setup top-level queues");
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
conf.setQueues(A, new String[] {"a1", "a2"});
conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue"));
conf.setCapacityByLabel(A, "red", 50);
@@ -325,9 +312,6 @@ public class TestQueueParsing {
conf.setCapacityByLabel(A2, "red", 40);
conf.setMaximumCapacityByLabel(A2, "red", 60);
- final String B1 = B + ".b1";
- final String B2 = B + ".b2";
- final String B3 = B + ".b3";
conf.setQueues(B, new String[] {"b1", "b2", "b3"});
conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue"));
conf.setCapacityByLabel(B, "red", 50);
@@ -356,12 +340,11 @@ public class TestQueueParsing {
private void setupQueueConfigurationWithLabelsInherit(
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100);
- conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100);
+ conf.setQueues(ROOT, new String[] {"a", "b"});
+ conf.setCapacityByLabel(ROOT, "red", 100);
+ conf.setCapacityByLabel(ROOT, "blue", 100);
// Set A configuration
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
conf.setQueues(A, new String[] {"a1", "a2"});
@@ -370,14 +353,10 @@ public class TestQueueParsing {
conf.setCapacityByLabel(A, "blue", 100);
// Set B configuraiton
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 90);
conf.setAccessibleNodeLabels(B, CommonNodeLabelsManager.EMPTY_STRING_SET);
// Define 2nd-level queues
- final String A1 = A + ".a1";
- final String A2 = A + ".a2";
-
conf.setCapacity(A1, 30);
conf.setMaximumCapacity(A1, 45);
conf.setCapacityByLabel(A1, "red", 50);
@@ -392,10 +371,9 @@ public class TestQueueParsing {
private void setupQueueConfigurationWithSingleLevel(
CapacitySchedulerConfiguration conf) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+ conf.setQueues(ROOT, new String[] {"a", "b"});
// Set A configuration
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue"));
@@ -403,7 +381,6 @@ public class TestQueueParsing {
conf.setCapacityByLabel(A, "blue", 90);
// Set B configuraiton
- final String B = CapacitySchedulerConfiguration.ROOT + ".b";
conf.setCapacity(B, 90);
conf.setAccessibleNodeLabels(B, ImmutableSet.of("red", "blue"));
conf.setCapacityByLabel(B, "red", 10);
@@ -469,7 +446,7 @@ public class TestQueueParsing {
public void testRootQueueParsing() throws Exception {
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
// non-100 percent value will throw IllegalArgumentException
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT, 90);
+ csConf.setCapacity(ROOT, 90);
}
@Test
@@ -712,7 +689,7 @@ public class TestQueueParsing {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
setupQueueConfiguration(csConf);
- csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, labels);
+ csConf.setAccessibleNodeLabels(ROOT, labels);
YarnConfiguration conf = new YarnConfiguration(csConf);
MockRM rm = createMockRMWithLabels(conf, labels);
@@ -807,7 +784,7 @@ public class TestQueueParsing {
// some of the resources of a parent queue unallocated
assumeThat(csConf.isLegacyQueueMode(), is(true));
setupQueueConfiguration(csConf);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);
+ csConf.setCapacity(C2, 5);
YarnConfiguration conf = new YarnConfiguration(csConf);
MockRM rm = createMockRMWithLabels(conf, ImmutableSet.of("red", "blue"));
@@ -829,8 +806,7 @@ public class TestQueueParsing {
assumeThat(csConf.isLegacyQueueMode(), is(true));
setupQueueConfigurationWithLabels(csConf);
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
- "red", 24);
+ csConf.setCapacityByLabel(B3, "red", 24);
YarnConfiguration conf = new YarnConfiguration(csConf);
MockRM rm = createMockRMWithLabels(conf, ImmutableSet.of("red", "blue"));
@@ -852,12 +828,9 @@ public class TestQueueParsing {
assumeThat(csConf.isLegacyQueueMode(), is(true));
setupQueueConfigurationWithLabels(csConf);
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
- "red", 24);
- csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
- ImmutableSet.of(RMNodeLabelsManager.ANY));
- csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
- ImmutableSet.of(RMNodeLabelsManager.ANY));
+ csConf.setCapacityByLabel(B3, "red", 24);
+ csConf.setAccessibleNodeLabels(ROOT, ImmutableSet.of(RMNodeLabelsManager.ANY));
+ csConf.setAccessibleNodeLabels(B, ImmutableSet.of(RMNodeLabelsManager.ANY));
YarnConfiguration conf = new YarnConfiguration(csConf);
MockRM rm = createMockRMWithLabels(conf, ImmutableSet.of("red", "blue"));
@@ -868,21 +841,21 @@ public class TestQueueParsing {
public void testQueueParsingWithMoveQueue() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
- csConf.setQueues("root", new String[] { "a" });
- csConf.setQueues("root.a", new String[] { "x", "y" });
- csConf.setCapacity("root.a", 100);
- csConf.setCapacity("root.a.x", 50);
- csConf.setCapacity("root.a.y", 50);
+ csConf.setQueues(ROOT, new String[] {"a"});
+ csConf.setQueues(A, new String[] {"x", "y"});
+ csConf.setCapacity(A, 100);
+ csConf.setCapacity(AX, 50);
+ csConf.setCapacity(AY, 50);
YarnConfiguration conf = new YarnConfiguration(csConf);
MockRM rm = createMockRMWithoutLabels(conf);
CapacityScheduler capacityScheduler = (CapacityScheduler) rm.getResourceScheduler();
- csConf.setQueues("root", new String[] { "a", "x" });
- csConf.setQueues("root.a", new String[] { "y" });
- csConf.setCapacity("root.x", 50);
- csConf.setCapacity("root.a", 50);
- csConf.setCapacity("root.a.y", 100);
+ csConf.setQueues(ROOT, new String[] {"a", "x"});
+ csConf.setQueues(A, new String[] {"y"});
+ csConf.setCapacity(X, 50);
+ csConf.setCapacity(A, 50);
+ csConf.setCapacity(AY, 100);
capacityScheduler.reinitialize(csConf, rm.getRMContext());
ServiceOperations.stopQuietly(rm);
@@ -899,12 +872,11 @@ public class TestQueueParsing {
assumeThat(csConf.isLegacyQueueMode(), is(true));
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"});
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
- csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ csConf.setQueues(ROOT, new String[] {"a"});
+ csConf.setCapacityByLabel(ROOT, "x", 100);
+ csConf.setCapacityByLabel(ROOT, "y", 100);
+ csConf.setCapacityByLabel(ROOT, "z", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setCapacity(A, 100);
csConf.setAccessibleNodeLabels(A, labels);
csConf.setCapacityByLabel(A, "x", 100);
@@ -928,9 +900,9 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = (CapacityScheduler) rm.getResourceScheduler();
// Add a new b4 queue
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".b",
+ csConf.setQueues(B,
new String[] { "b1", "b2", "b3", "b4" });
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".b.b4", 0f);
+ csConf.setCapacity(B4, 0f);
ParentQueue bQ = (ParentQueue) capacityScheduler.getQueue("b");
checkEqualsToQueueSet(bQ.getChildQueues(),
new String[] { "b1", "b2", "b3" });
@@ -955,12 +927,11 @@ public class TestQueueParsing {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"});
- csConf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "x", 100);
- csConf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "y", 100);
- csConf.setLabeledQueueWeight(CapacitySchedulerConfiguration.ROOT, "z", 100);
+ csConf.setQueues(ROOT, new String[] {"a"});
+ csConf.setLabeledQueueWeight(ROOT, "x", 100);
+ csConf.setLabeledQueueWeight(ROOT, "y", 100);
+ csConf.setLabeledQueueWeight(ROOT, "z", 100);
- final String A = CapacitySchedulerConfiguration.ROOT + ".a";
csConf.setNonLabeledQueueWeight(A, 100);
csConf.setAccessibleNodeLabels(A, labels);
csConf.setLabeledQueueWeight(A, "x", 100);
@@ -975,10 +946,10 @@ public class TestQueueParsing {
verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "y", 1f);
verifyQueueAbsCapacity(rm, CapacitySchedulerConfiguration.ROOT, "z", 1f);
- verifyQueueAbsCapacity(rm, A, "", 1f);
- verifyQueueAbsCapacity(rm, A, "x", 1f);
- verifyQueueAbsCapacity(rm, A, "y", 1f);
- verifyQueueAbsCapacity(rm, A, "z", 1f);
+ verifyQueueAbsCapacity(rm, A_PATH, "", 1f);
+ verifyQueueAbsCapacity(rm, A_PATH, "x", 1f);
+ verifyQueueAbsCapacity(rm, A_PATH, "y", 1f);
+ verifyQueueAbsCapacity(rm, A_PATH, "z", 1f);
ServiceOperations.stopQuietly(rm);
}
@@ -991,17 +962,17 @@ public class TestQueueParsing {
final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
// Define top-level queues
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
// Set default value
csConf.setDefaultUserLimit(20);
csConf.setDefaultUserLimitFactor(2.0f);
// Set A configuration and let B use default values
- csConf.setCapacity(queueA, 50);
- csConf.setUserLimit(queueA, 15);
- csConf.setUserLimitFactor(queueA, 1.5f);
- csConf.setCapacity(queueB, 50);
+ csConf.setCapacity(A, 50);
+ csConf.setUserLimit(A, 15);
+ csConf.setUserLimitFactor(A, 1.5f);
+ csConf.setCapacity(B, 50);
YarnConfiguration conf = new YarnConfiguration(csConf);
MockRM rm = createMockRMWithoutLabels(conf);
@@ -1010,33 +981,33 @@ public class TestQueueParsing {
CapacityScheduler capacityScheduler = (CapacityScheduler) rm.getResourceScheduler();
Assert.assertEquals(15,
- ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimit(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimit(), DELTA);
Assert.assertEquals(1.5,
- ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimitFactor(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimitFactor(), DELTA);
Assert.assertEquals(20,
- ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimit(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimit(), DELTA);
Assert.assertEquals(2.0,
- ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimitFactor(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimitFactor(), DELTA);
// Use hadoop default value
csConf = new CapacitySchedulerConfiguration();
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
- csConf.setCapacity(queueA, 50);
- csConf.setUserLimit(queueA, 15);
- csConf.setUserLimitFactor(queueA, 1.5f);
- csConf.setCapacity(queueB, 50);
+ csConf.setQueues(ROOT, new String[] {"a", "b"});
+ csConf.setCapacity(A, 50);
+ csConf.setUserLimit(A, 15);
+ csConf.setUserLimitFactor(A, 1.5f);
+ csConf.setCapacity(B, 50);
capacityScheduler.reinitialize(csConf, rm.getRMContext());
Assert.assertEquals(15,
- ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimit(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimit(), DELTA);
Assert.assertEquals(1.5,
- ((LeafQueue)capacityScheduler.getQueue(queueA)).getUserLimitFactor(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(A_PATH)).getUserLimitFactor(), DELTA);
Assert.assertEquals(100,
- ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimit(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimit(), DELTA);
Assert.assertEquals(1,
- ((LeafQueue)capacityScheduler.getQueue(queueB)).getUserLimitFactor(), DELTA);
+ ((LeafQueue)capacityScheduler.getQueue(B_PATH)).getUserLimitFactor(), DELTA);
}
private void verifyQueueAbsCapacity(MockRM rm, String queuePath, String label,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java
index 91171966c119..dc6437930676 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueuePath.java
@@ -22,23 +22,32 @@ import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.junit.Assert;
import org.junit.Test;
+import java.util.ArrayList;
import java.util.List;
public class TestQueuePath {
private static final String TEST_QUEUE = "root.level_1.level_2.level_3";
+ private static final QueuePath TEST_QUEUE_PATH = new QueuePath(TEST_QUEUE);
+ private static final QueuePath QUEUE_PATH_WITH_EMPTY_PART = new QueuePath("root..level_2");
+ private static final QueuePath QUEUE_PATH_WITH_EMPTY_LEAF = new QueuePath("root.level_1.");
+ private static final QueuePath ROOT_PATH = new QueuePath(CapacitySchedulerConfiguration.ROOT);
+ private static final QueuePath EMPTY_PATH = new QueuePath("");
+ private static final QueuePath ONE_LEVEL_WILDCARDED_TEST_PATH =
+ new QueuePath("root.level_1.level_2.*");
+ private static final QueuePath TWO_LEVEL_WILDCARDED_TEST_PATH =
+ new QueuePath("root.level_1.*.*");
+ private static final QueuePath THREE_LEVEL_WILDCARDED_TEST_PATH =
+ new QueuePath("root.*.*.*");
@Test
public void testCreation() {
- QueuePath queuePath = new QueuePath(TEST_QUEUE);
-
- Assert.assertEquals(TEST_QUEUE, queuePath.getFullPath());
- Assert.assertEquals("root.level_1.level_2", queuePath.getParent());
- Assert.assertEquals("level_3", queuePath.getLeafName());
+ Assert.assertEquals(TEST_QUEUE, TEST_QUEUE_PATH.getFullPath());
+ Assert.assertEquals("root.level_1.level_2", TEST_QUEUE_PATH.getParent());
+ Assert.assertEquals("level_3", TEST_QUEUE_PATH.getLeafName());
- QueuePath rootPath = new QueuePath(CapacitySchedulerConfiguration.ROOT);
- Assert.assertNull(rootPath.getParent());
+ Assert.assertNull(ROOT_PATH.getParent());
- QueuePath appendedPath = queuePath.createNewLeaf("level_4");
+ QueuePath appendedPath = TEST_QUEUE_PATH.createNewLeaf("level_4");
Assert.assertEquals(TEST_QUEUE + CapacitySchedulerConfiguration.DOT
+ "level_4", appendedPath.getFullPath());
Assert.assertEquals("root.level_1.level_2.level_3", appendedPath.getParent());
@@ -47,13 +56,9 @@ public class TestQueuePath {
@Test
public void testEmptyPart() {
- QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2");
- QueuePath queuePathWithEmptyLeaf = new QueuePath("root.level_1.");
- QueuePath queuePathWithoutEmptyPart = new QueuePath(TEST_QUEUE);
-
- Assert.assertTrue(queuePathWithEmptyPart.hasEmptyPart());
- Assert.assertTrue(queuePathWithEmptyLeaf.hasEmptyPart());
- Assert.assertFalse(queuePathWithoutEmptyPart.hasEmptyPart());
+ Assert.assertTrue(QUEUE_PATH_WITH_EMPTY_PART.hasEmptyPart());
+ Assert.assertTrue(QUEUE_PATH_WITH_EMPTY_LEAF.hasEmptyPart());
+ Assert.assertFalse(TEST_QUEUE_PATH.hasEmptyPart());
}
@Test
@@ -68,14 +73,10 @@ public class TestQueuePath {
@Test
public void testIterator() {
- QueuePath queuePath = new QueuePath(TEST_QUEUE);
- QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2");
- QueuePath rootPath = new QueuePath(CapacitySchedulerConfiguration.ROOT);
-
- List<String> queuePathCollection = ImmutableList.copyOf(queuePath.iterator());
+ List<String> queuePathCollection = ImmutableList.copyOf(TEST_QUEUE_PATH.iterator());
List<String> queuePathWithEmptyPartCollection = ImmutableList.copyOf(
- queuePathWithEmptyPart.iterator());
- List<String> rootPathCollection = ImmutableList.copyOf(rootPath.iterator());
+ QUEUE_PATH_WITH_EMPTY_PART.iterator());
+ List<String> rootPathCollection = ImmutableList.copyOf(ROOT_PATH.iterator());
Assert.assertEquals(4, queuePathCollection.size());
Assert.assertEquals(CapacitySchedulerConfiguration.ROOT, queuePathCollection.get(0));
@@ -92,14 +93,10 @@ public class TestQueuePath {
@Test
public void testReversePathIterator() {
- QueuePath queuePath = new QueuePath(TEST_QUEUE);
- QueuePath queuePathWithEmptyPart = new QueuePath("root..level_2");
- QueuePath rootPath = new QueuePath(CapacitySchedulerConfiguration.ROOT);
-
- List<String> queuePathCollection = ImmutableList.copyOf(queuePath.reverseIterator());
+ List<String> queuePathCollection = ImmutableList.copyOf(TEST_QUEUE_PATH.reverseIterator());
List<String> queuePathWithEmptyPartCollection = ImmutableList.copyOf(
- queuePathWithEmptyPart.reverseIterator());
- List<String> rootPathCollection = ImmutableList.copyOf(rootPath.reverseIterator());
+ QUEUE_PATH_WITH_EMPTY_PART.reverseIterator());
+ List<String> rootPathCollection = ImmutableList.copyOf(ROOT_PATH.reverseIterator());
Assert.assertEquals(4, queuePathCollection.size());
Assert.assertEquals(CapacitySchedulerConfiguration.ROOT,
@@ -128,4 +125,101 @@ public class TestQueuePath {
Assert.assertEquals(empty, emptySame);
Assert.assertNotEquals(null, queuePath);
}
+
+ @Test
+ public void testInvalidPath() {
+ Assert.assertFalse(TEST_QUEUE_PATH.isInvalid());
+ Assert.assertFalse(ROOT_PATH.isInvalid());
+ Assert.assertTrue(EMPTY_PATH.isInvalid());
+ Assert.assertTrue(new QueuePath("invalidPath").isInvalid());
+ }
+
+ @Test
+ public void testGetParentObject() {
+ Assert.assertEquals(new QueuePath("root.level_1.level_2"),
+ TEST_QUEUE_PATH.getParentObject());
+ Assert.assertEquals(ROOT_PATH, new QueuePath("root.level_1").getParentObject());
+ Assert.assertNull(ROOT_PATH.getParentObject());
+ }
+
+ @Test
+ public void testGetPathComponents() {
+ Assert.assertArrayEquals(TEST_QUEUE_PATH.getPathComponents(),
+ new String[] {"root", "level_1", "level_2", "level_3"});
+ Assert.assertArrayEquals(ROOT_PATH.getPathComponents(), new String[] {"root"});
+ Assert.assertArrayEquals(EMPTY_PATH.getPathComponents(), new String[] {""});
+ }
+
+ @Test
+ public void testWildcardedQueuePathsWithOneLevelWildCard() {
+ int maxAutoCreatedQueueDepth = 1;
+
+ List<QueuePath> expectedPaths = new ArrayList<>();
+ expectedPaths.add(TEST_QUEUE_PATH);
+ expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH);
+
+ List<QueuePath> wildcardedPaths = TEST_QUEUE_PATH
+ .getWildcardedQueuePaths(maxAutoCreatedQueueDepth);
+
+ Assert.assertEquals(expectedPaths, wildcardedPaths);
+ }
+
+ @Test
+ public void testWildcardedQueuePathsWithTwoLevelWildCard() {
+ int maxAutoCreatedQueueDepth = 2;
+
+ List<QueuePath> expectedPaths = new ArrayList<>();
+ expectedPaths.add(TEST_QUEUE_PATH);
+ expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH);
+ expectedPaths.add(TWO_LEVEL_WILDCARDED_TEST_PATH);
+
+ List<QueuePath> wildcardedPaths = TEST_QUEUE_PATH
+ .getWildcardedQueuePaths(maxAutoCreatedQueueDepth);
+
+ Assert.assertEquals(expectedPaths, wildcardedPaths);
+ }
+
+ @Test
+ public void testWildcardedQueuePathsWithThreeLevelWildCard() {
+ int maxAutoCreatedQueueDepth = 3;
+
+ List<QueuePath> expectedPaths = new ArrayList<>();
+ expectedPaths.add(TEST_QUEUE_PATH);
+ expectedPaths.add(ONE_LEVEL_WILDCARDED_TEST_PATH);
+ expectedPaths.add(TWO_LEVEL_WILDCARDED_TEST_PATH);
... 2286 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org