You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2021/06/24 18:15:50 UTC
[hadoop] branch trunk updated: YARN-10780. Optimise retrieval of
configured node labels in CS queues. Contributed by Andras Gyori.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0934e78 YARN-10780. Optimise retrieval of configured node labels in CS queues. Contributed by Andras Gyori.
0934e78 is described below
commit 0934e783cf687eee4f5f23e65500b91d18b96f65
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Thu Jun 24 20:15:10 2021 +0200
YARN-10780. Optimise retrieval of configured node labels in CS queues. Contributed by Andras Gyori.
---
.../scheduler/capacity/AbstractCSQueue.java | 40 ++++++++---
.../scheduler/capacity/AutoCreatedLeafQueue.java | 20 +++++-
.../scheduler/capacity/CSQueueUtils.java | 9 ++-
.../capacity/CapacitySchedulerConfiguration.java | 36 ++++++++++
.../capacity/CapacitySchedulerQueueManager.java | 13 ++++
.../scheduler/capacity/ConfiguredNodeLabels.java | 77 ++++++++++++++++++++++
.../scheduler/capacity/ManagedParentQueue.java | 16 +++--
.../scheduler/capacity/ParentQueue.java | 5 +-
.../TestCapacitySchedulerNewQueueAutoCreation.java | 15 +++++
.../scheduler/capacity/TestReservationQueue.java | 5 ++
10 files changed, 210 insertions(+), 26 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 94ade5e..2f6ca5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -76,6 +76,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.UNDEFINED;
public abstract class AbstractCSQueue implements CSQueue {
@@ -95,6 +96,7 @@ public abstract class AbstractCSQueue implements CSQueue {
final ResourceCalculator resourceCalculator;
Set<String> accessibleLabels;
+ protected Set<String> configuredNodeLabels;
Set<String> resourceTypes;
final RMNodeLabelsManager labelManager;
String defaultLabelExpression;
@@ -208,7 +210,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void setupConfigurableCapacities(
CapacitySchedulerConfiguration configuration) {
CSQueueUtils.loadCapacitiesByLabelsFromConf(getQueuePath(), queueCapacities,
- configuration);
+ configuration, configuredNodeLabels);
}
@Override
@@ -360,7 +362,7 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.lock();
try {
- if (isDynamicQueue()) {
+ if (isDynamicQueue() || this instanceof AbstractAutoCreatedLeafQueue) {
setDynamicQueueProperties(configuration);
}
// get labels
@@ -386,6 +388,17 @@ public abstract class AbstractCSQueue implements CSQueue {
this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
}
+ if (csContext.getCapacitySchedulerQueueManager() != null
+ && csContext.getCapacitySchedulerQueueManager()
+ .getConfiguredNodeLabels() != null) {
+ this.configuredNodeLabels = csContext.getCapacitySchedulerQueueManager()
+ .getConfiguredNodeLabels().getLabelsByQueue(getQueuePath());
+ } else {
+ // Fallback to suboptimal but correct logic
+ this.configuredNodeLabels = csContext.getConfiguration()
+ .getConfiguredNodeLabels(queuePath);
+ }
+
// After we setup labels, we can setup capacities
setupConfigurableCapacities(configuration);
updateAbsoluteCapacities();
@@ -487,6 +500,19 @@ public abstract class AbstractCSQueue implements CSQueue {
if (getParent() instanceof ParentQueue) {
((ParentQueue) getParent()).getAutoCreatedQueueTemplate()
.setTemplateEntriesForChild(configuration, getQueuePath());
+
+ String parentTemplate = String.format("%s.%s", getParent().getQueuePath(),
+ AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX);
+ parentTemplate = parentTemplate.substring(0, parentTemplate.lastIndexOf(
+ DOT));
+ Set<String> parentNodeLabels = csContext
+ .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+ .getLabelsByQueue(parentTemplate);
+
+ if (parentNodeLabels != null && parentNodeLabels.size() > 1) {
+ csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+ .setLabelsByQueue(queuePath, new HashSet<>(parentNodeLabels));
+ }
}
}
@@ -571,10 +597,7 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void updateConfigurableResourceRequirement(String queuePath,
Resource clusterResource) {
- CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- Set<String> configuredNodelabels = conf.getConfiguredNodeLabels(queuePath);
-
- for (String label : configuredNodelabels) {
+ for (String label : configuredNodeLabels) {
Resource minResource = getMinimumAbsoluteResource(queuePath, label);
Resource maxResource = getMaximumAbsoluteResource(queuePath, label);
@@ -1578,9 +1601,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
void updateEffectiveResources(Resource clusterResource) {
- Set<String> configuredNodelabels =
- csContext.getConfiguration().getConfiguredNodeLabels(getQueuePath());
- for (String label : configuredNodelabels) {
+ for (String label : configuredNodeLabels) {
Resource resourceByLabel = labelManager.getResourceByLabel(label,
clusterResource);
@@ -1715,5 +1736,4 @@ public abstract class AbstractCSQueue implements CSQueue {
writeLock.unlock();
}
}
-
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index dd77a80..7311be7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
@@ -28,6 +27,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
/**
* Leaf queues which are auto created by an underlying implementation of
@@ -122,6 +123,23 @@ public class AutoCreatedLeafQueue extends AbstractAutoCreatedLeafQueue {
}
}
+ @Override
+ protected void setDynamicQueueProperties(
+ CapacitySchedulerConfiguration configuration) {
+ String parentTemplate = String.format("%s.%s", getParent().getQueuePath(),
+ CapacitySchedulerConfiguration
+ .AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX);
+ Set<String> parentNodeLabels = csContext
+ .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+ .getLabelsByQueue(parentTemplate);
+
+ if (parentNodeLabels != null && parentNodeLabels.size() > 1) {
+ csContext.getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+ .setLabelsByQueue(getQueuePath(),
+ new HashSet<>(parentNodeLabels));
+ }
+ }
+
private void validate(final CSQueue newlyParsedQueue) throws IOException {
if (!(newlyParsedQueue instanceof AutoCreatedLeafQueue) || !newlyParsedQueue
.getQueuePath().equals(getQueuePath())) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index e2865cf..f43ea30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -66,13 +66,12 @@ public class CSQueueUtils {
return (parentAbsMaxCapacity * maximumCapacity);
}
- public static void loadCapacitiesByLabelsFromConf(String queuePath,
- QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) {
+ public static void loadCapacitiesByLabelsFromConf(
+ String queuePath, QueueCapacities queueCapacities,
+ CapacitySchedulerConfiguration csConf, Set<String> nodeLabels) {
queueCapacities.clearConfigurableFields();
- Set<String> configuredNodelabels =
- csConf.getConfiguredNodeLabels(queuePath);
- for (String label : configuredNodelabels) {
+ for (String label : nodeLabels) {
if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
queueCapacities.setCapacity(label,
csConf.getNonLabeledQueueCapacity(queuePath) / 100);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 622fef6..4afa2e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -1564,6 +1564,42 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return configuredNodeLabels;
}
+ /**
+ * Get configured node labels for all queues that have accessible-node-labels
+ * prefixed properties set.
+ * @return configured node labels
+ */
+ public Map<String, Set<String>> getConfiguredNodeLabelsByQueue() {
+ Map<String, Set<String>> labelsByQueue = new HashMap<>();
+ Map<String, String> schedulerEntries = getPropsWithPrefix(
+ CapacitySchedulerConfiguration.PREFIX);
+
+ for (Map.Entry<String, String> propertyEntry
+ : schedulerEntries.entrySet()) {
+ String key = propertyEntry.getKey();
+ // Consider all keys that has accessible-node-labels prefix, excluding
+ // <queue-path>.accessible-node-labels itself
+ if (key.contains(ACCESSIBLE_NODE_LABELS + DOT)) {
+ // Find <label-name> in
+ // <queue-path>.accessible-node-labels.<label-name>.property
+ int labelStartIdx =
+ key.indexOf(ACCESSIBLE_NODE_LABELS)
+ + ACCESSIBLE_NODE_LABELS.length() + 1;
+ int labelEndIndx = key.indexOf('.', labelStartIdx);
+ String labelName = key.substring(labelStartIdx, labelEndIndx);
+ // Find queuePath and exclude "." at the end
+ String queuePath = key.substring(0, key.indexOf(
+ ACCESSIBLE_NODE_LABELS) - 1);
+ if (!labelsByQueue.containsKey(queuePath)) {
+ labelsByQueue.put(queuePath, new HashSet<>());
+ labelsByQueue.get(queuePath).add(RMNodeLabelsManager.NO_LABEL);
+ }
+ labelsByQueue.get(queuePath).add(labelName);
+ }
+ }
+ return labelsByQueue;
+ }
+
public Integer getDefaultApplicationPriorityConfPerQueue(String queue) {
Integer defaultPriority = getInt(getQueuePrefix(queue)
+ DEFAULT_APPLICATION_PRIORITY,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index a86a253..e5b41ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -84,6 +84,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
private QueueStateManager<CSQueue, CapacitySchedulerConfiguration>
queueStateManager;
+ private ConfiguredNodeLabels configuredNodeLabels;
/**
* Construct the service.
@@ -98,6 +99,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
this.labelManager = labelManager;
this.queueStateManager = new QueueStateManager<>();
this.appPriorityACLManager = appPriorityACLManager;
+ this.configuredNodeLabels = new ConfiguredNodeLabels();
}
@Override
@@ -165,6 +167,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
*/
public void initializeQueues(CapacitySchedulerConfiguration conf)
throws IOException {
+ configuredNodeLabels = new ConfiguredNodeLabels(conf);
root = parseQueue(this.csContext, conf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues, NOOP);
setQueueAcls(authorizer, appPriorityACLManager, queues);
@@ -180,6 +183,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
throws IOException {
// Parse new queues
CSQueueStore newQueues = new CSQueueStore();
+ configuredNodeLabels = new ConfiguredNodeLabels(newConf);
CSQueue newRoot = parseQueue(this.csContext, newConf, null,
CapacitySchedulerConfiguration.ROOT, newQueues, queues, NOOP);
@@ -619,6 +623,15 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
return parentsToCreate;
}
+ /**
+ * Get {@code ConfiguredNodeLabels} which contains the configured node labels
+ * for all queues.
+ * @return configured node labels
+ */
+ public ConfiguredNodeLabels getConfiguredNodeLabels() {
+ return configuredNodeLabels;
+ }
+
private LeafQueue createAutoQueue(ApplicationPlacementContext queue)
throws SchedulerDynamicEditException {
List<String> parentsToCreate = determineMissingParents(queue);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfiguredNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfiguredNodeLabels.java
new file mode 100644
index 0000000..ba82c6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ConfiguredNodeLabels.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Contains node labels for all queues extracted from configuration properties.
+ * A queue has a configured node label if it has a property set with an
+ * accessible-node-labels prefix.
+ * Example:
+ * yarn.scheduler.capacity.root.accessible-node-labels.test-label.capacity
+ */
+public class ConfiguredNodeLabels {
+ private final Map<String, Set<String>> configuredNodeLabelsByQueue;
+ private static final Set<String> NO_LABEL =
+ ImmutableSet.of(RMNodeLabelsManager.NO_LABEL);
+
+ public ConfiguredNodeLabels() {
+ configuredNodeLabelsByQueue = new HashMap<>();
+ }
+
+ public ConfiguredNodeLabels(
+ CapacitySchedulerConfiguration conf) {
+ this.configuredNodeLabelsByQueue = conf.getConfiguredNodeLabelsByQueue();
+ }
+
+ /**
+ * Returns a set of configured node labels for a queue. If no labels are set
+ * for a queue, it defaults to a one element immutable collection containing
+ * empty label.
+ * @param queuePath path of the queue
+ * @return configured node labels or an immutable set containing the empty
+ * label
+ */
+ public Set<String> getLabelsByQueue(String queuePath) {
+ Set<String> labels = configuredNodeLabelsByQueue.get(queuePath);
+
+ if (labels == null) {
+ return NO_LABEL;
+ }
+
+ return ImmutableSet.copyOf(labels);
+ }
+
+ /**
+ * Set node labels for a specific queue.
+ * @param queuePath path of the queue
+ * @param nodeLabels configured node labels to set
+ */
+ public void setLabelsByQueue(
+ String queuePath, Collection<String> nodeLabels) {
+ configuredNodeLabelsByQueue.put(queuePath, new HashSet<>(nodeLabels));
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
index 88fae00..e415ac1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
@@ -37,6 +37,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Auto Creation enabled Parent queue. This queue initially does not have any
@@ -164,10 +165,13 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
CapacitySchedulerConfiguration conf =
super.initializeLeafQueueConfigs(leafQueueTemplateConfPrefix);
builder.configuration(conf);
+ String templateQueuePath = csContext.getConfiguration()
+ .getAutoCreatedQueueTemplateConfPrefix(getQueuePath());
- for (String nodeLabel : conf
- .getConfiguredNodeLabels(csContext.getConfiguration()
- .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()))) {
+ Set<String> templateConfiguredNodeLabels = csContext
+ .getCapacitySchedulerQueueManager().getConfiguredNodeLabels()
+ .getLabelsByQueue(templateQueuePath);
+ for (String nodeLabel : templateConfiguredNodeLabels) {
Resource templateMinResource = conf.getMinimumResourceRequirement(
nodeLabel, csContext.getConfiguration()
.getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
@@ -182,10 +186,10 @@ public class ManagedParentQueue extends AbstractManagedParentQueue {
//Load template capacities
QueueCapacities queueCapacities = new QueueCapacities(false);
- CSQueueUtils.loadCapacitiesByLabelsFromConf(csContext.getConfiguration()
- .getAutoCreatedQueueTemplateConfPrefix(getQueuePath()),
+ CSQueueUtils.loadCapacitiesByLabelsFromConf(templateQueuePath,
queueCapacities,
- csContext.getConfiguration());
+ csContext.getConfiguration(),
+ templateConfiguredNodeLabels);
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index ee60155..8ff4d91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,7 +25,6 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
@@ -1258,9 +1257,7 @@ public class ParentQueue extends AbstractCSQueue {
}
// Update effective capacity in all parent queue.
- Set<String> configuredNodelabels = csContext.getConfiguration()
- .getConfiguredNodeLabels(getQueuePath());
- for (String label : configuredNodelabels) {
+ for (String label : configuredNodeLabels) {
calculateEffectiveResourcesAndCapacity(label, clusterResource);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
index 7d0d537..4229676 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNewQueueAutoCreation.java
@@ -706,6 +706,21 @@ public class TestCapacitySchedulerNewQueueAutoCreation
LeafQueue a3 = createQueue("root.a.a3");
Assert.assertFalse("auto queue deletion should be turned off on a3",
a3.isEligibleForAutoDeletion());
+
+ // Set the capacity of label TEST
+ csConf.set(AutoCreatedQueueTemplate.getAutoQueueTemplatePrefix(
+ "root.c") + "accessible-node-labels.TEST.capacity", "6w");
+ csConf.setQueues("root", new String[]{"a", "b", "c"});
+ csConf.setAutoQueueCreationV2Enabled("root.c", true);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ LeafQueue c1 = createQueue("root.c.c1");
+ Assert.assertEquals("weight is not set for label TEST", 6f,
+ c1.getQueueCapacities().getWeight("TEST"), 1e-6);
+ cs.reinitialize(csConf, mockRM.getRMContext());
+ c1 = (LeafQueue) cs.getQueue("root.c.c1");
+ Assert.assertEquals("weight is not set for label TEST", 6f,
+ c1.getQueueCapacities().getWeight("TEST"), 1e-6);
+
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
index 84de7cc..8407922 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -60,7 +60,12 @@ public class TestReservationQueue {
YarnConfiguration conf = new YarnConfiguration();
csContext = mock(CapacitySchedulerContext.class);
+ CapacitySchedulerQueueManager csQm = mock(
+ CapacitySchedulerQueueManager.class);
+ ConfiguredNodeLabels labels = new ConfiguredNodeLabels(csConf);
+ when(csQm.getConfiguredNodeLabels()).thenReturn(labels);
when(csContext.getConfiguration()).thenReturn(csConf);
+ when(csContext.getCapacitySchedulerQueueManager()).thenReturn(csQm);
when(csContext.getConf()).thenReturn(conf);
when(csContext.getMinimumResourceCapability()).thenReturn(
Resources.createResource(GB, 1));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org