You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/05/08 02:36:33 UTC
hadoop git commit: YARN-2918. RM should not fail on startup if
queue's configured labels do not exist in cluster-node-labels. Contributed by
Wangda Tan (cherry picked from commit
f489a4ec969f3727d03c8e85d51af1018fc0b2a1)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a3abe8d7e -> d817fbb34
YARN-2918. RM should not fail on startup if queue's configured labels do not exist in cluster-node-labels. Contributed by Wangda Tan
(cherry picked from commit f489a4ec969f3727d03c8e85d51af1018fc0b2a1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d817fbb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d817fbb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d817fbb3
Branch: refs/heads/branch-2
Commit: d817fbb34d6e34991c6e512c20d71387750a98f4
Parents: a3abe8d
Author: Jian He <ji...@apache.org>
Authored: Thu May 7 17:35:41 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu May 7 17:36:24 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../ApplicationMasterService.java | 2 +-
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 5 +-
.../scheduler/SchedulerUtils.java | 67 +++++----
.../scheduler/capacity/AbstractCSQueue.java | 6 +-
.../scheduler/capacity/CSQueueUtils.java | 28 +---
.../CapacitySchedulerConfiguration.java | 40 +++++-
.../scheduler/capacity/LeafQueue.java | 5 +-
.../scheduler/TestSchedulerUtils.java | 139 ++++++++++++++++---
.../scheduler/capacity/TestQueueParsing.java | 104 +++++++++++++-
11 files changed, 304 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bc19862..6c6dbca 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -288,6 +288,9 @@ Release 2.8.0 - UNRELEASED
YARN-3584. Fixed attempt diagnostics format shown on the UI. (nijel via
jianhe)
+ YARN-2918. RM should not fail on startup if queue's configured labels do
+ not exist in cluster-node-labels. (Wangda Tan via jianhe)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 7244b17..cd1dacf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -504,7 +504,7 @@ public class ApplicationMasterService extends AbstractService implements
try {
RMServerUtils.normalizeAndValidateRequests(ask,
rScheduler.getMaximumResourceCapability(), app.getQueue(),
- rScheduler);
+ rScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index ca21f11..7990421 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -383,7 +383,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
try {
SchedulerUtils.normalizeAndValidateRequest(amReq,
scheduler.getMaximumResourceCapability(),
- submissionContext.getQueue(), scheduler, isRecovery);
+ submissionContext.getQueue(), scheduler, isRecovery, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("RM app submission failed in validating AM resource request"
+ " for application " + submissionContext.getApplicationId(), e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 083189a..4669a28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -91,11 +91,12 @@ public class RMServerUtils {
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
- Resource maximumResource, String queueName, YarnScheduler scheduler)
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext)
throws InvalidResourceRequestException {
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
- queueName, scheduler);
+ queueName, scheduler, rmContext);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 5e9ea5d..0ef5c1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -210,7 +211,7 @@ public class SchedulerUtils {
public static void normalizeAndValidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
- boolean isRecovery)
+ boolean isRecovery, RMContext rmContext)
throws InvalidResourceRequestException {
QueueInfo queueInfo = null;
@@ -222,15 +223,16 @@ public class SchedulerUtils {
}
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
- validateResourceRequest(resReq, maximumResource, queueInfo);
+ validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
}
}
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler)
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext)
throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
- false);
+ false, rmContext);
}
/**
@@ -239,8 +241,8 @@ public class SchedulerUtils {
*
* @throws InvalidResourceRequestException when there is invalid request
*/
- public static void validateResourceRequest(ResourceRequest resReq,
- Resource maximumResource, QueueInfo queueInfo)
+ private static void validateResourceRequest(ResourceRequest resReq,
+ Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
if (resReq.getCapability().getMemory() < 0 ||
resReq.getCapability().getMemory() > maximumResource.getMemory()) {
@@ -282,7 +284,7 @@ public class SchedulerUtils {
if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
- labelExp)) {
+ labelExp, rmContext)) {
throw new InvalidResourceRequestException("Invalid resource request"
+ ", queue="
+ queueInfo.getQueueName()
@@ -295,40 +297,37 @@ public class SchedulerUtils {
}
}
}
-
- public static void checkIfLabelInClusterNodeLabels(RMNodeLabelsManager mgr,
- Set<String> labels) throws IOException {
- if (mgr == null) {
- if (labels != null && !labels.isEmpty()) {
- throw new IOException("NodeLabelManager is null, please check");
- }
- return;
- }
-
- if (labels != null) {
- for (String label : labels) {
- if (!label.equals(RMNodeLabelsManager.ANY)
- && !mgr.containsNodeLabel(label)) {
- throw new IOException("NodeLabelManager doesn't include label = "
- + label + ", please check.");
- }
- }
- }
- }
+ /**
+ * Check queue label expression, check if node label in queue's
+ * node-label-expression existed in clusterNodeLabels if rmContext != null
+ */
public static boolean checkQueueLabelExpression(Set<String> queueLabels,
- String labelExpression) {
- if (queueLabels != null && queueLabels.contains(RMNodeLabelsManager.ANY)) {
- return true;
- }
+ String labelExpression, RMContext rmContext) {
// if label expression is empty, we can allocate container on any node
if (labelExpression == null) {
return true;
}
for (String str : labelExpression.split("&&")) {
- if (!str.trim().isEmpty()
- && (queueLabels == null || !queueLabels.contains(str.trim()))) {
- return false;
+ str = str.trim();
+ if (!str.trim().isEmpty()) {
+ // check queue label
+ if (queueLabels == null) {
+ return false;
+ } else {
+ if (!queueLabels.contains(str)
+ && !queueLabels.contains(RMNodeLabelsManager.ANY)) {
+ return false;
+ }
+ }
+
+ // check node label manager contains this label
+ if (null != rmContext) {
+ RMNodeLabelsManager nlm = rmContext.getNodeLabelManager();
+ if (nlm != null && !nlm.containsNodeLabel(str)) {
+ return false;
+ }
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 47cea19..8b4637a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -118,11 +118,9 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void setupConfigurableCapacities() {
CSQueueUtils.loadUpdateAndCheckCapacities(
getQueuePath(),
- accessibleLabels,
csContext.getConfiguration(),
queueCapacities,
- parent == null ? null : parent.getQueueCapacities(),
- csContext.getRMContext().getNodeLabelManager());
+ parent == null ? null : parent.getQueueCapacities());
}
@Override
@@ -248,8 +246,6 @@ public abstract class AbstractCSQueue implements CSQueue {
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
}
- SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
- this.accessibleLabels);
// inherit from parent if labels not set
if (this.defaultLabelExpression == null && parent != null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 0cc7589..1206026 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -116,10 +116,9 @@ class CSQueueUtils {
* - Check if capacities/absolute-capacities legal
*/
public static void loadUpdateAndCheckCapacities(String queuePath,
- Set<String> accessibleLabels, CapacitySchedulerConfiguration csConf,
- QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities,
- RMNodeLabelsManager nlm) {
- loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm,
+ CapacitySchedulerConfiguration csConf,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ loadCapacitiesByLabelsFromConf(queuePath,
queueCapacities, csConf);
updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
@@ -127,28 +126,13 @@ class CSQueueUtils {
capacitiesSanityCheck(queuePath, queueCapacities);
}
- // Considered NO_LABEL, ANY and null cases
- private static Set<String> normalizeAccessibleNodeLabels(Set<String> labels,
- RMNodeLabelsManager mgr) {
- Set<String> accessibleLabels = new HashSet<String>();
- if (labels != null) {
- accessibleLabels.addAll(labels);
- }
- if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) {
- accessibleLabels.addAll(mgr.getClusterNodeLabelNames());
- }
- accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL);
-
- return accessibleLabels;
- }
-
private static void loadCapacitiesByLabelsFromConf(String queuePath,
- Set<String> labels, RMNodeLabelsManager mgr,
QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) {
queueCapacities.clearConfigurableFields();
- labels = normalizeAccessibleNodeLabels(labels, mgr);
+ Set<String> configuredNodelabels =
+ csConf.getConfiguredNodeLabels(queuePath);
- for (String label : labels) {
+ for (String label : configuredNodelabels) {
if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL,
csConf.getNonLabeledQueueCapacity(queuePath) / 100);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index b00f25c..55e360d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -23,8 +23,10 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.StringTokenizer;
@@ -44,13 +46,14 @@ import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.*;
-
-
import com.google.common.collect.ImmutableSet;
public class CapacitySchedulerConfiguration extends ReservationSchedulerConfiguration {
@@ -909,4 +912,35 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
defaultVal);
return preemptionDisabled;
}
+
+ /**
+ * Get configured node labels in a given queuePath
+ */
+ public Set<String> getConfiguredNodeLabels(String queuePath) {
+ Set<String> configuredNodeLabels = new HashSet<String>();
+ Entry<String, String> e = null;
+
+ Iterator<Entry<String, String>> iter = iterator();
+ while (iter.hasNext()) {
+ e = iter.next();
+ String key = e.getKey();
+
+ if (key.startsWith(getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS
+ + DOT)) {
+ // Find <label-name> in
+ // <queue-path>.accessible-node-labels.<label-name>.property
+ int labelStartIdx =
+ key.indexOf(ACCESSIBLE_NODE_LABELS)
+ + ACCESSIBLE_NODE_LABELS.length() + 1;
+ int labelEndIndx = key.indexOf('.', labelStartIdx);
+ String labelName = key.substring(labelStartIdx, labelEndIndx);
+ configuredNodeLabels.add(labelName);
+ }
+ }
+
+ // always add NO_LABEL
+ configuredNodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+
+ return configuredNodeLabels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 56ade84..15d3289 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -185,8 +186,8 @@ public class LeafQueue extends AbstractCSQueue {
maxAMResourcePerQueuePercent =
conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
- if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
- this.defaultLabelExpression)) {
+ if (!SchedulerUtils.checkQueueLabelExpression(
+ this.accessibleLabels, this.defaultLabelExpression, null)) {
throw new IOException("Invalid default label expression of "
+ " queue="
+ getQueueName()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 8abaeb6..4befe0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -64,8 +65,10 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -83,12 +86,15 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
public class TestSchedulerUtils {
private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class);
+ private RMContext rmContext = getMockRMContext();
+
@Test (timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -206,6 +212,9 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x"),
+ NodeLabel.newInstance("y")));
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -213,22 +222,44 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression(" ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is a subset of queue labels");
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
+ }
+
+ // same as above, but cluster node labels don't contains label being
+ // requested. should fail
+ try {
+ // set queue accessible node labesl to [x, y]
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), ResourceRequest.ANY, resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler, rmContext);
+
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
// queue has labels, failed cases (when ask a label not included by queue)
@@ -236,6 +267,9 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x"),
+ NodeLabel.newInstance("y")));
Resource resource = Resources.createResource(
0,
@@ -244,9 +278,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("z");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// we don't allow specify more than two node labels in a single expression
@@ -255,6 +292,9 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x"),
+ NodeLabel.newInstance("y")));
Resource resource = Resources.createResource(
0,
@@ -263,9 +303,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x && y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// queue doesn't have label, succeed (when request no label)
@@ -280,15 +323,15 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression(" ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is empty");
@@ -299,6 +342,9 @@ public class TestSchedulerUtils {
// set queue accessible node labels to empty
queueAccessibleNodeLabels.clear();
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x")));
+
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -306,9 +352,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x"));
}
// queue is "*", always succeeded
@@ -317,6 +366,10 @@ public class TestSchedulerUtils {
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x"),
+ NodeLabel.newInstance("y"), NodeLabel.newInstance("z")));
+
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -324,18 +377,39 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("z");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when queue can access any labels");
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y", "z"));
+ }
+
+ // same as above, but cluster node labels don't contains label, should fail
+ try {
+ // set queue accessible node labels to empty
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), ResourceRequest.ANY, resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler, rmContext);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
// we don't allow resource name other than ANY and specify label
@@ -343,6 +417,9 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x"),
+ NodeLabel.newInstance("y")));
Resource resource = Resources.createResource(
0,
@@ -351,9 +428,12 @@ public class TestSchedulerUtils {
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// we don't allow resource name other than ANY and specify label even if
@@ -363,6 +443,8 @@ public class TestSchedulerUtils {
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays
.asList(CommonNodeLabelsManager.ANY));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of(NodeLabel.newInstance("x")));
Resource resource = Resources.createResource(
0,
@@ -371,9 +453,12 @@ public class TestSchedulerUtils {
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x"));
}
}
@@ -395,7 +480,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Zero memory should be accepted");
}
@@ -409,7 +494,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Zero vcores should be accepted");
}
@@ -424,7 +509,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Max memory should be accepted");
}
@@ -439,7 +524,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Max vcores should not be accepted");
}
@@ -453,7 +538,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("Negative memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -468,7 +553,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("Negative vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -484,7 +569,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("More than max memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -501,7 +586,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("More than max vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -632,4 +717,12 @@ public class TestSchedulerUtils {
Assert.assertNull(applications.get(appId));
return app;
}
+
+ private static RMContext getMockRMContext() {
+ RMContext rmContext = mock(RMContext.class);
+ RMNodeLabelsManager nlm = new NullRMNodeLabelsManager();
+ nlm.init(new Configuration(false));
+ when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+ return rmContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d817fbb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index 0206772..8d04700 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -551,7 +551,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(capacityScheduler);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -579,7 +579,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -607,7 +607,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -635,7 +635,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf =
@@ -769,4 +769,100 @@ public class TestQueueParsing {
Assert.assertEquals(0.10 * 0.4, a2.getAbsoluteCapacity(), DELTA);
Assert.assertEquals(0.15, a2.getAbsoluteMaximumCapacity(), DELTA);
}
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfiguration(csConf);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfigurationWithLabels(csConf);
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
+ "red", 24);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfigurationWithLabels(csConf);
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
+ "red", 24);
+ csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
+ ImmutableSet.of(RMNodeLabelsManager.ANY));
+ csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
+ ImmutableSet.of(RMNodeLabelsManager.ANY));
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
}