You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/07 19:24:20 UTC
[1/8] hadoop git commit: YARN-2920. Changed CapacityScheduler to kill
containers on nodes where node labels are changed. Contributed by Wangda Tan
(cherry picked from commit fdf042dfffa4d2474e3cac86cfb8fe9ee4648beb)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 2073fc0f8 -> 233626490
YARN-2920. Changed CapacityScheduler to kill containers on nodes where node labels are changed. Contributed by Wangda Tan
(cherry picked from commit fdf042dfffa4d2474e3cac86cfb8fe9ee4648beb)
(cherry picked from commit 411836b74c6c02c0b5aebbbce29c209d93db1de2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88f022da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88f022da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88f022da
Branch: refs/heads/branch-2.6.1
Commit: 88f022da245c5d34997494a822124d07f8a5f72f
Parents: 2073fc0
Author: Jian He <ji...@apache.org>
Authored: Mon Dec 22 16:50:15 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:18 2015 -0700
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 3 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 3 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/ResourceManager.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 23 +++
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +-
.../scheduler/SchedulerNode.java | 24 ++-
.../scheduler/capacity/AbstractCSQueue.java | 5 +
.../scheduler/capacity/CSQueue.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 88 +++++++--
.../scheduler/capacity/LeafQueue.java | 26 +--
.../scheduler/capacity/ParentQueue.java | 24 ++-
.../common/fica/FiCaSchedulerNode.java | 11 +-
.../event/NodeLabelsUpdateSchedulerEvent.java | 37 ++++
.../scheduler/event/SchedulerEventType.java | 1 +
.../yarn/server/resourcemanager/MockNodes.java | 3 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 193 +++++++++++++++++++
17 files changed, 414 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index fdddcf4..ee6eb7b 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -162,7 +163,7 @@ public class NodeInfo {
@Override
public Set<String> getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 3b185ae..b64be1b 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -150,6 +151,6 @@ public class RMNodeWrapper implements RMNode {
@Override
public Set<String> getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8244d61..fa1e120 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -171,6 +171,9 @@ Release 2.6.1 - UNRELEASED
YARN-3733. Fix DominantRC#compare() does not work as expected if
cluster resource is empty. (Rohith Sharmaks via wangda)
+ YARN-2920. Changed CapacityScheduler to kill containers on nodes where
+ node labels are changed. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index ea762c0..353851c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -430,6 +430,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setAMFinishingMonitor(amFinishingMonitor);
RMNodeLabelsManager nlm = createNodeLabelManager();
+ nlm.setRMContext(rmContext);
addService(nlm);
rmContext.setNodeLabelManager(nlm);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index ba1727c..d9828e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -35,7 +35,10 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.collect.ImmutableSet;
@@ -57,6 +60,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
new ConcurrentHashMap<String, Queue>();
protected AccessControlList adminAcl;
+ private RMContext rmContext = null;
+
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
@@ -331,6 +336,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
return map;
}
+ @SuppressWarnings("unchecked")
private void updateResourceMappings(Map<String, Host> before,
Map<String, Host> after) {
// Get NMs in before only
@@ -341,6 +347,10 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
for (Entry<String, Host> entry : after.entrySet()) {
allNMs.addAll(entry.getValue().nms.keySet());
}
+
+ // Map used to notify RM
+ Map<NodeId, Set<String>> newNodeToLabelsMap =
+ new HashMap<NodeId, Set<String>>();
// traverse all nms
for (NodeId nodeId : allNMs) {
@@ -379,6 +389,9 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
Node newNM;
if ((newNM = getNMInNodeSet(nodeId, after, true)) != null) {
Set<String> newLabels = getLabelsByNode(nodeId, after);
+
+ newNodeToLabelsMap.put(nodeId, ImmutableSet.copyOf(newLabels));
+
// no label in the past
if (newLabels.isEmpty()) {
// update labels
@@ -405,6 +418,12 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
}
}
}
+
+ // Notify RM
+ if (rmContext != null && rmContext.getDispatcher() != null) {
+ rmContext.getDispatcher().getEventHandler().handle(
+ new NodeLabelsUpdateSchedulerEvent(newNodeToLabelsMap));
+ }
}
public Resource getResourceByLabel(String label, Resource clusterResource) {
@@ -452,4 +471,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
}
return false;
}
+
+ public void setRMContext(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b92f399..b57b7cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -860,9 +861,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
@Override
public Set<String> getNodeLabels() {
- if (context.getNodeLabelManager() == null) {
+ RMNodeLabelsManager nlm = context.getNodeLabelManager();
+ if (nlm == null || nlm.getLabelsOnNode(nodeId) == null) {
return CommonNodeLabelsManager.EMPTY_STRING_SET;
}
- return context.getNodeLabelManager().getLabelsOnNode(nodeId);
+ return nlm.getLabelsOnNode(nodeId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index f4d8731..d1922ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,11 +34,14 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.collect.ImmutableSet;
+
/**
* Represents a YARN Cluster Node from the viewpoint of the scheduler.
@@ -61,8 +65,11 @@ public abstract class SchedulerNode {
private final RMNode rmNode;
private final String nodeName;
-
- public SchedulerNode(RMNode node, boolean usePortForNodeName) {
+
+ private volatile Set<String> labels = null;
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> labels) {
this.rmNode = node;
this.availableResource = Resources.clone(node.getTotalCapability());
this.totalResourceCapability = Resources.clone(node.getTotalCapability());
@@ -71,6 +78,11 @@ public abstract class SchedulerNode {
} else {
nodeName = rmNode.getHostName();
}
+ this.labels = ImmutableSet.copyOf(labels);
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName) {
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
}
public RMNode getRMNode() {
@@ -275,4 +287,12 @@ public abstract class SchedulerNode {
}
allocateContainer(rmContainer);
}
+
+ public Set<String> getLabels() {
+ return labels;
+ }
+
+ public void updateLabels(Set<String> labels) {
+ this.labels = labels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fc0fbb4..1f6696d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -447,4 +447,9 @@ public abstract class AbstractCSQueue implements CSQueue {
public Map<QueueACL, AccessControlList> getACLs() {
return acls;
}
+
+ @Private
+ public Resource getUsedResourceByLabel(String nodeLabel) {
+ return usedResourcesByNodeLabels.get(nodeLabel);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 6438d6c..07a7e0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -144,6 +144,14 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
public Resource getUsedResources();
/**
+ * Get the currently utilized resources which allocated at nodes with label
+ * specified
+ *
+ * @return used resources by the queue and it's children
+ */
+ public Resource getUsedResourceByLabel(String nodeLabel);
+
+ /**
* Get the current run-state of the queue
* @return current run-state
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 042c83c..c49c498 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -25,15 +25,15 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -48,12 +48,15 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -79,12 +82,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping.MappingType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -93,6 +99,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -106,11 +113,6 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
-
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -966,6 +968,51 @@ public class CapacityScheduler extends
updateNodeResource(nm, resourceOption);
root.updateClusterResource(clusterResource);
}
+
+ /**
+ * Process node labels update on a node.
+ *
+ * TODO: Currently capacity scheduler will kill containers on a node when
+ * labels on the node changed. It is a simply solution to ensure guaranteed
+ * capacity on labels of queues. When YARN-2498 completed, we can let
+ * preemption policy to decide if such containers need to be killed or just
+ * keep them running.
+ */
+ private synchronized void updateLabelsOnNode(NodeId nodeId,
+ Set<String> newLabels) {
+ FiCaSchedulerNode node = nodes.get(nodeId);
+ if (null == node) {
+ return;
+ }
+
+ // labels is same, we don't need do update
+ if (node.getLabels().size() == newLabels.size()
+ && node.getLabels().containsAll(newLabels)) {
+ return;
+ }
+
+ // Kill running containers since label is changed
+ for (RMContainer rmContainer : node.getRunningContainers()) {
+ ContainerId containerId = rmContainer.getContainerId();
+ completedContainer(rmContainer,
+ ContainerStatus.newInstance(containerId,
+ ContainerState.COMPLETE,
+ String.format(
+ "Container=%s killed since labels on the node=%s changed",
+ containerId.toString(), nodeId.toString()),
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL);
+ }
+
+ // Unreserve container on this node
+ RMContainer reservedContainer = node.getReservedContainer();
+ if (null != reservedContainer) {
+ dropContainerReservation(reservedContainer);
+ }
+
+ // Update node labels after we've done this
+ node.updateLabels(newLabels);
+ }
private synchronized void allocateContainersToNode(FiCaSchedulerNode node) {
if (rmContext.isWorkPreservingRecoveryEnabled()
@@ -1049,6 +1096,19 @@ public class CapacityScheduler extends
nodeResourceUpdatedEvent.getResourceOption());
}
break;
+ case NODE_LABELS_UPDATE:
+ {
+ NodeLabelsUpdateSchedulerEvent labelUpdateEvent =
+ (NodeLabelsUpdateSchedulerEvent) event;
+
+ for (Entry<NodeId, Set<String>> entry : labelUpdateEvent
+ .getUpdatedNodeToLabels().entrySet()) {
+ NodeId id = entry.getKey();
+ Set<String> labels = entry.getValue();
+ updateLabelsOnNode(id, labels);
+ }
+ }
+ break;
case NODE_UPDATE:
{
NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event;
@@ -1117,14 +1177,8 @@ public class CapacityScheduler extends
}
private synchronized void addNode(RMNode nodeManager) {
- // update this node to node label manager
- if (labelManager != null) {
- labelManager.activateNode(nodeManager.getNodeID(),
- nodeManager.getTotalCapability());
- }
-
this.nodes.put(nodeManager.getNodeID(), new FiCaSchedulerNode(nodeManager,
- usePortForNodeName));
+ usePortForNodeName, nodeManager.getNodeLabels()));
Resources.addTo(clusterResource, nodeManager.getTotalCapability());
root.updateClusterResource(clusterResource);
int numNodes = numNodeManagers.incrementAndGet();
@@ -1135,6 +1189,12 @@ public class CapacityScheduler extends
if (scheduleAsynchronously && numNodes == 1) {
asyncSchedulerThread.beginSchedule();
}
+
+ // update this node to node label manager
+ if (labelManager != null) {
+ labelManager.activateNode(nodeManager.getNodeID(),
+ nodeManager.getTotalCapability());
+ }
}
private synchronized void removeNode(RMNode nodeInfo) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 60b5a59..509b0f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -809,7 +809,7 @@ public class LeafQueue extends AbstractCSQueue {
// if our queue cannot access this node, just return
if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels,
- labelManager.getLabelsOnNode(node.getNodeID()))) {
+ node.getLabels())) {
return NULL_ASSIGNMENT;
}
@@ -878,7 +878,7 @@ public class LeafQueue extends AbstractCSQueue {
// Check queue max-capacity limit
if (!canAssignToThisQueue(clusterResource, required,
- labelManager.getLabelsOnNode(node.getNodeID()), application, true)) {
+ node.getLabels(), application, true)) {
return NULL_ASSIGNMENT;
}
@@ -911,7 +911,7 @@ public class LeafQueue extends AbstractCSQueue {
// Book-keeping
// Note: Update headroom to account for current allocation too...
allocateResource(clusterResource, application, assigned,
- labelManager.getLabelsOnNode(node.getNodeID()));
+ node.getLabels());
// Don't reset scheduling opportunities for non-local assignments
// otherwise the app will be delayed for each non-local assignment.
@@ -1561,7 +1561,7 @@ public class LeafQueue extends AbstractCSQueue {
// check if the resource request can access the label
if (!SchedulerUtils.checkNodeLabelExpression(
- labelManager.getLabelsOnNode(node.getNodeID()),
+ node.getLabels(),
request.getNodeLabelExpression())) {
// this is a reserved container, but we cannot allocate it now according
// to label not match. This can be caused by node label changed
@@ -1752,8 +1752,7 @@ public class LeafQueue extends AbstractCSQueue {
// Book-keeping
if (removed) {
releaseResource(clusterResource, application,
- container.getResource(),
- labelManager.getLabelsOnNode(node.getNodeID()));
+ container.getResource(), node.getLabels());
LOG.info("completedContainer" +
" container=" + container +
" queue=" + this +
@@ -1950,9 +1949,10 @@ public class LeafQueue extends AbstractCSQueue {
}
// Careful! Locking order is important!
synchronized (this) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, attempt, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
}
getParent().recoverContainer(clusterResource, attempt, rmContainer);
}
@@ -1989,9 +1989,10 @@ public class LeafQueue extends AbstractCSQueue {
public void attachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
@@ -2006,9 +2007,10 @@ public class LeafQueue extends AbstractCSQueue {
public void detachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
releaseResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer.getContainer()
- .getNodeId()));
+ .getResource(), node.getLabels());
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 6ffaf4c..fd598f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -73,6 +73,7 @@ public class ParentQueue extends AbstractCSQueue {
private final boolean rootQueue;
final Comparator<CSQueue> queueComparator;
volatile int numApplications;
+ private final CapacitySchedulerContext scheduler;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -80,7 +81,7 @@ public class ParentQueue extends AbstractCSQueue {
public ParentQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
-
+ this.scheduler = cs;
this.queueComparator = cs.getQueueComparator();
this.rootQueue = (parent == null);
@@ -420,10 +421,10 @@ public class ParentQueue extends AbstractCSQueue {
Resource clusterResource, FiCaSchedulerNode node, boolean needToUnreserve) {
CSAssignment assignment =
new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
+ Set<String> nodeLabels = node.getLabels();
// if our queue cannot access this node, just return
- if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels,
- labelManager.getLabelsOnNode(node.getNodeID()))) {
+ if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
return assignment;
}
@@ -434,7 +435,6 @@ public class ParentQueue extends AbstractCSQueue {
}
boolean localNeedToUnreserve = false;
- Set<String> nodeLabels = labelManager.getLabelsOnNode(node.getNodeID());
// Are we over maximum-capacity for this queue?
if (!canAssignToThisQueue(clusterResource, nodeLabels)) {
@@ -641,7 +641,7 @@ public class ParentQueue extends AbstractCSQueue {
// Book keeping
synchronized (this) {
super.releaseResource(clusterResource, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(node.getNodeID()));
+ .getResource(), node.getLabels());
LOG.info("completedContainer" +
" queue=" + getQueueName() +
@@ -703,9 +703,10 @@ public class ParentQueue extends AbstractCSQueue {
}
// Careful! Locking order is important!
synchronized (this) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
super.allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
}
if (parent != null) {
parent.recoverContainer(clusterResource, attempt, rmContainer);
@@ -730,9 +731,10 @@ public class ParentQueue extends AbstractCSQueue {
public void attachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
super.allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
@@ -748,9 +750,11 @@ public class ParentQueue extends AbstractCSQueue {
public void detachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
super.releaseResource(clusterResource,
rmContainer.getContainer().getResource(),
- labelManager.getLabelsOnNode(rmContainer.getContainer().getNodeId()));
+ node.getLabels());
LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 5227aac..fe6db47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica;
+import java.util.Set;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -32,9 +34,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
public class FiCaSchedulerNode extends SchedulerNode {
private static final Log LOG = LogFactory.getLog(FiCaSchedulerNode.class);
+
+ public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> nodeLabels) {
+ super(node, usePortForNodeName, nodeLabels);
+ }
public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) {
- super(node, usePortForNodeName);
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
new file mode 100644
index 0000000..7723e25
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+public class NodeLabelsUpdateSchedulerEvent extends SchedulerEvent {
+ private Map<NodeId, Set<String>> nodeToLabels;
+
+ public NodeLabelsUpdateSchedulerEvent(Map<NodeId, Set<String>> nodeToLabels) {
+ super(SchedulerEventType.NODE_LABELS_UPDATE);
+ this.nodeToLabels = nodeToLabels;
+ }
+
+ public Map<NodeId, Set<String>> getUpdatedNodeToLabels() {
+ return nodeToLabels;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 062f831..13aecb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -25,6 +25,7 @@ public enum SchedulerEventType {
NODE_REMOVED,
NODE_UPDATE,
NODE_RESOURCE_UPDATE,
+ NODE_LABELS_UPDATE,
// Source: RMApp
APP_ADDED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 228f200..278c151 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
@@ -206,7 +207,7 @@ public class MockNodes {
@Override
public Set<String> getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
new file mode 100644
index 0000000..261fa01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+public class TestCapacitySchedulerNodeLabelUpdate {
+ private final int GB = 1024;
+
+ private YarnConfiguration conf;
+
+ RMNodeLabelsManager mgr;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ mgr = new MemoryRMNodeLabelsManager();
+ mgr.init(conf);
+ }
+
+ private Configuration getConfigurationWithQueueLabels(Configuration config) {
+ CapacitySchedulerConfiguration conf =
+ new CapacitySchedulerConfiguration(config);
+
+ // Define top-level queues
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"});
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ conf.setCapacity(A, 100);
+ conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z"));
+ conf.setCapacityByLabel(A, "x", 100);
+ conf.setCapacityByLabel(A, "y", 100);
+ conf.setCapacityByLabel(A, "z", 100);
+
+ return conf;
+ }
+
+ private Set<String> toSet(String... elements) {
+ Set<String> set = Sets.newHashSet(elements);
+ return set;
+ }
+
+ private void checkUsedResource(MockRM rm, String queueName, int memory) {
+ checkUsedResource(rm, queueName, memory, RMNodeLabelsManager.NO_LABEL);
+ }
+
+ private void checkUsedResource(MockRM rm, String queueName, int memory,
+ String label) {
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+ CSQueue queue = scheduler.getQueue(queueName);
+ Assert.assertEquals(memory, queue.getUsedResourceByLabel(label).getMemory());
+ }
+
+ @Test (timeout = 30000)
+ public void testNodeUpdate() throws Exception {
+ // set node -> label
+ mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));
+
+ // set mapping:
+ // h1 -> x
+ // h2 -> y
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+
+ // inject node label manager
+ MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm.getRMContext().setNodeLabelManager(mgr);
+ rm.start();
+ MockNM nm1 = rm.registerNode("h1:1234", 8000);
+ MockNM nm2 = rm.registerNode("h2:1234", 8000);
+ MockNM nm3 = rm.registerNode("h3:1234", 8000);
+
+ ContainerId containerId;
+
+ // launch an app to queue a1 (label = x), and check all container will
+ // be allocated in h1
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
+
+ // request a container.
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
+ containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.ALLOCATED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=1G, ""=1G
+ checkUsedResource(rm, "a", 1024, "x");
+ checkUsedResource(rm, "a", 1024);
+
+ // change h1's label to z, container should be killed
+ mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h1", 0),
+ toSet("z")));
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.KILLED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=0G, ""=1G ("" not changed)
+ checkUsedResource(rm, "a", 0, "x");
+ checkUsedResource(rm, "a", 1024);
+
+ // request a container with label = y
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "y");
+ containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
+ Assert.assertTrue(rm.waitForState(nm2, containerId,
+ RMContainerState.ALLOCATED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used y=1G, ""=1G
+ checkUsedResource(rm, "a", 1024, "y");
+ checkUsedResource(rm, "a", 1024);
+
+ // change h2's label to no label, container should be killed
+ mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h2", 0),
+ CommonNodeLabelsManager.EMPTY_STRING_SET));
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.KILLED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=0G, y=0G, ""=1G ("" not changed)
+ checkUsedResource(rm, "a", 0, "x");
+ checkUsedResource(rm, "a", 0, "y");
+ checkUsedResource(rm, "a", 1024);
+
+ containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+
+ // change h3's label to z, AM container should be killed
+ mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h3", 0),
+ toSet("z")));
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.KILLED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=0G, y=0G, ""=1G ("" not changed)
+ checkUsedResource(rm, "a", 0, "x");
+ checkUsedResource(rm, "a", 0, "y");
+ checkUsedResource(rm, "a", 0);
+
+ rm.close();
+ }
+}
[6/8] hadoop git commit: YARN-2694. Ensure only single node label
specified in ResourceRequest. Contributed by Wangda Tan (cherry picked from
commit c1957fef29b07fea70938e971b30532a1e131fd0)
Posted by vi...@apache.org.
YARN-2694. Ensure only single node label specified in ResourceRequest. Contributed by Wangda Tan
(cherry picked from commit c1957fef29b07fea70938e971b30532a1e131fd0)
(cherry picked from commit 3ddafaa7c854dcf21ecc790c276927e7c869e62c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/637e7f9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/637e7f9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/637e7f9e
Branch: refs/heads/branch-2.6.1
Commit: 637e7f9e393db9a9e73f9683a398aec7d6aab909
Parents: 4c94f07
Author: Jian He <ji...@apache.org>
Authored: Fri Feb 6 11:34:12 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 21:07:51 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/api/records/ResourceRequest.java | 21 ++++---
.../hadoop/yarn/client/api/AMRMClient.java | 3 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 37 ++++++++++++-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 23 +++++---
.../yarn/client/api/impl/TestAMRMClient.java | 58 ++++++++++++++++----
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 14 ++++-
.../nodelabels/CommonNodeLabelsManager.java | 45 +++++++++++++--
.../nodelabels/TestCommonNodeLabelsManager.java | 47 +++++++++++++---
.../ApplicationMasterService.java | 5 +-
.../scheduler/SchedulerUtils.java | 19 +++++++
.../yarn/server/resourcemanager/MockAM.java | 17 ++++--
.../nodelabels/TestRMNodeLabelsManager.java | 50 ++++++++---------
.../scheduler/TestSchedulerUtils.java | 52 +++++++++++++++---
.../capacity/TestContainerAllocation.java | 32 +++++------
.../webapp/TestRMWebServicesNodeLabels.java | 17 +++---
16 files changed, 329 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 58c2139..2c654dc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -186,6 +186,9 @@ Release 2.6.1 - UNRELEASED
YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage
to track used-resources-by-label.(Wangda Tan via jianhe)
+ YARN-2694. Ensure only single node label specified in ResourceRequest.
+ (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 7f86cae..2f17ac9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -253,24 +253,27 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
/**
* Get node-label-expression for this Resource Request. If this is set, all
* containers allocated to satisfy this resource-request will be only on those
- * nodes that satisfy this node-label-expression
+ * nodes that satisfy this node-label-expression.
+ *
+ * Please note that node label expression now can only take effect when the
+ * resource request has resourceName = ANY
*
* @return node-label-expression
*/
@Public
@Evolving
- public abstract String getNodeLabelExpression();
+ public abstract String getNodeLabelExpression();
/**
- * Set node label expression of this resource request. Now only
- * support AND(&&), in the future will provide support for OR(||), NOT(!).
+ * Set node label expression of this resource request. Now only support
+ * specifying a single node label. In the future we will support more complex
+ * node label expression specification like AND(&&), OR(||), etc.
*
- * Examples:
- * - GPU && LARGE_MEM, ask for node has label GPU and LARGE_MEM together
- * - "" (empty) means ask for node doesn't have label on it, this is default
- * behavior
+ * Any please note that node label expression now can only take effect when
+ * the resource request has resourceName = ANY
*
- * @param nodelabelExpression node-label-expression of this ResourceRequest
+ * @param nodelabelExpression
+ * node-label-expression of this ResourceRequest
*/
@Public
@Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 6f8c65a..9923806 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -169,7 +169,8 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
* If true, containers for this request may be assigned on hosts
* and racks other than the ones explicitly requested.
* @param nodeLabelsExpression
- * Set node labels to allocate resource
+ * Set node labels to allocate resource, now we only support
+ * asking for only a single node label
*/
public ContainerRequest(Resource capability, String[] nodes,
String[] racks, Priority priority, boolean relaxLocality,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 2fb9ebb..c823474 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -422,6 +422,8 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
checkLocalityRelaxationConflict(req.getPriority(), dedupedRacks, true);
checkLocalityRelaxationConflict(req.getPriority(), inferredRacks,
req.getRelaxLocality());
+ // check if the node label expression specified is valid
+ checkNodeLabelExpression(req);
if (req.getNodes() != null) {
HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes());
@@ -587,6 +589,37 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
}
+ /**
+ * Valid if a node label expression specified on container request is valid or
+ * not
+ *
+ * @param containerRequest
+ */
+ private void checkNodeLabelExpression(T containerRequest) {
+ String exp = containerRequest.getNodeLabelExpression();
+
+ if (null == exp || exp.isEmpty()) {
+ return;
+ }
+
+ // Don't support specifying >= 2 node labels in a node label expression now
+ if (exp.contains("&&") || exp.contains("||")) {
+ throw new InvalidContainerRequestException(
+ "Cannot specify more than two node labels"
+ + " in a single node label expression");
+ }
+
+ // Don't allow specify node label against ANY request
+ if ((containerRequest.getRacks() != null &&
+ (!containerRequest.getRacks().isEmpty()))
+ ||
+ (containerRequest.getNodes() != null &&
+ (!containerRequest.getNodes().isEmpty()))) {
+ throw new InvalidContainerRequestException(
+ "Cannot specify node label with rack and node");
+ }
+ }
+
private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
// This code looks weird but is needed because of the following scenario.
// A ResourceRequest is removed from the remoteRequestTable. A 0 container
@@ -641,7 +674,9 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
resourceRequestInfo.containerRequests.add(req);
}
- resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
+ if (ResourceRequest.ANY.equals(resourceName)) {
+ resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
+ }
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 89d87cf..285295f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLa
import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
@Private
@@ -99,8 +100,11 @@ public class RMAdminCLI extends HAAdmin {
new UsageInfo("[label1,label2,label3] (label splitted by \",\")",
"remove from cluster node labels"))
.put("-replaceLabelsOnNode",
- new UsageInfo("[node1:port,label1,label2 node2:port,label1,label2]",
- "replace labels on nodes"))
+ new UsageInfo(
+ "[node1[:port]=label1,label2 node2[:port]=label1,label2]",
+ "replace labels on nodes"
+ + " (please note that we do not support specifying multiple"
+ + " labels on a single host for now.)"))
.put("-directlyAccessNodeLabelStore",
new UsageInfo("", "Directly access node label store, "
+ "with this option, all node label related operations"
@@ -377,8 +381,7 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
- private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args)
- throws IOException {
+ private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args) {
Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
for (String nodeToLabels : args.split("[ \n]")) {
@@ -389,10 +392,9 @@ public class RMAdminCLI extends HAAdmin {
String[] splits = nodeToLabels.split(",");
String nodeIdStr = splits[0];
-
- if (nodeIdStr.trim().isEmpty()) {
- throw new IOException("node name cannot be empty");
- }
+
+ Preconditions.checkArgument(!nodeIdStr.trim().isEmpty(),
+ "node name cannot be empty");
NodeId nodeId = ConverterUtils.toNodeIdWithDefaultPort(nodeIdStr);
map.put(nodeId, new HashSet<String>());
@@ -402,6 +404,11 @@ public class RMAdminCLI extends HAAdmin {
map.get(nodeId).add(splits[i].trim().toLowerCase());
}
}
+
+ int nLabels = map.get(nodeId).size();
+ Preconditions.checkArgument(nLabels <= 1, "%d labels specified on host=%s"
+ + ", please note that we do not support specifying multiple"
+ + " labels on a single host for now.", nLabels, nodeIdStr);
}
return map;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 3c6918c..7d29d05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.client.api.impl;
-import com.google.common.base.Supplier;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -40,7 +38,6 @@ import java.util.List;
import java.util.Set;
import java.util.TreeSet;
-import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -75,6 +72,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -89,6 +87,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -96,6 +95,8 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.mortbay.log.Log;
+import com.google.common.base.Supplier;
+
public class TestAMRMClient {
static Configuration conf = null;
static MiniYARNCluster yarnCluster = null;
@@ -148,7 +149,6 @@ public class TestAMRMClient {
racks = new String[]{ rack };
}
- @SuppressWarnings("deprecation")
@Before
public void startApp() throws Exception {
// submit new app
@@ -678,21 +678,57 @@ public class TestAMRMClient {
AMRMClientImpl<ContainerRequest> client =
new AMRMClientImpl<ContainerRequest>();
- // add x, y to ANY
+ // add exp=x to ANY
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
- 1), null, null, Priority.UNDEFINED, true, "x && y"));
+ 1), null, null, Priority.UNDEFINED, true, "x"));
Assert.assertEquals(1, client.ask.size());
- Assert.assertEquals("x && y", client.ask.iterator().next()
+ Assert.assertEquals("x", client.ask.iterator().next()
.getNodeLabelExpression());
- // add x, y and a, b to ANY, only a, b should be kept
+ // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
- 1), null, null, Priority.UNDEFINED, true, "x && y"));
+ 1), null, null, Priority.UNDEFINED, true, "x"));
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
- 1), null, null, Priority.UNDEFINED, true, "a && b"));
+ 1), null, null, Priority.UNDEFINED, true, "a"));
Assert.assertEquals(1, client.ask.size());
- Assert.assertEquals("a && b", client.ask.iterator().next()
+ Assert.assertEquals("a", client.ask.iterator().next()
.getNodeLabelExpression());
+
+ // add exp=x to ANY, rack and node, only resource request has ANY resource
+ // name will be assigned the label expression
+ // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
+ client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
+ 1), null, null, Priority.UNDEFINED, true,
+ "y"));
+ Assert.assertEquals(1, client.ask.size());
+ for (ResourceRequest req : client.ask) {
+ if (ResourceRequest.ANY.equals(req.getResourceName())) {
+ Assert.assertEquals("y", req.getNodeLabelExpression());
+ } else {
+ Assert.assertNull(req.getNodeLabelExpression());
+ }
+ }
+ }
+
+ private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
+ ContainerRequest request) {
+ try {
+ client.addContainerRequest(request);
+ } catch (InvalidContainerRequestException e) {
+ return;
+ }
+ Assert.fail();
+ }
+
+ @Test(timeout=30000)
+ public void testAskWithInvalidNodeLabels() {
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
+
+ // specified exp with more than one node labels
+ verifyAddRequestFailed(client,
+ new ContainerRequest(Resource.newInstance(1024, 1), null, null,
+ Priority.UNDEFINED, true, "x && y"));
}
private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index 6176a3e..c8b3a32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -470,7 +470,7 @@ public class TestRMAdminCLI {
// Successfully replace labels
dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
String[] args =
- { "-replaceLabelsOnNode", "node1,x,y node2,y",
+ { "-replaceLabelsOnNode", "node1,x node2,y",
"-directlyAccessNodeLabelStore" };
assertEquals(0, rmAdminCLI.run(args));
assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
@@ -494,7 +494,7 @@ public class TestRMAdminCLI {
// Successfully replace labels
dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
String[] args =
- { "-replaceLabelsOnNode", "node1:8000,x,y node2:8000,y",
+ { "-replaceLabelsOnNode", "node1:8000,x node2:8000,y",
"-directlyAccessNodeLabelStore" };
assertEquals(0, rmAdminCLI.run(args));
assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
@@ -502,6 +502,16 @@ public class TestRMAdminCLI {
assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
NodeId.newInstance("node2", 8000)));
}
+
+ @Test
+ public void testReplaceMultipleLabelsOnSingleNode() throws Exception {
+ // Successfully replace labels
+ dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
+ String[] args =
+ { "-replaceLabelsOnNode", "node1,x,y",
+ "-directlyAccessNodeLabelStore" };
+ assertTrue(0 != rmAdminCLI.run(args));
+ }
private void testError(String[] args, String template,
ByteArrayOutputStream data, int resultCode) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index daefe8d..b30689b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -43,11 +43,12 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels;
import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEvent;
import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEventType;
import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels;
+import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels;
import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -287,14 +288,35 @@ public class CommonNodeLabelsManager extends AbstractService {
// check all labels being added existed
Set<String> knownLabels = labelCollections.keySet();
for (Entry<NodeId, Set<String>> entry : addedLabelsToNode.entrySet()) {
- if (!knownLabels.containsAll(entry.getValue())) {
+ NodeId nodeId = entry.getKey();
+ Set<String> labels = entry.getValue();
+
+ if (!knownLabels.containsAll(labels)) {
String msg =
"Not all labels being added contained by known "
+ "label collections, please check" + ", added labels=["
- + StringUtils.join(entry.getValue(), ",") + "]";
+ + StringUtils.join(labels, ",") + "]";
LOG.error(msg);
throw new IOException(msg);
}
+
+ // In YARN-2694, we temporarily disable user add more than 1 labels on a
+ // same host
+ if (!labels.isEmpty()) {
+ Set<String> newLabels = new HashSet<String>(getLabelsByNode(nodeId));
+ newLabels.addAll(labels);
+ // we don't allow number of labels on a node > 1 after added labels
+ if (newLabels.size() > 1) {
+ String msg =
+ String.format(
+ "%d labels specified on host=%s after add labels to node"
+ + ", please note that we do not support specifying multiple"
+ + " labels on a single host for now.",
+ newLabels.size(), nodeId.getHost());
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ }
}
}
@@ -537,11 +559,24 @@ public class CommonNodeLabelsManager extends AbstractService {
// check all labels being added existed
Set<String> knownLabels = labelCollections.keySet();
for (Entry<NodeId, Set<String>> entry : replaceLabelsToNode.entrySet()) {
- if (!knownLabels.containsAll(entry.getValue())) {
+ NodeId nodeId = entry.getKey();
+ Set<String> labels = entry.getValue();
+
+ // As in YARN-2694, we disable user add more than 1 labels on a same host
+ if (labels.size() > 1) {
+ String msg = String.format("%d labels specified on host=%s"
+ + ", please note that we do not support specifying multiple"
+ + " labels on a single host for now.", labels.size(),
+ nodeId.getHost());
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+
+ if (!knownLabels.containsAll(labels)) {
String msg =
"Not all labels being replaced contained by known "
+ "label collections, please check" + ", new labels=["
- + StringUtils.join(entry.getValue(), ",") + "]";
+ + StringUtils.join(labels, ",") + "]";
LOG.error(msg);
throw new IOException(msg);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
index a56a595..2fb1676 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
@@ -221,19 +221,17 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
ImmutableMap.of(toNodeId("n1"), CommonNodeLabelsManager.EMPTY_STRING_SET));
// add label on node
- mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
- toNodeId("n2"), toSet("p2")));
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
assertMapEquals(
mgr.getNodeLabels(),
ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"),
- toSet("p2", "p3"), toNodeId("n3"), toSet("p3")));
+ toSet("p3"), toNodeId("n3"), toSet("p3")));
assertMapEquals(mgr.lastNodeToLabels,
- ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"),
- toSet("p2", "p3")));
+ ImmutableMap.of(toNodeId("n1"), toSet("p1")));
// remove labels on node
mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
- toNodeId("n2"), toSet("p2", "p3"), toNodeId("n3"), toSet("p3")));
+ toNodeId("n2"), toSet("p3"), toNodeId("n3"), toSet("p3")));
Assert.assertEquals(0, mgr.getNodeLabels().size());
assertMapEquals(mgr.lastNodeToLabels, ImmutableMap.of(toNodeId("n1"),
CommonNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n2"),
@@ -270,10 +268,10 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
@Test(timeout = 5000)
public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
mgr.addToCluserNodeLabels(toSet(" p1", "p2"));
- mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ", "p2")));
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ")));
assertMapEquals(
mgr.getNodeLabels(),
- ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+ ImmutableMap.of(toNodeId("n1"), toSet("p1")));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2")));
assertMapEquals(
mgr.getNodeLabels(),
@@ -281,4 +279,37 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2 ")));
Assert.assertTrue(mgr.getNodeLabels().isEmpty());
}
+
+ @Test(timeout = 5000)
+ public void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
+ boolean failed = false;
+ // As in YARN-2694, we temporarily disable no more than one label existed in
+ // one host
+ mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+ try {
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+ } catch (IOException e) {
+ failed = true;
+ }
+ Assert.assertTrue("Should failed when set > 1 labels on a host", failed);
+
+ try {
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+ } catch (IOException e) {
+ failed = true;
+ }
+ Assert.assertTrue("Should failed when add > 1 labels on a host", failed);
+
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+ // add a same label to a node, #labels in this node is still 1, shouldn't
+ // fail
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+ try {
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p2")));
+ } catch (IOException e) {
+ failed = true;
+ }
+ Assert.assertTrue("Should failed when #labels > 1 on a host after add",
+ failed);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 2a4166c..c896bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -488,10 +488,11 @@ public class ApplicationMasterService extends AbstractService implements
RMApp app =
this.rmContext.getRMApps().get(applicationId);
- // set label expression for Resource Requests
+ // set label expression for Resource Requests if resourceName=ANY
ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
for (ResourceRequest req : ask) {
- if (null == req.getNodeLabelExpression()) {
+ if (null == req.getNodeLabelExpression()
+ && ResourceRequest.ANY.equals(req.getResourceName())) {
req.setNodeLabelExpression(asc.getNodeLabelExpression());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c41aae0..bdc9da1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -262,6 +262,25 @@ public class SchedulerUtils {
}
String labelExp = resReq.getNodeLabelExpression();
+ // we don't allow specify label expression other than resourceName=ANY now
+ if (!ResourceRequest.ANY.equals(resReq.getResourceName())
+ && labelExp != null && !labelExp.trim().isEmpty()) {
+ throw new InvalidResourceRequestException(
+ "Invailid resource request, queue=" + queueInfo.getQueueName()
+ + " specified node label expression in a "
+ + "resource request has resource name = "
+ + resReq.getResourceName());
+ }
+
+ // we don't allow specify label expression with more than one node labels now
+ if (labelExp != null && labelExp.contains("&&")) {
+ throw new InvalidResourceRequestException(
+ "Invailid resource request, queue=" + queueInfo.getQueueName()
+ + " specified more than one node label "
+ + "in a node label expression, node label expression = "
+ + labelExp);
+ }
+
if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
labelExp)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index e2e3cc1..a3968f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -165,12 +165,17 @@ public class MockAM {
int containers, String labelExpression) throws Exception {
List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
for (String host : hosts) {
- ResourceRequest hostReq = createResourceReq(host, memory, priority,
- containers, labelExpression);
- reqs.add(hostReq);
- ResourceRequest rackReq = createResourceReq("/default-rack", memory,
- priority, containers, labelExpression);
- reqs.add(rackReq);
+ // only add host/rack request when asked host isn't ANY
+ if (!host.equals(ResourceRequest.ANY)) {
+ ResourceRequest hostReq =
+ createResourceReq(host, memory, priority, containers,
+ labelExpression);
+ reqs.add(hostReq);
+ ResourceRequest rackReq =
+ createResourceReq("/default-rack", memory, priority, containers,
+ labelExpression);
+ reqs.add(rackReq);
+ }
}
ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index ed675f3..7c3d9ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -206,16 +206,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
/*
* Node->Labels:
- * host1 : red, blue
- * host2 : blue, yellow
+ * host1 : red
+ * host2 : blue
* host3 : yellow
* host4 :
*/
mgr.addToCluserNodeLabels(toSet("red", "blue", "yellow"));
- mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"),
- toSet("red", "blue")));
- mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"),
- toSet("blue", "yellow")));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red")));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue")));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow")));
// active two NM to n1, one large and one small
@@ -243,31 +241,29 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
- mgr.getQueueResource("Q2", q2Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ mgr.getQueueResource("Q2", q2Label, clusterResource));
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q4", q4Label, clusterResource));
Assert.assertEquals(clusterResource,
mgr.getQueueResource("Q5", q5Label, clusterResource));
- mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host1"), toSet("red"),
- toNodeId("host2"), toSet("blue", "yellow")));
- mgr.addLabelsToNode(ImmutableMap.of(toNodeId("host3"), toSet("red")));
+ mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host2"), toSet("blue")));
/*
* Check resource after changes some labels
* Node->Labels:
- * host1 : blue (was: red, blue)
- * host2 : (was: blue, yellow)
- * host3 : red, yellow (was: yellow)
+ * host1 : red
+ * host2 : (was: blue)
+ * host3 : yellow
* host4 :
*/
// check resource
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q2", q2Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q3", q3Label, clusterResource));
@@ -279,9 +275,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
/*
* Check resource after deactive/active some nodes
* Node->Labels:
- * (deactived) host1 : blue
+ * (deactived) host1 : red
* host2 :
- * (deactived and then actived) host3 : red, yellow
+ * (deactived and then actived) host3 : yellow
* host4 :
*/
mgr.deactivateNode(NodeId.newInstance("host1", 1));
@@ -289,7 +285,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
mgr.activateNode(NodeId.newInstance("host3", 1), SMALL_RESOURCE);
// check resource
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q1", q1Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q2", q2Label, clusterResource));
@@ -326,9 +322,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q2", q2Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -339,7 +335,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
* Active NMs in nodes already have NM
* Node->Labels:
* host2 :
- * host3 : red, yellow (3 NMs)
+ * host3 : yellow (3 NMs)
* host4 : (2 NMs)
*/
mgr.activateNode(NodeId.newInstance("host3", 2), SMALL_RESOURCE);
@@ -349,9 +345,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 6),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q2", q2Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 6),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -362,7 +358,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
* Deactive NMs in nodes already have NMs
* Node->Labels:
* host2 :
- * host3 : red, yellow (2 NMs)
+ * host3 : yellow (2 NMs)
* host4 : (0 NMs)
*/
mgr.deactivateNode(NodeId.newInstance("host3", 3));
@@ -372,9 +368,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q2", q2Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q4", q4Label, clusterResource));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index a7c7ad6..8abaeb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -214,11 +214,7 @@ public class TestSchedulerUtils {
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
-
- resReq.setNodeLabelExpression("x && y");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
-
+
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
@@ -253,6 +249,8 @@ public class TestSchedulerUtils {
} catch (InvalidResourceRequestException e) {
}
+ // we don't allow specify more than two node labels in a single expression
+ // now
try {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
@@ -263,7 +261,7 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
- resReq.setNodeLabelExpression("x && y && z");
+ resReq.setNodeLabelExpression("x && y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
fail("Should fail");
@@ -328,7 +326,7 @@ public class TestSchedulerUtils {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
- resReq.setNodeLabelExpression("x && y && z");
+ resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
@@ -337,7 +335,45 @@ public class TestSchedulerUtils {
scheduler);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
- fail("Should be valid when request labels is empty");
+ fail("Should be valid when queue can access any labels");
+ }
+
+ // we don't allow resource name other than ANY and specify label
+ try {
+ // set queue accessible node labesl to [x, y]
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), "rack", resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
+ }
+
+ // we don't allow resource name other than ANY and specify label even if
+ // queue has accessible label = *
+ try {
+ // set queue accessible node labesl to *
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays
+ .asList(CommonNodeLabelsManager.ANY));
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), "rack", resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index ad834ac..a9f9420 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -87,6 +87,7 @@ public class TestContainerAllocation {
@Test(timeout = 3000000)
public void testExcessReservationThanNodeManagerCapacity() throws Exception {
+ @SuppressWarnings("resource")
MockRM rm = new MockRM(conf);
rm.start();
@@ -393,6 +394,7 @@ public class TestContainerAllocation {
}
}
+ @SuppressWarnings("unchecked")
private <E> Set<E> toSet(E... elements) {
Set<E> set = Sets.newHashSet(elements);
return set;
@@ -449,7 +451,7 @@ public class TestContainerAllocation {
return conf;
}
- @Test(timeout = 300000)
+ @Test (timeout = 300000)
public void testContainerAllocationWithSingleUserLimits() throws Exception {
final RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
mgr.init(conf);
@@ -470,7 +472,7 @@ public class TestContainerAllocation {
rm1.getRMContext().setNodeLabelManager(mgr);
rm1.start();
MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x
- MockNM nm2 = rm1.registerNode("h2:1234", 8000); // label = y
+ rm1.registerNode("h2:1234", 8000); // label = y
MockNM nm3 = rm1.registerNode("h3:1234", 8000); // label = <empty>
// launch an app to queue a1 (label = x), and check all container will
@@ -518,9 +520,9 @@ public class TestContainerAllocation {
*
* Node structure:
* h1 : x
- * h2 : x, y
+ * h2 : y
* h3 : y
- * h4 : y, z
+ * h4 : z
* h5 : NO
*
* Total resource:
@@ -540,9 +542,9 @@ public class TestContainerAllocation {
// set node -> label
mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));
mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0),
- toSet("x"), NodeId.newInstance("h2", 0), toSet("x", "y"),
+ toSet("x"), NodeId.newInstance("h2", 0), toSet("y"),
NodeId.newInstance("h3", 0), toSet("y"), NodeId.newInstance("h4", 0),
- toSet("y", "z"), NodeId.newInstance("h5", 0),
+ toSet("z"), NodeId.newInstance("h5", 0),
RMNodeLabelsManager.EMPTY_STRING_SET));
// inject node label manager
@@ -568,12 +570,10 @@ public class TestContainerAllocation {
RMApp app1 = rm1.submitApp(1024, "app", "user", null, "a1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
- // request a container (label = x && y). can only allocate on nm2
- am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "x && y");
+ // request a container (label = y). can be allocated on nm2
+ am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "y");
containerId =
- ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
- Assert.assertFalse(rm1.waitForState(nm1, containerId,
- RMContainerState.ALLOCATED, 10 * 1000));
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L);
Assert.assertTrue(rm1.waitForState(nm2, containerId,
RMContainerState.ALLOCATED, 10 * 1000));
checkTaskContainersHost(am1.getApplicationAttemptId(), containerId, rm1,
@@ -609,12 +609,10 @@ public class TestContainerAllocation {
checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1,
"h3");
- // try to allocate container (request label = y && z) on nm3 (label = y) and
- // nm4 (label = y,z). Will sucessfully allocate on nm4 only.
- am3.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "y && z");
- containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3);
- Assert.assertFalse(rm1.waitForState(nm3, containerId,
- RMContainerState.ALLOCATED, 10 * 1000));
+ // try to allocate container (request label = z) on nm4 (label = y,z).
+ // Will successfully allocate on nm4 only.
+ am3.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "z");
+ containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3L);
Assert.assertTrue(rm1.waitForState(nm4, containerId,
RMContainerState.ALLOCATED, 10 * 1000));
checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index df5aecb..5a0bdd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -155,7 +155,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.path("replace-labels")
.queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+ .entity("{\"nodeLabels\": [\"a\"]}",
MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
LOG.info("posted node nodelabel");
@@ -168,8 +168,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
- jarr = json.getJSONArray("nodeLabels");
- assertEquals(2, jarr.length());
+ assertEquals("a", json.getString("nodeLabels"));
+
// Replace
response =
@@ -178,9 +178,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.path("replace-labels")
.queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
LOG.info("posted node nodelabel");
+
// Verify
response =
r.path("ws").path("v1").path("cluster")
@@ -189,13 +190,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
- assertEquals("a", json.getString("nodeLabels"));
+ assertEquals("b", json.getString("nodeLabels"));
// Replace labels using node-to-labels
NodeToLabelsInfo ntli = new NodeToLabelsInfo();
NodeLabelsInfo nli = new NodeLabelsInfo();
nli.getNodeLabels().add("a");
- nli.getNodeLabels().add("b");
ntli.getNodeToLabels().put("nid:0", nli);
response =
r.path("ws").path("v1").path("cluster")
@@ -214,9 +214,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
ntli = response.getEntity(NodeToLabelsInfo.class);
nli = ntli.getNodeToLabels().get("nid:0");
- assertEquals(2, nli.getNodeLabels().size());
+ assertEquals(1, nli.getNodeLabels().size());
assertTrue(nli.getNodeLabels().contains("a"));
- assertTrue(nli.getNodeLabels().contains("b"));
// Remove all
response =
@@ -267,7 +266,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.path("replace-labels")
.queryParam("user.name", notUserName)
.accept(MediaType.APPLICATION_JSON)
- .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+ .entity("{\"nodeLabels\": [\"b\"]}",
MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
// Verify
[7/8] hadoop git commit: YARN-3124. Fixed CS LeafQueue/ParentQueue to
use QueueCapacities to track capacities-by-label. Contributed by Wangda Tan
(cherry picked from commit 18a594257e052e8f10a03e5594e6cc6901dc56be)
Posted by vi...@apache.org.
YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track capacities-by-label. Contributed by Wangda Tan
(cherry picked from commit 18a594257e052e8f10a03e5594e6cc6901dc56be)
(cherry picked from commit 1be2d64ddddaa6322909073cfaf7f2f2eb46e18d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee2b6bc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee2b6bc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee2b6bc2
Branch: refs/heads/branch-2.6.1
Commit: ee2b6bc248eaa3d204bd6985d44ac8aea56b9728
Parents: 637e7f9
Author: Jian He <ji...@apache.org>
Authored: Thu Feb 12 14:58:09 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sun Sep 6 11:54:40 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 2 +
.../scheduler/capacity/AbstractCSQueue.java | 229 ++++++-------------
.../scheduler/capacity/CSQueue.java | 41 +---
.../scheduler/capacity/CSQueueUtils.java | 160 +++++++++----
.../CapacitySchedulerConfiguration.java | 73 ++----
.../scheduler/capacity/LeafQueue.java | 208 +++++------------
.../scheduler/capacity/ParentQueue.java | 92 ++------
.../scheduler/capacity/PlanQueue.java | 13 +-
.../scheduler/capacity/QueueCapacities.java | 68 +++++-
.../scheduler/capacity/ReservationQueue.java | 10 +-
.../scheduler/capacity/TestCSQueueUtils.java | 24 +-
.../capacity/TestCapacityScheduler.java | 6 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 3 +-
.../scheduler/capacity/TestQueueCapacities.java | 2 +-
.../scheduler/capacity/TestQueueParsing.java | 14 +-
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
17 files changed, 412 insertions(+), 540 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2c654dc..a6bbdf3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -189,6 +189,9 @@ Release 2.6.1 - UNRELEASED
YARN-2694. Ensure only single node label specified in ResourceRequest.
(Wangda Tan via jianhe)
+ YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track
+ capacities-by-label. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 971acea..ece5cff 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -204,6 +204,8 @@
<Field name="absoluteNodeLabelCapacities" />
<Field name="reservationsContinueLooking" />
<Field name="absoluteCapacityByNodeLabels" />
+ <Field name="authorizer" />
+ <Field name="parent" />
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 3c1663f..52629b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -39,25 +39,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
public abstract class AbstractCSQueue implements CSQueue {
CSQueue parent;
final String queueName;
-
- float capacity;
- float maximumCapacity;
- float absoluteCapacity;
- float absoluteMaxCapacity;
- float absoluteUsedCapacity = 0.0f;
- float usedCapacity = 0.0f;
volatile int numContainers;
- final Resource minimumAllocation;
- final Resource maximumAllocation;
+ Resource minimumAllocation;
+ Resource maximumAllocation;
QueueState state;
final QueueMetrics metrics;
@@ -65,10 +57,6 @@ public abstract class AbstractCSQueue implements CSQueue {
Set<String> accessibleLabels;
RMNodeLabelsManager labelManager;
String defaultLabelExpression;
- Map<String, Float> absoluteCapacityByNodeLabels;
- Map<String, Float> capacitiyByNodeLabels;
- Map<String, Float> absoluteMaxCapacityByNodeLabels;
- Map<String, Float> maxCapacityByNodeLabels;
Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
@@ -77,13 +65,16 @@ public abstract class AbstractCSQueue implements CSQueue {
// Track resource usage-by-label like used-resource/pending-resource, etc.
ResourceUsage queueUsage;
+ // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
+ // etc.
+ QueueCapacities queueCapacities;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
+ protected CapacitySchedulerContext csContext;
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
- this.minimumAllocation = cs.getMinimumResourceCapability();
- this.maximumAllocation = cs.getMaximumResourceCapability();
this.labelManager = cs.getRMContext().getNodeLabelManager();
this.parent = parent;
this.queueName = queueName;
@@ -94,65 +85,53 @@ public abstract class AbstractCSQueue implements CSQueue {
QueueMetrics.forQueue(getQueuePath(), parent,
cs.getConfiguration().getEnableUserMetrics(),
cs.getConf());
-
- // get labels
- this.accessibleLabels = cs.getConfiguration().getAccessibleNodeLabels(getQueuePath());
- this.defaultLabelExpression = cs.getConfiguration()
- .getDefaultNodeLabelExpression(getQueuePath());
-
- // inherit from parent if labels not set
- if (this.accessibleLabels == null && parent != null) {
- this.accessibleLabels = parent.getAccessibleNodeLabels();
- }
- SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
- this.accessibleLabels);
-
- // inherit from parent if labels not set
- if (this.defaultLabelExpression == null && parent != null
- && this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
- this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
- }
-
- // set capacity by labels
- capacitiyByNodeLabels =
- cs.getConfiguration().getNodeLabelCapacities(getQueuePath(), accessibleLabels,
- labelManager);
+ this.csContext = cs;
- // set maximum capacity by labels
- maxCapacityByNodeLabels =
- cs.getConfiguration().getMaximumNodeLabelCapacities(getQueuePath(),
- accessibleLabels, labelManager);
+ // initialize ResourceUsage
queueUsage = new ResourceUsage();
+
+ // initialize QueueCapacities
+ queueCapacities = new QueueCapacities(parent == null);
+ }
+
+ protected void setupConfigurableCapacities() {
+ CSQueueUtils.loadUpdateAndCheckCapacities(
+ getQueuePath(),
+ accessibleLabels,
+ csContext.getConfiguration(),
+ queueCapacities,
+ parent == null ? null : parent.getQueueCapacities(),
+ csContext.getRMContext().getNodeLabelManager());
}
@Override
public synchronized float getCapacity() {
- return capacity;
+ return queueCapacities.getCapacity();
}
@Override
public synchronized float getAbsoluteCapacity() {
- return absoluteCapacity;
+ return queueCapacities.getAbsoluteCapacity();
}
@Override
public float getAbsoluteMaximumCapacity() {
- return absoluteMaxCapacity;
+ return queueCapacities.getAbsoluteMaximumCapacity();
}
@Override
public synchronized float getAbsoluteUsedCapacity() {
- return absoluteUsedCapacity;
+ return queueCapacities.getAbsoluteUsedCapacity();
}
@Override
public float getMaximumCapacity() {
- return maximumCapacity;
+ return queueCapacities.getMaximumCapacity();
}
@Override
public synchronized float getUsedCapacity() {
- return usedCapacity;
+ return queueCapacities.getUsedCapacity();
}
@Override
@@ -210,12 +189,12 @@ public abstract class AbstractCSQueue implements CSQueue {
@Override
public synchronized void setUsedCapacity(float usedCapacity) {
- this.usedCapacity = usedCapacity;
+ queueCapacities.setUsedCapacity(usedCapacity);
}
@Override
public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) {
- this.absoluteUsedCapacity = absUsedCapacity;
+ queueCapacities.setAbsoluteUsedCapacity(absUsedCapacity);
}
/**
@@ -224,21 +203,16 @@ public abstract class AbstractCSQueue implements CSQueue {
*/
synchronized void setMaxCapacity(float maximumCapacity) {
// Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
+ CSQueueUtils.checkMaxCapacity(getQueueName(),
+ queueCapacities.getCapacity(), maximumCapacity);
float absMaxCapacity =
CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absoluteCapacity,
+ CSQueueUtils.checkAbsoluteCapacity(getQueueName(),
+ queueCapacities.getAbsoluteCapacity(),
absMaxCapacity);
- this.maximumCapacity = maximumCapacity;
- this.absoluteMaxCapacity = absMaxCapacity;
- }
-
- @Override
- public float getAbsActualCapacity() {
- // for now, simply return actual capacity = guaranteed capacity for parent
- // queue
- return absoluteCapacity;
+ queueCapacities.setMaximumCapacity(maximumCapacity);
+ queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
}
@Override
@@ -246,39 +220,35 @@ public abstract class AbstractCSQueue implements CSQueue {
return defaultLabelExpression;
}
- synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
- float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity,
- QueueState state, Map<QueueACL, AccessControlList> acls,
- Set<String> labels, String defaultLabelExpression,
- Map<String, Float> nodeLabelCapacities,
- Map<String, Float> maximumNodeLabelCapacities,
- boolean reservationContinueLooking)
+ synchronized void setupQueueConfigs(Resource clusterResource)
throws IOException {
- // Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absoluteCapacity,
- absoluteMaxCapacity);
-
- this.capacity = capacity;
- this.absoluteCapacity = absoluteCapacity;
-
- this.maximumCapacity = maximumCapacity;
- this.absoluteMaxCapacity = absoluteMaxCapacity;
-
- this.state = state;
+ // get labels
+ this.accessibleLabels =
+ csContext.getConfiguration().getAccessibleNodeLabels(getQueuePath());
+ this.defaultLabelExpression = csContext.getConfiguration()
+ .getDefaultNodeLabelExpression(getQueuePath());
- this.acls = acls;
+ // inherit from parent if labels not set
+ if (this.accessibleLabels == null && parent != null) {
+ this.accessibleLabels = parent.getAccessibleNodeLabels();
+ }
+ SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
+ this.accessibleLabels);
- // set labels
- this.accessibleLabels = labels;
+ // inherit from parent if labels not set
+ if (this.defaultLabelExpression == null && parent != null
+ && this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
+ this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
+ }
+
+ // After we setup labels, we can setup capacities
+ setupConfigurableCapacities();
- // set label expression
- this.defaultLabelExpression = defaultLabelExpression;
+ this.minimumAllocation = csContext.getMinimumResourceCapability();
+ this.maximumAllocation = csContext.getMaximumResourceCapability();
- // copy node label capacity
- this.capacitiyByNodeLabels = new HashMap<String, Float>(nodeLabelCapacities);
- this.maxCapacityByNodeLabels =
- new HashMap<String, Float>(maximumNodeLabelCapacities);
+ this.state = csContext.getConfiguration().getState(getQueuePath());
+ this.acls = csContext.getConfiguration().getAcls(getQueuePath());
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -305,30 +275,18 @@ public abstract class AbstractCSQueue implements CSQueue {
}
}
}
-
- // calculate absolute capacity by each node label
- this.absoluteCapacityByNodeLabels =
- CSQueueUtils.computeAbsoluteCapacityByNodeLabels(
- this.capacitiyByNodeLabels, parent);
-
- // calculate maximum capacity by each node label
- this.absoluteMaxCapacityByNodeLabels =
- CSQueueUtils.computeAbsoluteMaxCapacityByNodeLabels(
- maximumNodeLabelCapacities, parent);
-
- // check absoluteMaximumNodeLabelCapacities is valid
- CSQueueUtils.checkAbsoluteCapacitiesByLabel(getQueueName(),
- absoluteCapacityByNodeLabels, absoluteCapacityByNodeLabels);
-
- this.reservationsContinueLooking = reservationContinueLooking;
+
+ this.reservationsContinueLooking = csContext.getConfiguration()
+ .getReservationContinueLook();
+
}
protected QueueInfo getQueueInfo() {
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(queueName);
queueInfo.setAccessibleNodeLabels(accessibleLabels);
- queueInfo.setCapacity(capacity);
- queueInfo.setMaximumCapacity(maximumCapacity);
+ queueInfo.setCapacity(queueCapacities.getCapacity());
+ queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity());
queueInfo.setQueueState(state);
queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
queueInfo.setCurrentCapacity(getUsedCapacity());
@@ -379,51 +337,6 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Private
- public float getCapacityByNodeLabel(String label) {
- if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) {
- if (null == parent) {
- return 1f;
- }
- return getCapacity();
- }
-
- if (!capacitiyByNodeLabels.containsKey(label)) {
- return 0f;
- } else {
- return capacitiyByNodeLabels.get(label);
- }
- }
-
- @Private
- public float getAbsoluteCapacityByNodeLabel(String label) {
- if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) {
- if (null == parent) {
- return 1f;
- }
- return getAbsoluteCapacity();
- }
-
- if (!absoluteCapacityByNodeLabels.containsKey(label)) {
- return 0f;
- } else {
- return absoluteCapacityByNodeLabels.get(label);
- }
- }
-
- @Private
- public float getAbsoluteMaximumCapacityByNodeLabel(String label) {
- if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) {
- return getAbsoluteMaximumCapacity();
- }
-
- if (!absoluteMaxCapacityByNodeLabels.containsKey(label)) {
- return 0f;
- } else {
- return absoluteMaxCapacityByNodeLabels.get(label);
- }
- }
-
- @Private
public boolean getReservationContinueLooking() {
return reservationsContinueLooking;
}
@@ -432,14 +345,14 @@ public abstract class AbstractCSQueue implements CSQueue {
public Map<QueueACL, AccessControlList> getACLs() {
return acls;
}
-
+
@Private
- public Resource getUsedResourceByLabel(String nodeLabel) {
- return queueUsage.getUsed(nodeLabel);
+ public QueueCapacities getQueueCapacities() {
+ return queueCapacities;
}
- @VisibleForTesting
- public ResourceUsage getResourceUsage() {
+ @Private
+ public ResourceUsage getQueueResourceUsage() {
return queueUsage;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 07a7e0e..1dfb872 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -75,15 +76,6 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
* @return configured queue capacity
*/
public float getCapacity();
-
- /**
- * Get actual <em>capacity</em> of the queue, this may be different from
- * configured capacity when mis-config take place, like add labels to the
- * cluster
- *
- * @return actual queue capacity
- */
- public float getAbsActualCapacity();
/**
* Get capacity of the parent of the queue as a function of the
@@ -144,14 +136,6 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
public Resource getUsedResources();
/**
- * Get the currently utilized resources which allocated at nodes with label
- * specified
- *
- * @return used resources by the queue and it's children
- */
- public Resource getUsedResourceByLabel(String nodeLabel);
-
- /**
* Get the current run-state of the queue
* @return current run-state
*/
@@ -279,25 +263,16 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*/
public void attachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer container);
-
+
/**
- * Get absolute capacity by label of this queue can use
- * @param nodeLabel
- * @return absolute capacity by label of this queue can use
+ * Get QueueCapacities of this queue
+ * @return queueCapacities
*/
- public float getAbsoluteCapacityByNodeLabel(String nodeLabel);
+ public QueueCapacities getQueueCapacities();
/**
- * Get absolute max capacity by label of this queue can use
- * @param nodeLabel
- * @return absolute capacity by label of this queue can use
- */
- public float getAbsoluteMaximumCapacityByNodeLabel(String nodeLabel);
-
- /**
- * Get capacity by node label
- * @param nodeLabel
- * @return capacity by node label
+ * Get ResourceUsage of this queue
+ * @return resourceUsage
*/
- public float getCapacityByNodeLabel(String nodeLabel);
+ public ResourceUsage getQueueResourceUsage();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index f458057..865b0b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -17,13 +17,14 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -34,6 +35,9 @@ class CSQueueUtils {
final static float EPSILON = 0.0001f;
+ /*
+ * Used only by tests
+ */
public static void checkMaxCapacity(String queueName,
float capacity, float maximumCapacity) {
if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) {
@@ -43,6 +47,9 @@ class CSQueueUtils {
}
}
+ /*
+ * Used only by tests
+ */
public static void checkAbsoluteCapacity(String queueName,
float absCapacity, float absMaxCapacity) {
if (absMaxCapacity < (absCapacity - EPSILON)) {
@@ -53,19 +60,33 @@ class CSQueueUtils {
}
}
- public static void checkAbsoluteCapacitiesByLabel(String queueName,
- Map<String, Float> absCapacities,
- Map<String, Float> absMaximumCapacities) {
- for (Entry<String, Float> entry : absCapacities.entrySet()) {
- String label = entry.getKey();
- float absCapacity = entry.getValue();
- float absMaxCapacity = absMaximumCapacities.get(label);
- if (absMaxCapacity < (absCapacity - EPSILON)) {
- throw new IllegalArgumentException("Illegal call to setMaxCapacity. "
- + "Queue '" + queueName + "' has " + "an absolute capacity ("
- + absCapacity + ") greater than "
- + "its absolute maximumCapacity (" + absMaxCapacity + ") of label="
- + label);
+ /**
+ * Check sanity of capacities:
+ * - capacity <= maxCapacity
+ * - absCapacity <= absMaximumCapacity
+ */
+ private static void capacitiesSanityCheck(String queueName,
+ QueueCapacities queueCapacities) {
+ for (String label : queueCapacities.getExistingNodeLabels()) {
+ float capacity = queueCapacities.getCapacity(label);
+ float maximumCapacity = queueCapacities.getMaximumCapacity(label);
+ if (capacity > maximumCapacity) {
+ throw new IllegalArgumentException("Illegal queue capacity setting, "
+ + "(capacity=" + capacity + ") > (maximum-capacity="
+ + maximumCapacity + "). When label=[" + label + "]");
+ }
+
+ // Actually, this may not needed since we have verified capacity <=
+ // maximumCapacity. And the way we compute absolute capacity (abs(x) =
+ // cap(x) * cap(x.parent) * ...) is a monotone increasing function. But
+ // just keep it here to make sure our compute abs capacity method works
+ // correctly.
+ float absCapacity = queueCapacities.getAbsoluteCapacity(label);
+ float absMaxCapacity = queueCapacities.getAbsoluteMaximumCapacity(label);
+ if (absCapacity > absMaxCapacity) {
+ throw new IllegalArgumentException("Illegal queue capacity setting, "
+ + "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity="
+ + absMaxCapacity + "). When label=[" + label + "]");
}
}
}
@@ -77,37 +98,94 @@ class CSQueueUtils {
return (parentAbsMaxCapacity * maximumCapacity);
}
- public static Map<String, Float> computeAbsoluteCapacityByNodeLabels(
- Map<String, Float> nodeLabelToCapacities, CSQueue parent) {
- if (parent == null) {
- return nodeLabelToCapacities;
+ /**
+ * This method intends to be used by ReservationQueue, ReservationQueue will
+ * not appear in configuration file, so we shouldn't do load capacities
+ * settings in configuration for reservation queue.
+ */
+ public static void updateAndCheckCapacitiesByLabel(String queuePath,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
+
+ capacitiesSanityCheck(queuePath, queueCapacities);
+ }
+
+ /**
+ * Do following steps for capacities
+ * - Load capacities from configuration
+ * - Update absolute capacities for new capacities
+ * - Check if capacities/absolute-capacities legal
+ */
+ public static void loadUpdateAndCheckCapacities(String queuePath,
+ Set<String> accessibleLabels, CapacitySchedulerConfiguration csConf,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities,
+ RMNodeLabelsManager nlm) {
+ loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm,
+ queueCapacities, csConf);
+
+ updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
+
+ capacitiesSanityCheck(queuePath, queueCapacities);
+ }
+
+ // Considered NO_LABEL, ANY and null cases
+ private static Set<String> normalizeAccessibleNodeLabels(Set<String> labels,
+ RMNodeLabelsManager mgr) {
+ Set<String> accessibleLabels = new HashSet<String>();
+ if (labels != null) {
+ accessibleLabels.addAll(labels);
}
-
- Map<String, Float> absoluteCapacityByNodeLabels =
- new HashMap<String, Float>();
- for (Entry<String, Float> entry : nodeLabelToCapacities.entrySet()) {
- String label = entry.getKey();
- float capacity = entry.getValue();
- absoluteCapacityByNodeLabels.put(label,
- capacity * parent.getAbsoluteCapacityByNodeLabel(label));
+ if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) {
+ accessibleLabels.addAll(mgr.getClusterNodeLabels());
}
- return absoluteCapacityByNodeLabels;
+ accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL);
+
+ return accessibleLabels;
}
- public static Map<String, Float> computeAbsoluteMaxCapacityByNodeLabels(
- Map<String, Float> maximumNodeLabelToCapacities, CSQueue parent) {
- if (parent == null) {
- return maximumNodeLabelToCapacities;
+ private static void loadCapacitiesByLabelsFromConf(String queuePath,
+ Set<String> labels, RMNodeLabelsManager mgr,
+ QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) {
+ queueCapacities.clearConfigurableFields();
+ labels = normalizeAccessibleNodeLabels(labels, mgr);
+
+ for (String label : labels) {
+ if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
+ queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL,
+ csConf.getNonLabeledQueueCapacity(queuePath) / 100);
+ queueCapacities.setMaximumCapacity(CommonNodeLabelsManager.NO_LABEL,
+ csConf.getNonLabeledQueueMaximumCapacity(queuePath) / 100);
+ } else {
+ queueCapacities.setCapacity(label,
+ csConf.getLabeledQueueCapacity(queuePath, label) / 100);
+ queueCapacities.setMaximumCapacity(label,
+ csConf.getLabeledQueueMaximumCapacity(queuePath, label) / 100);
+ }
}
- Map<String, Float> absoluteMaxCapacityByNodeLabels =
- new HashMap<String, Float>();
- for (Entry<String, Float> entry : maximumNodeLabelToCapacities.entrySet()) {
- String label = entry.getKey();
- float maxCapacity = entry.getValue();
- absoluteMaxCapacityByNodeLabels.put(label,
- maxCapacity * parent.getAbsoluteMaximumCapacityByNodeLabel(label));
+ }
+
+ // Set absolute capacities for {capacity, maximum-capacity}
+ private static void updateAbsoluteCapacitiesByNodeLabels(
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ for (String label : queueCapacities.getExistingNodeLabels()) {
+ float capacity = queueCapacities.getCapacity(label);
+ if (capacity > 0f) {
+ queueCapacities.setAbsoluteCapacity(
+ label,
+ capacity
+ * (parentQueueCapacities == null ? 1 : parentQueueCapacities
+ .getAbsoluteCapacity(label)));
+ }
+
+ float maxCapacity = queueCapacities.getMaximumCapacity(label);
+ if (maxCapacity > 0f) {
+ queueCapacities.setAbsoluteMaximumCapacity(
+ label,
+ maxCapacity
+ * (parentQueueCapacities == null ? 1 : parentQueueCapacities
+ .getAbsoluteMaximumCapacity(label)));
+ }
}
- return absoluteMaxCapacityByNodeLabels;
}
@Lock(CSQueue.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 23bf381..f5e86b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -287,6 +287,9 @@ public class CapacitySchedulerConfiguration extends Configuration {
}
private String getNodeLabelPrefix(String queue, String label) {
+ if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
+ return getQueuePrefix(queue);
+ }
return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT;
}
@@ -325,7 +328,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
getMaximumApplicationMasterResourcePercent());
}
- public float getCapacity(String queue) {
+ public float getNonLabeledQueueCapacity(String queue) {
float capacity = queue.equals("root") ? 100.0f : getFloat(
getQueuePrefix(queue) + CAPACITY, UNDEFINED);
if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
@@ -347,7 +350,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
", capacity=" + capacity);
}
- public float getMaximumCapacity(String queue) {
+ public float getNonLabeledQueueMaximumCapacity(String queue) {
float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY,
MAXIMUM_CAPACITY_VALUE);
maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ?
@@ -451,57 +454,29 @@ public class CapacitySchedulerConfiguration extends Configuration {
return Collections.unmodifiableSet(set);
}
- public Map<String, Float> getNodeLabelCapacities(String queue,
- Set<String> labels, RMNodeLabelsManager mgr) {
- Map<String, Float> nodeLabelCapacities = new HashMap<String, Float>();
-
- if (labels == null) {
- return nodeLabelCapacities;
+ private float internalGetLabeledQueueCapacity(String queue, String label, String suffix,
+ float defaultValue) {
+ String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
+ float capacity = getFloat(capacityPropertyName, defaultValue);
+ if (capacity < MINIMUM_CAPACITY_VALUE
+ || capacity > MAXIMUM_CAPACITY_VALUE) {
+ throw new IllegalArgumentException("Illegal capacity of " + capacity
+ + " for node-label=" + label + " in queue=" + queue
+ + ", valid capacity should in range of [0, 100].");
}
-
- for (String label : labels.contains(CommonNodeLabelsManager.ANY) ? mgr
- .getClusterNodeLabels() : labels) {
- String capacityPropertyName = getNodeLabelPrefix(queue, label) + CAPACITY;
- float capacity = getFloat(capacityPropertyName, 0f);
- if (capacity < MINIMUM_CAPACITY_VALUE
- || capacity > MAXIMUM_CAPACITY_VALUE) {
- throw new IllegalArgumentException("Illegal capacity of " + capacity
- + " for node-label=" + label + " in queue=" + queue
- + ", valid capacity should in range of [0, 100].");
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("CSConf - getCapacityOfLabel: prefix="
- + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity);
- }
-
- nodeLabelCapacities.put(label, capacity / 100f);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CSConf - getCapacityOfLabel: prefix="
+ + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity);
}
- return nodeLabelCapacities;
+ return capacity;
}
- public Map<String, Float> getMaximumNodeLabelCapacities(String queue,
- Set<String> labels, RMNodeLabelsManager mgr) {
- Map<String, Float> maximumNodeLabelCapacities = new HashMap<String, Float>();
- if (labels == null) {
- return maximumNodeLabelCapacities;
- }
-
- for (String label : labels.contains(CommonNodeLabelsManager.ANY) ? mgr
- .getClusterNodeLabels() : labels) {
- float maxCapacity =
- getFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY,
- 100f);
- if (maxCapacity < MINIMUM_CAPACITY_VALUE
- || maxCapacity > MAXIMUM_CAPACITY_VALUE) {
- throw new IllegalArgumentException("Illegal " + "capacity of "
- + maxCapacity + " for label=" + label + " in queue=" + queue);
- }
- LOG.debug("CSConf - getCapacityOfLabel: prefix="
- + getNodeLabelPrefix(queue, label) + ", capacity=" + maxCapacity);
-
- maximumNodeLabelCapacities.put(label, maxCapacity / 100f);
- }
- return maximumNodeLabelCapacities;
+ public float getLabeledQueueCapacity(String queue, String label) {
+ return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f);
+ }
+
+ public float getLabeledQueueMaximumCapacity(String queue, String label) {
+ return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f);
}
public String getDefaultNodeLabelExpression(String queue) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6fbc8e4..f9f9ca9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -97,7 +97,7 @@ public class LeafQueue extends AbstractCSQueue {
Set<FiCaSchedulerApp> pendingApplications;
- private final float minimumAllocationFactor;
+ private float minimumAllocationFactor;
private Map<String, User> users = new HashMap<String, User>();
@@ -123,53 +123,7 @@ public class LeafQueue extends AbstractCSQueue {
super(cs, queueName, parent, old);
this.scheduler = cs;
- this.activeUsersManager = new ActiveUsersManager(metrics);
- this.minimumAllocationFactor =
- Resources.ratio(resourceCalculator,
- Resources.subtract(maximumAllocation, minimumAllocation),
- maximumAllocation);
-
- float capacity = getCapacityFromConf();
- float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
-
- float maximumCapacity =
- (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
- float absoluteMaxCapacity =
- CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
-
- // Initially set to absoluteMax, will be updated to more accurate
- // max avail value during assignContainers
- absoluteMaxAvailCapacity = absoluteMaxCapacity;
-
- int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
- float userLimitFactor =
- cs.getConfiguration().getUserLimitFactor(getQueuePath());
-
- int maxApplications =
- cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath());
- if (maxApplications < 0) {
- int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
- maxApplications = (int)(maxSystemApps * absoluteCapacity);
- }
- maxApplicationsPerUser =
- (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
-
- float maxAMResourcePerQueuePercent = cs.getConfiguration()
- .getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
-
- QueueState state = cs.getConfiguration().getState(getQueuePath());
-
- Map<QueueACL, AccessControlList> acls =
- cs.getConfiguration().getAcls(getQueuePath());
-
- setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor,
- maxApplications, maxAMResourcePerQueuePercent, maxApplicationsPerUser,
- state, acls, cs.getConfiguration().getNodeLocalityDelay(), accessibleLabels,
- defaultLabelExpression, this.capacitiyByNodeLabels,
- this.maxCapacityByNodeLabels,
- cs.getConfiguration().getReservationContinueLook());
-
+ this.activeUsersManager = new ActiveUsersManager(metrics);
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
+ ", fullname=" + getQueuePath());
@@ -180,34 +134,13 @@ public class LeafQueue extends AbstractCSQueue {
this.pendingApplications =
new TreeSet<FiCaSchedulerApp>(applicationComparator);
this.activeApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator);
+
+ setupQueueConfigs(cs.getClusterResource());
}
-
- // externalizing in method, to allow overriding
- protected float getCapacityFromConf() {
- return (float)scheduler.getConfiguration().getCapacity(getQueuePath()) / 100;
- }
-
- protected synchronized void setupQueueConfigs(
- Resource clusterResource,
- float capacity, float absoluteCapacity,
- float maximumCapacity, float absoluteMaxCapacity,
- int userLimit, float userLimitFactor,
- int maxApplications, float maxAMResourcePerQueuePercent,
- int maxApplicationsPerUser, QueueState state,
- Map<QueueACL, AccessControlList> acls, int nodeLocalityDelay,
- Set<String> labels, String defaultLabelExpression,
- Map<String, Float> capacitieByLabel,
- Map<String, Float> maximumCapacitiesByLabel,
- boolean revervationContinueLooking) throws IOException {
- super.setupQueueConfigs(clusterResource, capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, state, acls, labels,
- defaultLabelExpression, capacitieByLabel, maximumCapacitiesByLabel,
- revervationContinueLooking);
- // Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- float absCapacity = getParent().getAbsoluteCapacity() * capacity;
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absCapacity,
- absoluteMaxCapacity);
+
+ protected synchronized void setupQueueConfigs(Resource clusterResource)
+ throws IOException {
+ super.setupQueueConfigs(clusterResource);
this.lastClusterResource = clusterResource;
updateAbsoluteCapacityResource(clusterResource);
@@ -217,16 +150,28 @@ public class LeafQueue extends AbstractCSQueue {
// and all queues may not be realized yet, we'll use (optimistic)
// absoluteMaxCapacity (it will be replaced with the more accurate
// absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
- updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
+ updateHeadroomInfo(clusterResource,
+ queueCapacities.getAbsoluteMaximumCapacity());
- this.absoluteCapacity = absCapacity;
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ userLimit = conf.getUserLimit(getQueuePath());
+ userLimitFactor = conf.getUserLimitFactor(getQueuePath());
- this.userLimit = userLimit;
- this.userLimitFactor = userLimitFactor;
+ // Initially set to absoluteMax, will be updated to more accurate
+ // max avail value during assignContainers
+ absoluteMaxAvailCapacity = queueCapacities.getAbsoluteMaximumCapacity();
- this.maxApplications = maxApplications;
- this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
- this.maxApplicationsPerUser = maxApplicationsPerUser;
+ maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath());
+ if (maxApplications < 0) {
+ int maxSystemApps = conf.getMaximumSystemApplications();
+ maxApplications =
+ (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity());
+ }
+ maxApplicationsPerUser =
+ (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
+
+ maxAMResourcePerQueuePercent =
+ conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
this.defaultLabelExpression)) {
@@ -242,7 +187,12 @@ public class LeafQueue extends AbstractCSQueue {
getAccessibleNodeLabels().iterator(), ',')));
}
- this.nodeLocalityDelay = nodeLocalityDelay;
+ nodeLocalityDelay = conf.getNodeLocalityDelay();
+
+ this.minimumAllocationFactor =
+ Resources.ratio(resourceCalculator,
+ Resources.subtract(maximumAllocation, minimumAllocation),
+ maximumAllocation);
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -250,21 +200,21 @@ public class LeafQueue extends AbstractCSQueue {
}
StringBuilder labelStrBuilder = new StringBuilder();
- if (labels != null) {
- for (String s : labels) {
+ if (accessibleLabels != null) {
+ for (String s : accessibleLabels) {
labelStrBuilder.append(s);
labelStrBuilder.append(",");
}
}
LOG.info("Initializing " + queueName + "\n" +
- "capacity = " + capacity +
+ "capacity = " + queueCapacities.getCapacity() +
" [= (float) configuredCapacity / 100 ]" + "\n" +
- "asboluteCapacity = " + absoluteCapacity +
+ "asboluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
" [= parentAbsoluteCapacity * capacity ]" + "\n" +
- "maxCapacity = " + maximumCapacity +
+ "maxCapacity = " + queueCapacities.getMaximumCapacity() +
" [= configuredMaxCapacity ]" + "\n" +
- "absoluteMaxCapacity = " + absoluteMaxCapacity +
+ "absoluteMaxCapacity = " + queueCapacities.getAbsoluteMaximumCapacity() +
" [= 1.0 maximumCapacity undefined, " +
"(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" +
"\n" +
@@ -279,7 +229,7 @@ public class LeafQueue extends AbstractCSQueue {
"maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
- "usedCapacity = " + usedCapacity +
+ "usedCapacity = " + queueCapacities.getUsedCapacity() +
" [= usedResourcesMemory / " +
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"absoluteUsedCapacity = " + absoluteUsedCapacity +
@@ -435,8 +385,8 @@ public class LeafQueue extends AbstractCSQueue {
public String toString() {
return queueName + ": " +
- "capacity=" + capacity + ", " +
- "absoluteCapacity=" + absoluteCapacity + ", " +
+ "capacity=" + queueCapacities.getCapacity() + ", " +
+ "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
"usedResources=" + queueUsage.getUsed() + ", " +
"usedCapacity=" + getUsedCapacity() + ", " +
"absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
@@ -483,23 +433,7 @@ public class LeafQueue extends AbstractCSQueue {
" from " + newlyParsedQueue.getQueuePath());
}
- LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;
- setupQueueConfigs(
- clusterResource,
- newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity,
- newlyParsedLeafQueue.maximumCapacity,
- newlyParsedLeafQueue.absoluteMaxCapacity,
- newlyParsedLeafQueue.userLimit, newlyParsedLeafQueue.userLimitFactor,
- newlyParsedLeafQueue.maxApplications,
- newlyParsedLeafQueue.maxAMResourcePerQueuePercent,
- newlyParsedLeafQueue.getMaxApplicationsPerUser(),
- newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
- newlyParsedLeafQueue.getNodeLocalityDelay(),
- newlyParsedLeafQueue.accessibleLabels,
- newlyParsedLeafQueue.defaultLabelExpression,
- newlyParsedLeafQueue.capacitiyByNodeLabels,
- newlyParsedLeafQueue.maxCapacityByNodeLabels,
- newlyParsedLeafQueue.reservationsContinueLooking);
+ setupQueueConfigs(clusterResource);
// queue metrics are updated, more resource may be available
// activate the pending applications if possible
@@ -1022,7 +956,8 @@ public class LeafQueue extends AbstractCSQueue {
application.getCurrentReservation()),
labelManager.getResourceByLabel(label, clusterResource));
- if (potentialNewWithoutReservedCapacity <= absoluteMaxCapacity) {
+ if (potentialNewWithoutReservedCapacity <= queueCapacities
+ .getAbsoluteMaximumCapacity()) {
if (LOG.isDebugEnabled()) {
LOG.debug("try to use reserved: "
+ getQueueName()
@@ -1036,8 +971,9 @@ public class LeafQueue extends AbstractCSQueue {
+ Resources.divide(resourceCalculator, clusterResource,
queueUsage.getUsed(), clusterResource) + " required " + required
+ " potentialNewWithoutReservedCapacity: "
- + potentialNewWithoutReservedCapacity + " ( " + " max-capacity: "
- + absoluteMaxCapacity + ")");
+ + potentialNewWithoutReservedCapacity + " ( "
+ + " max-capacity: "
+ + queueCapacities.getAbsoluteMaximumCapacity() + ")");
}
// we could potentially use this node instead of reserved node
return true;
@@ -1046,7 +982,8 @@ public class LeafQueue extends AbstractCSQueue {
// Otherwise, if any of the label of this node beyond queue limit, we
// cannot allocate on this node. Consider a small epsilon here.
- if (potentialNewCapacity > getAbsoluteMaximumCapacityByNodeLabel(label) + 1e-4) {
+ if (potentialNewCapacity > queueCapacities
+ .getAbsoluteMaximumCapacity(label) + 1e-4) {
canAssign = false;
break;
}
@@ -1061,7 +998,8 @@ public class LeafQueue extends AbstractCSQueue {
queueUsage.getUsed(label),
labelManager.getResourceByLabel(label, clusterResource))
+ " potentialNewCapacity: " + potentialNewCapacity + " ( "
- + " max-capacity: " + absoluteMaxCapacity + ")");
+ + " max-capacity: " + queueCapacities.getAbsoluteMaximumCapacity()
+ + ")");
}
}
@@ -1144,7 +1082,7 @@ public class LeafQueue extends AbstractCSQueue {
Resources.multiplyAndNormalizeUp(resourceCalculator,
labelManager.getResourceByLabel(firstLabel,
clusterResource),
- getAbsoluteCapacityByNodeLabel(firstLabel),
+ queueCapacities.getAbsoluteCapacity(firstLabel),
minimumAllocation));
} else {
// else there's no label on request, just to use absolute capacity as
@@ -1152,7 +1090,7 @@ public class LeafQueue extends AbstractCSQueue {
queueCapacity =
Resources.multiplyAndNormalizeUp(resourceCalculator, labelManager
.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, clusterResource),
- absoluteCapacity, minimumAllocation);
+ queueCapacities.getAbsoluteCapacity(), minimumAllocation);
}
// Allow progress for queues with miniscule capacity
@@ -1797,12 +1735,9 @@ public class LeafQueue extends AbstractCSQueue {
}
private void updateAbsoluteCapacityResource(Resource clusterResource) {
-
- absoluteCapacityResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- clusterResource,
- absoluteCapacity, minimumAllocation);
-
+ absoluteCapacityResource =
+ Resources.multiplyAndNormalizeUp(resourceCalculator, clusterResource,
+ queueCapacities.getAbsoluteCapacity(), minimumAllocation);
}
@Override
@@ -1813,7 +1748,8 @@ public class LeafQueue extends AbstractCSQueue {
// Update headroom info based on new cluster resource value
// absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
// during allocation
- updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
+ updateHeadroomInfo(clusterResource,
+ queueCapacities.getAbsoluteMaximumCapacity());
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -1986,35 +1922,13 @@ public class LeafQueue extends AbstractCSQueue {
getParent().detachContainer(clusterResource, application, rmContainer);
}
}
-
- @Override
- public float getAbsActualCapacity() {
- //? Is this actually used by anything at present?
- // There is a findbugs warning -re lastClusterResource (now excluded),
- // when this is used, verify that the access is mt correct and remove
- // the findbugs exclusion if possible
- if (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
- lastClusterResource, Resources.none())) {
- return absoluteCapacity;
- }
-
- Resource resourceRespectLabels =
- labelManager == null ? lastClusterResource : labelManager
- .getQueueResource(queueName, accessibleLabels, lastClusterResource);
- float absActualCapacity =
- Resources.divide(resourceCalculator, lastClusterResource,
- resourceRespectLabels, lastClusterResource);
-
- return absActualCapacity > absoluteCapacity ? absoluteCapacity
- : absActualCapacity;
- }
public void setCapacity(float capacity) {
- this.capacity = capacity;
+ queueCapacities.setCapacity(capacity);
}
public void setAbsoluteCapacity(float absoluteCapacity) {
- this.absoluteCapacity = absoluteCapacity;
+ queueCapacities.setAbsoluteCapacity(absoluteCapacity);
}
public void setMaxApplications(int maxApplications) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index de92c9c..5f7db78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -86,7 +86,7 @@ public class ParentQueue extends AbstractCSQueue {
this.rootQueue = (parent == null);
- float rawCapacity = cs.getConfiguration().getCapacity(getQueuePath());
+ float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath());
if (rootQueue &&
(rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) {
@@ -94,46 +94,20 @@ public class ParentQueue extends AbstractCSQueue {
"capacity of " + rawCapacity + " for queue " + queueName +
". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE);
}
-
- float capacity = (float) rawCapacity / 100;
- float parentAbsoluteCapacity =
- (rootQueue) ? 1.0f : parent.getAbsoluteCapacity();
- float absoluteCapacity = parentAbsoluteCapacity * capacity;
-
- float maximumCapacity =
- (float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
- float absoluteMaxCapacity =
- CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
-
- QueueState state = cs.getConfiguration().getState(getQueuePath());
-
- Map<QueueACL, AccessControlList> acls =
- cs.getConfiguration().getAcls(getQueuePath());
-
- setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
- defaultLabelExpression, capacitiyByNodeLabels, maxCapacityByNodeLabels,
- cs.getConfiguration().getReservationContinueLook());
this.childQueues = new TreeSet<CSQueue>(queueComparator);
+
+ setupQueueConfigs(cs.getClusterResource());
LOG.info("Initialized parent-queue " + queueName +
" name=" + queueName +
", fullname=" + getQueuePath());
}
- synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
- float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity,
- QueueState state, Map<QueueACL, AccessControlList> acls,
- Set<String> accessibleLabels, String defaultLabelExpression,
- Map<String, Float> nodeLabelCapacities,
- Map<String, Float> maximumCapacitiesByLabel,
- boolean reservationContinueLooking) throws IOException {
- super.setupQueueConfigs(clusterResource, capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
- defaultLabelExpression, nodeLabelCapacities, maximumCapacitiesByLabel,
- reservationContinueLooking);
- StringBuilder aclsString = new StringBuilder();
+ synchronized void setupQueueConfigs(Resource clusterResource)
+ throws IOException {
+ super.setupQueueConfigs(clusterResource);
+ StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
@@ -147,10 +121,10 @@ public class ParentQueue extends AbstractCSQueue {
}
LOG.info(queueName +
- ", capacity=" + capacity +
- ", asboluteCapacity=" + absoluteCapacity +
- ", maxCapacity=" + maximumCapacity +
- ", asboluteMaxCapacity=" + absoluteMaxCapacity +
+ ", capacity=" + this.queueCapacities.getCapacity() +
+ ", asboluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
+ ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() +
+ ", asboluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() +
", state=" + state +
", acls=" + aclsString +
", labels=" + labelStrBuilder.toString() + "\n" +
@@ -166,19 +140,19 @@ public class ParentQueue extends AbstractCSQueue {
}
float delta = Math.abs(1.0f - childCapacities); // crude way to check
// allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((capacity > 0) && (delta > PRECISION)) ||
- ((capacity == 0) && (childCapacities > 0))) {
+ if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) ||
+ ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
throw new IllegalArgumentException("Illegal" +
" capacity of " + childCapacities +
" for children of queue " + queueName);
}
// check label capacities
for (String nodeLabel : labelManager.getClusterNodeLabels()) {
- float capacityByLabel = getCapacityByNodeLabel(nodeLabel);
+ float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
// check children's labels
float sum = 0;
for (CSQueue queue : childQueues) {
- sum += queue.getCapacityByNodeLabel(nodeLabel);
+ sum += queue.getQueueCapacities().getCapacity(nodeLabel);
}
if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
|| (capacityByLabel == 0) && (sum > 0)) {
@@ -254,8 +228,8 @@ public class ParentQueue extends AbstractCSQueue {
public String toString() {
return queueName + ": " +
"numChildQueue= " + childQueues.size() + ", " +
- "capacity=" + capacity + ", " +
- "absoluteCapacity=" + absoluteCapacity + ", " +
+ "capacity=" + queueCapacities.getCapacity() + ", " +
+ "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
"usedResources=" + queueUsage.getUsed() +
"usedCapacity=" + getUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
@@ -263,9 +237,8 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized void reinitialize(
- CSQueue newlyParsedQueue, Resource clusterResource)
- throws IOException {
+ public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ Resource clusterResource) throws IOException {
// Sanity check
if (!(newlyParsedQueue instanceof ParentQueue) ||
!newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
@@ -276,18 +249,7 @@ public class ParentQueue extends AbstractCSQueue {
ParentQueue newlyParsedParentQueue = (ParentQueue)newlyParsedQueue;
// Set new configs
- setupQueueConfigs(clusterResource,
- newlyParsedParentQueue.capacity,
- newlyParsedParentQueue.absoluteCapacity,
- newlyParsedParentQueue.maximumCapacity,
- newlyParsedParentQueue.absoluteMaxCapacity,
- newlyParsedParentQueue.state,
- newlyParsedParentQueue.acls,
- newlyParsedParentQueue.accessibleLabels,
- newlyParsedParentQueue.defaultLabelExpression,
- newlyParsedParentQueue.capacitiyByNodeLabels,
- newlyParsedParentQueue.maxCapacityByNodeLabels,
- newlyParsedParentQueue.reservationsContinueLooking);
+ setupQueueConfigs(clusterResource);
// Re-configure existing child queues and add new ones
// The CS has already checked to ensure all existing child queues are present!
@@ -512,7 +474,7 @@ public class ParentQueue extends AbstractCSQueue {
labelManager.getResourceByLabel(label, clusterResource));
// if any of the label doesn't beyond limit, we can allocate on this node
if (currentAbsoluteLabelUsedCapacity >=
- getAbsoluteMaximumCapacityByNodeLabel(label)) {
+ queueCapacities.getAbsoluteMaximumCapacity(label)) {
if (LOG.isDebugEnabled()) {
LOG.debug(getQueueName() + " used=" + queueUsage.getUsed()
+ " current-capacity (" + queueUsage.getUsed(label) + ") "
@@ -540,7 +502,8 @@ public class ParentQueue extends AbstractCSQueue {
Resources.subtract(queueUsage.getUsed(), reservedResources),
clusterResource);
- if (capacityWithoutReservedCapacity <= absoluteMaxCapacity) {
+ if (capacityWithoutReservedCapacity <= queueCapacities
+ .getAbsoluteMaximumCapacity()) {
if (LOG.isDebugEnabled()) {
LOG.debug("parent: try to use reserved: " + getQueueName()
+ " usedResources: " + queueUsage.getUsed().getMemory()
@@ -550,7 +513,7 @@ public class ParentQueue extends AbstractCSQueue {
/ clusterResource.getMemory()
+ " potentialNewWithoutReservedCapacity: "
+ capacityWithoutReservedCapacity + " ( " + " max-capacity: "
- + absoluteMaxCapacity + ")");
+ + queueCapacities.getAbsoluteMaximumCapacity() + ")");
}
// we could potentially use this node instead of reserved node
return true;
@@ -760,13 +723,6 @@ public class ParentQueue extends AbstractCSQueue {
}
}
}
-
- @Override
- public float getAbsActualCapacity() {
- // for now, simply return actual capacity = guaranteed capacity for parent
- // queue
- return absoluteCapacity;
- }
public synchronized int getNumApplications() {
return numApplications;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 0725959..f91a6d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -101,16 +99,7 @@ public class PlanQueue extends ParentQueue {
}
// Set new configs
- setupQueueConfigs(clusterResource, newlyParsedParentQueue.getCapacity(),
- newlyParsedParentQueue.getAbsoluteCapacity(),
- newlyParsedParentQueue.getMaximumCapacity(),
- newlyParsedParentQueue.getAbsoluteMaximumCapacity(),
- newlyParsedParentQueue.getState(), newlyParsedParentQueue.getACLs(),
- newlyParsedParentQueue.accessibleLabels,
- newlyParsedParentQueue.defaultLabelExpression,
- newlyParsedParentQueue.capacitiyByNodeLabels,
- newlyParsedParentQueue.maxCapacityByNodeLabels,
- newlyParsedParentQueue.getReservationContinueLooking());
+ setupQueueConfigs(clusterResource);
updateQuotas(newlyParsedParentQueue.userLimit,
newlyParsedParentQueue.userLimitFactor,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index a0e6d8c..962a636 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -19,12 +19,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+
+import com.google.common.collect.Sets;
public class QueueCapacities {
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
@@ -32,13 +38,15 @@ public class QueueCapacities {
private Map<String, Capacities> capacitiesMap;
private ReadLock readLock;
private WriteLock writeLock;
+ private final boolean isRoot;
- public QueueCapacities() {
+ public QueueCapacities(boolean isRoot) {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
capacitiesMap = new HashMap<String, Capacities>();
+ this.isRoot = isRoot;
}
// Usage enum here to make implement cleaner
@@ -58,6 +66,18 @@ public class QueueCapacities {
public Capacities() {
capacitiesArr = new float[CapacityType.values().length];
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{used=" + capacitiesArr[0] + "%, ");
+ sb.append("abs_used=" + capacitiesArr[1] + "%, ");
+ sb.append("max_cap=" + capacitiesArr[2] + "%, ");
+ sb.append("abs_max_cap=" + capacitiesArr[3] + "%, ");
+ sb.append("cap=" + capacitiesArr[4] + "%, ");
+ sb.append("abs_cap=" + capacitiesArr[5] + "%}");
+ return sb.toString();
+ }
}
private float _get(String label, CapacityType type) {
@@ -127,6 +147,10 @@ public class QueueCapacities {
}
public float getCapacity(String label) {
+ if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL) && isRoot) {
+ return 1f;
+ }
+
return _get(label, CapacityType.CAP);
}
@@ -144,6 +168,9 @@ public class QueueCapacities {
}
public float getAbsoluteCapacity(String label) {
+ if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL) && isRoot) {
+ return 1f;
+ }
return _get(label, CapacityType.ABS_CAP);
}
@@ -188,4 +215,43 @@ public class QueueCapacities {
public void setAbsoluteMaximumCapacity(String label, float value) {
_set(label, CapacityType.ABS_MAX_CAP, value);
}
+
+ /**
+ * Clear configurable fields, like
+ * (absolute)capacity/(absolute)maximum-capacity, this will be used by queue
+ * reinitialize, when we reinitialize a queue, we will first clear all
+ * configurable fields, and load new values
+ */
+ public void clearConfigurableFields() {
+ try {
+ writeLock.lock();
+ for (String label : capacitiesMap.keySet()) {
+ _set(label, CapacityType.CAP, 0);
+ _set(label, CapacityType.MAX_CAP, 0);
+ _set(label, CapacityType.ABS_CAP, 0);
+ _set(label, CapacityType.ABS_MAX_CAP, 0);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public Set<String> getExistingNodeLabels() {
+ try {
+ readLock.lock();
+ return new HashSet<String>(capacitiesMap.keySet());
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ try {
+ readLock.lock();
+ return this.capacitiesMap.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index c4424b5..a8d17cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -62,11 +62,11 @@ public class ReservationQueue extends LeafQueue {
throw new IOException("Trying to reinitialize " + getQueuePath()
+ " from " + newlyParsedQueue.getQueuePath());
}
+ super.reinitialize(newlyParsedQueue, clusterResource);
CSQueueUtils.updateQueueStatistics(
parent.schedulerContext.getResourceCalculator(), newlyParsedQueue,
parent, parent.schedulerContext.getClusterResource(),
parent.schedulerContext.getMinimumResourceCapability());
- super.reinitialize(newlyParsedQueue, clusterResource);
updateQuotas(parent.getUserLimitForReservation(),
parent.getUserLimitFactor(),
parent.getMaxApplicationsForReservations(),
@@ -108,9 +108,9 @@ public class ReservationQueue extends LeafQueue {
maxApplicationsPerUser = maxAppsPerUserForReservation;
}
- // used by the super constructor, we initialize to zero
- protected float getCapacityFromConf() {
- return 0f;
+ @Override
+ protected void setupConfigurableCapacities() {
+ CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
+ queueCapacities, parent == null ? null : parent.getQueueCapacities());
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
index d643c9d..5135ba9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
@@ -202,33 +202,33 @@ public class TestCSQueueUtils {
LOG.info("t2 l2q2 " + result);
//some usage, but below the base capacity
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
- l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q2.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.4f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//usage gt base on parent sibling
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
- l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+ l1q2.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//same as last, but with usage also on direct parent
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
- l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//add to direct sibling, below the threshold of effect at present
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
@@ -236,9 +236,9 @@ public class TestCSQueueUtils {
//add to direct sibling, now above the threshold of effect
//(it's cumulative with prior tests)
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.1f, result, 0.000001f);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index edcd871..456178d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -389,11 +389,11 @@ public class TestCapacityScheduler {
public void testMaximumCapacitySetup() {
float delta = 0.0000001f;
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+ assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
conf.setMaximumCapacity(A, 50.0f);
- assertEquals(50.0f, conf.getMaximumCapacity(A),delta);
+ assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A),delta);
conf.setMaximumCapacity(A, -1);
- assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+ assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index 261fa01..4853241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -91,7 +91,8 @@ public class TestCapacitySchedulerNodeLabelUpdate {
String label) {
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
CSQueue queue = scheduler.getQueue(queueName);
- Assert.assertEquals(memory, queue.getUsedResourceByLabel(label).getMemory());
+ Assert.assertEquals(memory, queue.getQueueResourceUsage().getUsed(label)
+ .getMemory());
}
@Test (timeout = 30000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
index 89a5311..a468230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -99,7 +99,7 @@ public class TestQueueCapacities {
}
private void internalTestModifyAndRead(String label) throws Exception {
- QueueCapacities qc = new QueueCapacities();
+ QueueCapacities qc = new QueueCapacities(false);
// First get returns 0 always
Assert.assertEquals(0f, get(qc, suffix, label), 1e-8);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index cf2e5ce..fc03581 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -418,18 +418,18 @@ public class TestQueueParsing {
// check capacity of A2
CSQueue qA2 = capacityScheduler.getQueue("a2");
Assert.assertEquals(0.7, qA2.getCapacity(), DELTA);
- Assert.assertEquals(0.5, qA2.getCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.5, qA2.getQueueCapacities().getCapacity("red"), DELTA);
Assert.assertEquals(0.07, qA2.getAbsoluteCapacity(), DELTA);
- Assert.assertEquals(0.25, qA2.getAbsoluteCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.25, qA2.getQueueCapacities().getAbsoluteCapacity("red"), DELTA);
Assert.assertEquals(0.1275, qA2.getAbsoluteMaximumCapacity(), DELTA);
- Assert.assertEquals(0.3, qA2.getAbsoluteMaximumCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.3, qA2.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
// check capacity of B3
CSQueue qB3 = capacityScheduler.getQueue("b3");
Assert.assertEquals(0.18, qB3.getAbsoluteCapacity(), DELTA);
- Assert.assertEquals(0.125, qB3.getAbsoluteCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.125, qB3.getQueueCapacities().getAbsoluteCapacity("red"), DELTA);
Assert.assertEquals(0.35, qB3.getAbsoluteMaximumCapacity(), DELTA);
- Assert.assertEquals(1, qB3.getAbsoluteMaximumCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(1, qB3.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
}
private void
@@ -641,8 +641,8 @@ public class TestQueueParsing {
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.getQueue(CapacitySchedulerConfiguration.ROOT);
- Assert.assertEquals(0, root.getCapacityByNodeLabel("red"), DELTA);
- Assert.assertEquals(0, root.getCapacityByNodeLabel("blue"), DELTA);
+ Assert.assertEquals(0, root.getQueueCapacities().getCapacity("red"), DELTA);
+ Assert.assertEquals(0, root.getQueueCapacities().getCapacity("blue"), DELTA);
CSQueue a = capacityScheduler.getQueue("a");
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index d418dab..63a48bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -395,9 +395,9 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
String qshortName = qArr[qArr.length - 1];
assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f);
- assertEquals("capacity doesn't match", csConf.getCapacity(q),
+ assertEquals("capacity doesn't match", csConf.getNonLabeledQueueCapacity(q),
info.capacity, 1e-3f);
- float expectCapacity = csConf.getMaximumCapacity(q);
+ float expectCapacity = csConf.getNonLabeledQueueMaximumCapacity(q);
float expectAbsMaxCapacity = parentAbsMaxCapacity * (info.maxCapacity/100);
if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) {
expectCapacity = 100;
[5/8] hadoop git commit: YARN-3099. Capacity Scheduler
LeafQueue/ParentQueue should use ResourceUsage to track
used-resources-by-label. Contributed by Wangda Tan (cherry picked from commit
86358221fc85a7743052a0b4c1647353508bf308)
Posted by vi...@apache.org.
YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage to track used-resources-by-label. Contributed by Wangda Tan
(cherry picked from commit 86358221fc85a7743052a0b4c1647353508bf308)
(cherry picked from commit cabf97ae4f2dad53c7b9e3d10a67876b16d94074)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9281fbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9281fbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9281fbb
Branch: refs/heads/branch-2.6.1
Commit: d9281fbbab4a2e0475789e724f21faa52a465f62
Parents: b0ad553
Author: Jian He <ji...@apache.org>
Authored: Fri Jan 30 15:15:20 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:20 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/capacity/AbstractCSQueue.java | 48 +++----
.../scheduler/capacity/LeafQueue.java | 128 +++++++------------
.../scheduler/capacity/ParentQueue.java | 25 ++--
.../TestWorkPreservingRMRestart.java | 2 +-
.../scheduler/capacity/TestCSQueueUtils.java | 24 ++--
6 files changed, 93 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 975e4e1..bac13ff 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -180,6 +180,9 @@ Release 2.6.1 - UNRELEASED
YARN-2978. Fixed potential NPE while getting queue info. (Varun Saxena via
jianhe)
+ YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage
+ to track used-resources-by-label.(Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fec3a56..3c1663f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -35,10 +35,11 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
public abstract class AbstractCSQueue implements CSQueue {
@@ -64,16 +65,17 @@ public abstract class AbstractCSQueue implements CSQueue {
Set<String> accessibleLabels;
RMNodeLabelsManager labelManager;
String defaultLabelExpression;
- Resource usedResources = Resources.createResource(0, 0);
Map<String, Float> absoluteCapacityByNodeLabels;
Map<String, Float> capacitiyByNodeLabels;
- Map<String, Resource> usedResourcesByNodeLabels = new HashMap<String, Resource>();
Map<String, Float> absoluteMaxCapacityByNodeLabels;
Map<String, Float> maxCapacityByNodeLabels;
Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
boolean reservationsContinueLooking;
+
+ // Track resource usage-by-label like used-resource/pending-resource, etc.
+ ResourceUsage queueUsage;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -120,6 +122,7 @@ public abstract class AbstractCSQueue implements CSQueue {
maxCapacityByNodeLabels =
cs.getConfiguration().getMaximumNodeLabelCapacities(getQueuePath(),
accessibleLabels, labelManager);
+ queueUsage = new ResourceUsage();
}
@Override
@@ -153,8 +156,8 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized Resource getUsedResources() {
- return usedResources;
+ public Resource getUsedResources() {
+ return queueUsage.getUsed();
}
public synchronized int getNumContainers() {
@@ -344,22 +347,13 @@ public abstract class AbstractCSQueue implements CSQueue {
synchronized void allocateResource(Resource clusterResource,
Resource resource, Set<String> nodeLabels) {
- Resources.addTo(usedResources, resource);
// Update usedResources by labels
if (nodeLabels == null || nodeLabels.isEmpty()) {
- if (!usedResourcesByNodeLabels.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- usedResourcesByNodeLabels.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.addTo(usedResourcesByNodeLabels.get(RMNodeLabelsManager.NO_LABEL),
- resource);
+ queueUsage.incUsed(resource);
} else {
for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
- Resources.addTo(usedResourcesByNodeLabels.get(label), resource);
+ queueUsage.incUsed(label, resource);
}
}
@@ -370,23 +364,12 @@ public abstract class AbstractCSQueue implements CSQueue {
protected synchronized void releaseResource(Resource clusterResource,
Resource resource, Set<String> nodeLabels) {
- // Update queue metrics
- Resources.subtractFrom(usedResources, resource);
-
// Update usedResources by labels
if (null == nodeLabels || nodeLabels.isEmpty()) {
- if (!usedResourcesByNodeLabels.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- usedResourcesByNodeLabels.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.subtractFrom(
- usedResourcesByNodeLabels.get(RMNodeLabelsManager.NO_LABEL), resource);
+ queueUsage.decUsed(resource);
} else {
for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
- Resources.subtractFrom(usedResourcesByNodeLabels.get(label), resource);
+ queueUsage.decUsed(label, resource);
}
}
@@ -452,6 +435,11 @@ public abstract class AbstractCSQueue implements CSQueue {
@Private
public Resource getUsedResourceByLabel(String nodeLabel) {
- return usedResourcesByNodeLabels.get(nodeLabel);
+ return queueUsage.getUsed(nodeLabel);
+ }
+
+ @VisibleForTesting
+ public ResourceUsage getResourceUsage() {
+ return queueUsage;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 78b5fb6..6fbc8e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -117,10 +118,6 @@ public class LeafQueue extends AbstractCSQueue {
private volatile float absoluteMaxAvailCapacity;
- // sum of resources used by application masters for applications
- // running in this queue
- private final Resource usedAMResources = Resource.newInstance(0, 0);
-
public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
@@ -440,7 +437,7 @@ public class LeafQueue extends AbstractCSQueue {
return queueName + ": " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
- "usedResources=" + usedResources + ", " +
+ "usedResources=" + queueUsage.getUsed() + ", " +
"usedCapacity=" + getUsedCapacity() + ", " +
"absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
@@ -469,7 +466,7 @@ public class LeafQueue extends AbstractCSQueue {
ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
for (Map.Entry<String, User> entry: users.entrySet()) {
usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(
- entry.getValue().consumed), entry.getValue().getActiveApplications(),
+ entry.getValue().getUsed()), entry.getValue().getActiveApplications(),
entry.getValue().getPendingApplications()));
}
return usersToReturn;
@@ -638,7 +635,7 @@ public class LeafQueue extends AbstractCSQueue {
// Check am resource limit
Resource amIfStarted =
- Resources.add(application.getAMResource(), usedAMResources);
+ Resources.add(application.getAMResource(), queueUsage.getAMUsed());
if (LOG.isDebugEnabled()) {
LOG.debug("application AMResource " + application.getAMResource() +
@@ -683,9 +680,8 @@ public class LeafQueue extends AbstractCSQueue {
}
user.activateApplication();
activeApplications.add(application);
- Resources.addTo(usedAMResources, application.getAMResource());
- Resources.addTo(user.getConsumedAMResources(),
- application.getAMResource());
+ queueUsage.incAMUsed(application.getAMResource());
+ user.getResourceUsage().incAMUsed(application.getAMResource());
i.remove();
LOG.info("Application " + application.getApplicationId() +
" from user: " + application.getUser() +
@@ -736,9 +732,8 @@ public class LeafQueue extends AbstractCSQueue {
if (!wasActive) {
pendingApplications.remove(application);
} else {
- Resources.subtractFrom(usedAMResources, application.getAMResource());
- Resources.subtractFrom(user.getConsumedAMResources(),
- application.getAMResource());
+ queueUsage.decAMUsed(application.getAMResource());
+ user.getResourceUsage().decAMUsed(application.getAMResource());
}
applicationAttemptMap.remove(application.getApplicationAttemptId());
@@ -987,8 +982,8 @@ public class LeafQueue extends AbstractCSQueue {
*/
Resource headroom =
Resources.min(resourceCalculator, clusterResource,
- Resources.subtract(userLimit, user.getTotalConsumedResources()),
- Resources.subtract(queueMaxCap, usedResources)
+ Resources.subtract(userLimit, user.getUsed()),
+ Resources.subtract(queueMaxCap, queueUsage.getUsed())
);
return headroom;
}
@@ -1008,12 +1003,8 @@ public class LeafQueue extends AbstractCSQueue {
boolean canAssign = true;
for (String label : labelCanAccess) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
-
Resource potentialTotalCapacity =
- Resources.add(usedResourcesByNodeLabels.get(label), required);
+ Resources.add(queueUsage.getUsed(label), required);
float potentialNewCapacity =
Resources.divide(resourceCalculator, clusterResource,
@@ -1036,14 +1027,14 @@ public class LeafQueue extends AbstractCSQueue {
LOG.debug("try to use reserved: "
+ getQueueName()
+ " usedResources: "
- + usedResources
+ + queueUsage.getUsed()
+ " clusterResources: "
+ clusterResource
+ " reservedResources: "
+ application.getCurrentReservation()
+ " currentCapacity "
+ Resources.divide(resourceCalculator, clusterResource,
- usedResources, clusterResource) + " required " + required
+ queueUsage.getUsed(), clusterResource) + " required " + required
+ " potentialNewWithoutReservedCapacity: "
+ potentialNewWithoutReservedCapacity + " ( " + " max-capacity: "
+ absoluteMaxCapacity + ")");
@@ -1063,11 +1054,11 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
LOG.debug(getQueueName()
+ "Check assign to queue, label=" + label
- + " usedResources: " + usedResourcesByNodeLabels.get(label)
+ + " usedResources: " + queueUsage.getUsed(label)
+ " clusterResources: " + clusterResource
+ " currentCapacity "
+ Resources.divide(resourceCalculator, clusterResource,
- usedResourcesByNodeLabels.get(label),
+ queueUsage.getUsed(label),
labelManager.getResourceByLabel(label, clusterResource))
+ " potentialNewCapacity: " + potentialNewCapacity + " ( "
+ " max-capacity: " + absoluteMaxCapacity + ")");
@@ -1118,7 +1109,7 @@ public class LeafQueue extends AbstractCSQueue {
LOG.debug("Headroom calculation for user " + user + ": " +
" userLimit=" + userLimit +
" queueMaxCap=" + queueMaxCap +
- " consumed=" + queueUser.getTotalConsumedResources() +
+ " consumed=" + queueUser.getUsed() +
" headroom=" + headroom);
}
@@ -1173,8 +1164,8 @@ public class LeafQueue extends AbstractCSQueue {
Resource currentCapacity =
Resources.lessThan(resourceCalculator, clusterResource,
- usedResources, queueCapacity) ?
- queueCapacity : Resources.add(usedResources, required);
+ queueUsage.getUsed(), queueCapacity) ?
+ queueCapacity : Resources.add(queueUsage.getUsed(), required);
// Never allow a single user to take more than the
// queue's configured capacity * user-limit-factor.
@@ -1209,10 +1200,10 @@ public class LeafQueue extends AbstractCSQueue {
" userLimit=" + userLimit +
" userLimitFactor=" + userLimitFactor +
" required: " + required +
- " consumed: " + user.getTotalConsumedResources() +
+ " consumed: " + user.getUsed() +
" limit: " + limit +
" queueCapacity: " + queueCapacity +
- " qconsumed: " + usedResources +
+ " qconsumed: " + queueUsage.getUsed() +
" currentCapacity: " + currentCapacity +
" activeUsers: " + activeUsers +
" clusterCapacity: " + clusterResource
@@ -1237,7 +1228,7 @@ public class LeafQueue extends AbstractCSQueue {
// overhead of the AM, but it's a > check, not a >= check, so...
if (Resources
.greaterThan(resourceCalculator, clusterResource,
- user.getConsumedResourceByLabel(label),
+ user.getUsed(label),
limit)) {
// if enabled, check to see if could we potentially use this node instead
// of a reserved node if the application has reserved containers
@@ -1245,13 +1236,13 @@ public class LeafQueue extends AbstractCSQueue {
if (Resources.lessThanOrEqual(
resourceCalculator,
clusterResource,
- Resources.subtract(user.getTotalConsumedResources(),
+ Resources.subtract(user.getUsed(),
application.getCurrentReservation()), limit)) {
if (LOG.isDebugEnabled()) {
LOG.debug("User " + userName + " in queue " + getQueueName()
+ " will exceed limit based on reservations - " + " consumed: "
- + user.getTotalConsumedResources() + " reserved: "
+ + user.getUsed() + " reserved: "
+ application.getCurrentReservation() + " limit: " + limit);
}
return true;
@@ -1260,7 +1251,7 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
LOG.debug("User " + userName + " in queue " + getQueueName()
+ " will exceed limit - " + " consumed: "
- + user.getTotalConsumedResources() + " limit: " + limit);
+ + user.getUsed() + " limit: " + limit);
}
return false;
}
@@ -1682,7 +1673,7 @@ public class LeafQueue extends AbstractCSQueue {
" queue=" + this.toString() +
" usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
+ " used=" + queueUsage.getUsed() +
" cluster=" + clusterResource);
return request.getCapability();
@@ -1783,9 +1774,9 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
LOG.info(getQueueName() +
" user=" + userName +
- " used=" + usedResources + " numContainers=" + numContainers +
+ " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
" headroom = " + application.getHeadroom() +
- " user-resources=" + user.getTotalConsumedResources()
+ " user-resources=" + user.getUsed()
);
}
}
@@ -1801,8 +1792,8 @@ public class LeafQueue extends AbstractCSQueue {
metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
LOG.info(getQueueName() +
- " used=" + usedResources + " numContainers=" + numContainers +
- " user=" + userName + " user-resources=" + user.getTotalConsumedResources());
+ " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
+ " user=" + userName + " user-resources=" + user.getUsed());
}
private void updateAbsoluteCapacityResource(Resource clusterResource) {
@@ -1844,22 +1835,20 @@ public class LeafQueue extends AbstractCSQueue {
@VisibleForTesting
public static class User {
- Resource consumed = Resources.createResource(0, 0);
- Resource consumedAMResources = Resources.createResource(0, 0);
- Map<String, Resource> consumedByLabel = new HashMap<String, Resource>();
+ ResourceUsage userResourceUsage = new ResourceUsage();
int pendingApplications = 0;
int activeApplications = 0;
- public Resource getTotalConsumedResources() {
- return consumed;
+ public ResourceUsage getResourceUsage() {
+ return userResourceUsage;
}
- public Resource getConsumedResourceByLabel(String label) {
- Resource r = consumedByLabel.get(label);
- if (null != r) {
- return r;
- }
- return Resources.none();
+ public Resource getUsed() {
+ return userResourceUsage.getUsed();
+ }
+
+ public Resource getUsed(String label) {
+ return userResourceUsage.getUsed(label);
}
public int getPendingApplications() {
@@ -1871,7 +1860,7 @@ public class LeafQueue extends AbstractCSQueue {
}
public Resource getConsumedAMResources() {
- return consumedAMResources;
+ return userResourceUsage.getAMUsed();
}
public int getTotalApplications() {
@@ -1896,47 +1885,26 @@ public class LeafQueue extends AbstractCSQueue {
}
}
- public synchronized void assignContainer(Resource resource,
+ public void assignContainer(Resource resource,
Set<String> nodeLabels) {
- Resources.addTo(consumed, resource);
-
if (nodeLabels == null || nodeLabels.isEmpty()) {
- if (!consumedByLabel.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- consumedByLabel.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.addTo(consumedByLabel.get(RMNodeLabelsManager.NO_LABEL),
- resource);
+ userResourceUsage.incUsed(resource);
} else {
for (String label : nodeLabels) {
- if (!consumedByLabel.containsKey(label)) {
- consumedByLabel.put(label, Resources.createResource(0));
- }
- Resources.addTo(consumedByLabel.get(label), resource);
+ userResourceUsage.incUsed(label, resource);
}
}
}
- public synchronized void releaseContainer(Resource resource, Set<String> nodeLabels) {
- Resources.subtractFrom(consumed, resource);
-
- // Update usedResources by labels
+ public void releaseContainer(Resource resource, Set<String> nodeLabels) {
if (nodeLabels == null || nodeLabels.isEmpty()) {
- if (!consumedByLabel.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- consumedByLabel.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.subtractFrom(
- consumedByLabel.get(RMNodeLabelsManager.NO_LABEL), resource);
+ userResourceUsage.decUsed(resource);
} else {
for (String label : nodeLabels) {
- if (!consumedByLabel.containsKey(label)) {
- consumedByLabel.put(label, Resources.createResource(0));
- }
- Resources.subtractFrom(consumedByLabel.get(label), resource);
+ userResourceUsage.decUsed(label, resource);
}
}
- }
+ }
}
@Override
@@ -1995,7 +1963,7 @@ public class LeafQueue extends AbstractCSQueue {
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
+ " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
- + usedResources + " cluster=" + clusterResource);
+ + queueUsage.getUsed() + " cluster=" + clusterResource);
// Inform the parent queue
getParent().attachContainer(clusterResource, application, rmContainer);
}
@@ -2013,7 +1981,7 @@ public class LeafQueue extends AbstractCSQueue {
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
+ " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
- + usedResources + " cluster=" + clusterResource);
+ + queueUsage.getUsed() + " cluster=" + clusterResource);
// Inform the parent queue
getParent().detachContainer(clusterResource, application, rmContainer);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index f820cca..de92c9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -256,7 +256,7 @@ public class ParentQueue extends AbstractCSQueue {
"numChildQueue= " + childQueues.size() + ", " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
- "usedResources=" + usedResources +
+ "usedResources=" + queueUsage.getUsed() +
"usedCapacity=" + getUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers();
@@ -463,7 +463,7 @@ public class ParentQueue extends AbstractCSQueue {
" queue=" + getQueueName() +
" usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
+ " used=" + queueUsage.getUsed() +
" cluster=" + clusterResource);
} else {
@@ -506,19 +506,16 @@ public class ParentQueue extends AbstractCSQueue {
boolean canAssign = true;
for (String label : labelCanAccess) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
float currentAbsoluteLabelUsedCapacity =
Resources.divide(resourceCalculator, clusterResource,
- usedResourcesByNodeLabels.get(label),
+ queueUsage.getUsed(label),
labelManager.getResourceByLabel(label, clusterResource));
// if any of the label doesn't beyond limit, we can allocate on this node
if (currentAbsoluteLabelUsedCapacity >=
getAbsoluteMaximumCapacityByNodeLabel(label)) {
if (LOG.isDebugEnabled()) {
- LOG.debug(getQueueName() + " used=" + usedResources
- + " current-capacity (" + usedResourcesByNodeLabels.get(label) + ") "
+ LOG.debug(getQueueName() + " used=" + queueUsage.getUsed()
+ + " current-capacity (" + queueUsage.getUsed(label) + ") "
+ " >= max-capacity ("
+ labelManager.getResourceByLabel(label, clusterResource) + ")");
}
@@ -540,16 +537,16 @@ public class ParentQueue extends AbstractCSQueue {
.getReservedMB(), getMetrics().getReservedVirtualCores());
float capacityWithoutReservedCapacity = Resources.divide(
resourceCalculator, clusterResource,
- Resources.subtract(usedResources, reservedResources),
+ Resources.subtract(queueUsage.getUsed(), reservedResources),
clusterResource);
if (capacityWithoutReservedCapacity <= absoluteMaxCapacity) {
if (LOG.isDebugEnabled()) {
LOG.debug("parent: try to use reserved: " + getQueueName()
- + " usedResources: " + usedResources.getMemory()
+ + " usedResources: " + queueUsage.getUsed().getMemory()
+ " clusterResources: " + clusterResource.getMemory()
+ " reservedResources: " + reservedResources.getMemory()
- + " currentCapacity " + ((float) usedResources.getMemory())
+ + " currentCapacity " + ((float) queueUsage.getUsed().getMemory())
/ clusterResource.getMemory()
+ " potentialNewWithoutReservedCapacity: "
+ capacityWithoutReservedCapacity + " ( " + " max-capacity: "
@@ -645,7 +642,7 @@ public class ParentQueue extends AbstractCSQueue {
" queue=" + getQueueName() +
" usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
+ " used=" + queueUsage.getUsed() +
" cluster=" + clusterResource);
}
@@ -735,7 +732,7 @@ public class ParentQueue extends AbstractCSQueue {
.getResource(), node.getLabels());
LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
- + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+ + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
+ clusterResource);
// Inform the parent
if (parent != null) {
@@ -755,7 +752,7 @@ public class ParentQueue extends AbstractCSQueue {
node.getLabels());
LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
- + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+ + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
+ clusterResource);
// Inform the parent
if (parent != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 354a5ab..5c5c2df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -298,7 +298,7 @@ public class TestWorkPreservingRMRestart {
1e-8);
// assert user consumed resources.
assertEquals(usedResource, leafQueue.getUser(app.getUser())
- .getTotalConsumedResources());
+ .getUsed());
}
private void checkFifoQueue(SchedulerApplication schedulerApp,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
index a62889b..d643c9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
@@ -202,33 +202,33 @@ public class TestCSQueueUtils {
LOG.info("t2 l2q2 " + result);
//some usage, but below the base capacity
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
- Resources.addTo(l1q2.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.4f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//usage gt base on parent sibling
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.3f));
- Resources.addTo(l1q2.getUsedResources(), Resources.multiply(clusterResource, 0.3f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+ l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//same as last, but with usage also on direct parent
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
- Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//add to direct sibling, below the threshold of effect at present
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l2q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
@@ -236,9 +236,9 @@ public class TestCSQueueUtils {
//add to direct sibling, now above the threshold of effect
//(it's cumulative with prior tests)
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l2q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.1f, result, 0.000001f);
[2/8] hadoop git commit: YARN-2978. Fixed potential NPE while getting
queue info. Contributed by Varun Saxena (cherry picked from commit
dd57c2047bfd21910acc38c98153eedf1db75169)
Posted by vi...@apache.org.
YARN-2978. Fixed potential NPE while getting queue info. Contributed by Varun Saxena
(cherry picked from commit dd57c2047bfd21910acc38c98153eedf1db75169)
(cherry picked from commit c61e8a7bfa7236e354f859a889083fab3d7ca9eb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/419e18cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/419e18cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/419e18cb
Branch: refs/heads/branch-2.6.1
Commit: 419e18cb3743f9a146abb0c335883c2ac96583e6
Parents: 88f022d
Author: Jian He <ji...@apache.org>
Authored: Tue Jan 6 14:15:57 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:19 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../scheduler/capacity/AbstractCSQueue.java | 24 +++++++++++---------
.../scheduler/capacity/LeafQueue.java | 10 ++++----
.../scheduler/capacity/ParentQueue.java | 4 +---
4 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fa1e120..be9e2d5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -174,6 +174,9 @@ Release 2.6.1 - UNRELEASED
YARN-2920. Changed CapacityScheduler to kill containers on nodes where
node labels are changed. (Wangda Tan via jianhe)
+ YARN-2978. Fixed potential NPE while getting queue info. (Varun Saxena via
+ jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 1f6696d..fec3a56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -65,7 +65,6 @@ public abstract class AbstractCSQueue implements CSQueue {
RMNodeLabelsManager labelManager;
String defaultLabelExpression;
Resource usedResources = Resources.createResource(0, 0);
- QueueInfo queueInfo;
Map<String, Float> absoluteCapacityByNodeLabels;
Map<String, Float> capacitiyByNodeLabels;
Map<String, Resource> usedResourcesByNodeLabels = new HashMap<String, Resource>();
@@ -87,7 +86,6 @@ public abstract class AbstractCSQueue implements CSQueue {
this.parent = parent;
this.queueName = queueName;
this.resourceCalculator = cs.getResourceCalculator();
- this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
// must be called after parent and queueName is set
this.metrics = old != null ? old.getMetrics() :
@@ -99,9 +97,7 @@ public abstract class AbstractCSQueue implements CSQueue {
this.accessibleLabels = cs.getConfiguration().getAccessibleNodeLabels(getQueuePath());
this.defaultLabelExpression = cs.getConfiguration()
.getDefaultNodeLabelExpression(getQueuePath());
-
- this.queueInfo.setQueueName(queueName);
-
+
// inherit from parent if labels not set
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
@@ -280,12 +276,6 @@ public abstract class AbstractCSQueue implements CSQueue {
this.capacitiyByNodeLabels = new HashMap<String, Float>(nodeLabelCapacities);
this.maxCapacityByNodeLabels =
new HashMap<String, Float>(maximumNodeLabelCapacities);
-
- this.queueInfo.setAccessibleNodeLabels(this.accessibleLabels);
- this.queueInfo.setCapacity(this.capacity);
- this.queueInfo.setMaximumCapacity(this.maximumCapacity);
- this.queueInfo.setQueueState(this.state);
- this.queueInfo.setDefaultNodeLabelExpression(this.defaultLabelExpression);
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -330,6 +320,18 @@ public abstract class AbstractCSQueue implements CSQueue {
this.reservationsContinueLooking = reservationContinueLooking;
}
+ protected QueueInfo getQueueInfo() {
+ QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+ queueInfo.setQueueName(queueName);
+ queueInfo.setAccessibleNodeLabels(accessibleLabels);
+ queueInfo.setCapacity(capacity);
+ queueInfo.setMaximumCapacity(maximumCapacity);
+ queueInfo.setQueueState(state);
+ queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
+ queueInfo.setCurrentCapacity(getUsedCapacity());
+ return queueInfo;
+ }
+
@Private
public Resource getMaximumAllocation() {
return maximumAllocation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 509b0f2..78b5fb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -160,8 +160,6 @@ public class LeafQueue extends AbstractCSQueue {
float maxAMResourcePerQueuePercent = cs.getConfiguration()
.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
- this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
-
QueueState state = cs.getConfiguration().getState(getQueuePath());
Map<QueueACL, AccessControlList> acls =
@@ -237,14 +235,14 @@ public class LeafQueue extends AbstractCSQueue {
this.defaultLabelExpression)) {
throw new IOException("Invalid default label expression of "
+ " queue="
- + queueInfo.getQueueName()
+ + getQueueName()
+ " doesn't have permission to access all labels "
+ "in default label expression. labelExpression of resource request="
+ (this.defaultLabelExpression == null ? ""
: this.defaultLabelExpression)
+ ". Queue labels="
- + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
- .getAccessibleNodeLabels().iterator(), ',')));
+ + (getAccessibleNodeLabels() == null ? "" : StringUtils.join(
+ getAccessibleNodeLabels().iterator(), ',')));
}
this.nodeLocalityDelay = nodeLocalityDelay;
@@ -412,7 +410,7 @@ public class LeafQueue extends AbstractCSQueue {
@Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
- queueInfo.setCurrentCapacity(usedCapacity);
+ QueueInfo queueInfo = getQueueInfo();
return queueInfo;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index fd598f2..f820cca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -110,8 +110,6 @@ public class ParentQueue extends AbstractCSQueue {
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
- this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
-
setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
defaultLabelExpression, capacitiyByNodeLabels, maxCapacityByNodeLabels,
@@ -206,7 +204,7 @@ public class ParentQueue extends AbstractCSQueue {
@Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
- queueInfo.setCurrentCapacity(usedCapacity);
+ QueueInfo queueInfo = getQueueInfo();
List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
if (includeChildQueues) {
[8/8] hadoop git commit: YARN-2918. RM should not fail on startup if
queue's configured labels do not exist in cluster-node-labels. Contributed by
Wangda Tan (cherry picked from commit
f489a4ec969f3727d03c8e85d51af1018fc0b2a1)
Posted by vi...@apache.org.
YARN-2918. RM should not fail on startup if queue's configured labels do not exist in cluster-node-labels. Contributed by Wangda Tan
(cherry picked from commit f489a4ec969f3727d03c8e85d51af1018fc0b2a1)
(cherry picked from commit d817fbb34d6e34991c6e512c20d71387750a98f4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23362649
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23362649
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23362649
Branch: refs/heads/branch-2.6.1
Commit: 2336264900feb9dc265a96e660c6c5d1dbd17e2d
Parents: ee2b6bc
Author: Jian He <ji...@apache.org>
Authored: Thu May 7 17:35:41 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sun Sep 6 14:15:33 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../ApplicationMasterService.java | 2 +-
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 5 +-
.../scheduler/SchedulerUtils.java | 67 +++++-----
.../scheduler/capacity/AbstractCSQueue.java | 6 +-
.../scheduler/capacity/CSQueueUtils.java | 28 +---
.../CapacitySchedulerConfiguration.java | 33 +++++
.../scheduler/capacity/LeafQueue.java | 4 +-
.../scheduler/TestSchedulerUtils.java | 134 +++++++++++++++----
.../scheduler/capacity/TestQueueParsing.java | 104 +++++++++++++-
11 files changed, 294 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a6bbdf3..1ad0199 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.6.1 - UNRELEASED
YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track
capacities-by-label. (Wangda Tan via jianhe)
+ YARN-2918. RM should not fail on startup if queue's configured labels do
+ not exist in cluster-node-labels. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index c896bf5..8ed1621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -501,7 +501,7 @@ public class ApplicationMasterService extends AbstractService implements
try {
RMServerUtils.normalizeAndValidateRequests(ask,
rScheduler.getMaximumResourceCapability(), app.getQueue(),
- rScheduler);
+ rScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index fbae710..e7d6f4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -375,7 +375,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
try {
SchedulerUtils.normalizeAndValidateRequest(amReq,
scheduler.getMaximumResourceCapability(),
- submissionContext.getQueue(), scheduler, isRecovery);
+ submissionContext.getQueue(), scheduler, isRecovery, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("RM app submission failed in validating AM resource request"
+ " for application " + submissionContext.getApplicationId(), e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 800cfec..bbab0a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -90,11 +90,12 @@ public class RMServerUtils {
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
- Resource maximumResource, String queueName, YarnScheduler scheduler)
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext)
throws InvalidResourceRequestException {
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
- queueName, scheduler);
+ queueName, scheduler, rmContext);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index bdc9da1..d135485 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -209,7 +210,7 @@ public class SchedulerUtils {
public static void normalizeAndValidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
- boolean isRecovery)
+ boolean isRecovery, RMContext rmContext)
throws InvalidResourceRequestException {
QueueInfo queueInfo = null;
@@ -221,15 +222,16 @@ public class SchedulerUtils {
}
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
- validateResourceRequest(resReq, maximumResource, queueInfo);
+ validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
}
}
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler)
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext)
throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
- false);
+ false, rmContext);
}
/**
@@ -239,8 +241,8 @@ public class SchedulerUtils {
* @throws <code>InvalidResourceRequestException</code> when there is invalid
* request
*/
- public static void validateResourceRequest(ResourceRequest resReq,
- Resource maximumResource, QueueInfo queueInfo)
+ private static void validateResourceRequest(ResourceRequest resReq,
+ Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
if (resReq.getCapability().getMemory() < 0 ||
resReq.getCapability().getMemory() > maximumResource.getMemory()) {
@@ -283,7 +285,7 @@ public class SchedulerUtils {
if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
- labelExp)) {
+ labelExp, rmContext)) {
throw new InvalidResourceRequestException("Invalid resource request"
+ ", queue="
+ queueInfo.getQueueName()
@@ -316,26 +318,6 @@ public class SchedulerUtils {
return false;
}
- public static void checkIfLabelInClusterNodeLabels(RMNodeLabelsManager mgr,
- Set<String> labels) throws IOException {
- if (mgr == null) {
- if (labels != null && !labels.isEmpty()) {
- throw new IOException("NodeLabelManager is null, please check");
- }
- return;
- }
-
- if (labels != null) {
- for (String label : labels) {
- if (!label.equals(RMNodeLabelsManager.ANY)
- && !mgr.containsNodeLabel(label)) {
- throw new IOException("NodeLabelManager doesn't include label = "
- + label + ", please check.");
- }
- }
- }
- }
-
public static boolean checkNodeLabelExpression(Set<String> nodeLabels,
String labelExpression) {
// empty label expression can only allocate on node with empty labels
@@ -356,19 +338,36 @@ public class SchedulerUtils {
return true;
}
+ /**
+ * Check queue label expression, check if node label in queue's
+ * node-label-expression existed in clusterNodeLabels if rmContext != null
+ */
public static boolean checkQueueLabelExpression(Set<String> queueLabels,
- String labelExpression) {
- if (queueLabels != null && queueLabels.contains(RMNodeLabelsManager.ANY)) {
- return true;
- }
+ String labelExpression, RMContext rmContext) {
// if label expression is empty, we can allocate container on any node
if (labelExpression == null) {
return true;
}
for (String str : labelExpression.split("&&")) {
- if (!str.trim().isEmpty()
- && (queueLabels == null || !queueLabels.contains(str.trim()))) {
- return false;
+ str = str.trim();
+ if (!str.trim().isEmpty()) {
+ // check queue label
+ if (queueLabels == null) {
+ return false;
+ } else {
+ if (!queueLabels.contains(str)
+ && !queueLabels.contains(RMNodeLabelsManager.ANY)) {
+ return false;
+ }
+ }
+
+ // check node label manager contains this label
+ if (null != rmContext) {
+ RMNodeLabelsManager nlm = rmContext.getNodeLabelManager();
+ if (nlm != null && !nlm.containsNodeLabel(str)) {
+ return false;
+ }
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 52629b3..f7d49c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -97,11 +97,9 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void setupConfigurableCapacities() {
CSQueueUtils.loadUpdateAndCheckCapacities(
getQueuePath(),
- accessibleLabels,
csContext.getConfiguration(),
queueCapacities,
- parent == null ? null : parent.getQueueCapacities(),
- csContext.getRMContext().getNodeLabelManager());
+ parent == null ? null : parent.getQueueCapacities());
}
@Override
@@ -232,8 +230,6 @@ public abstract class AbstractCSQueue implements CSQueue {
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
}
- SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
- this.accessibleLabels);
// inherit from parent if labels not set
if (this.defaultLabelExpression == null && parent != null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 865b0b4..cbc20b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -117,10 +117,9 @@ class CSQueueUtils {
* - Check if capacities/absolute-capacities legal
*/
public static void loadUpdateAndCheckCapacities(String queuePath,
- Set<String> accessibleLabels, CapacitySchedulerConfiguration csConf,
- QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities,
- RMNodeLabelsManager nlm) {
- loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm,
+ CapacitySchedulerConfiguration csConf,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ loadCapacitiesByLabelsFromConf(queuePath,
queueCapacities, csConf);
updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
@@ -128,28 +127,13 @@ class CSQueueUtils {
capacitiesSanityCheck(queuePath, queueCapacities);
}
- // Considered NO_LABEL, ANY and null cases
- private static Set<String> normalizeAccessibleNodeLabels(Set<String> labels,
- RMNodeLabelsManager mgr) {
- Set<String> accessibleLabels = new HashSet<String>();
- if (labels != null) {
- accessibleLabels.addAll(labels);
- }
- if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) {
- accessibleLabels.addAll(mgr.getClusterNodeLabels());
- }
- accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL);
-
- return accessibleLabels;
- }
-
private static void loadCapacitiesByLabelsFromConf(String queuePath,
- Set<String> labels, RMNodeLabelsManager mgr,
QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) {
queueCapacities.clearConfigurableFields();
- labels = normalizeAccessibleNodeLabels(labels, mgr);
+ Set<String> configuredNodelabels =
+ csConf.getConfiguredNodeLabels(queuePath);
- for (String label : labels) {
+ for (String label : configuredNodelabels) {
if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL,
csConf.getNonLabeledQueueCapacity(queuePath) / 100);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index f5e86b3..7bee438 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -23,8 +23,10 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.StringTokenizer;
@@ -780,4 +782,35 @@ public class CapacitySchedulerConfiguration extends Configuration {
DEFAULT_RESERVATION_ENFORCEMENT_WINDOW);
return enforcementWindow;
}
+
+ /**
+ * Get configured node labels in a given queuePath
+ */
+ public Set<String> getConfiguredNodeLabels(String queuePath) {
+ Set<String> configuredNodeLabels = new HashSet<String>();
+ Entry<String, String> e = null;
+
+ Iterator<Entry<String, String>> iter = iterator();
+ while (iter.hasNext()) {
+ e = iter.next();
+ String key = e.getKey();
+
+ if (key.startsWith(getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS
+ + DOT)) {
+ // Find <label-name> in
+ // <queue-path>.accessible-node-labels.<label-name>.property
+ int labelStartIdx =
+ key.indexOf(ACCESSIBLE_NODE_LABELS)
+ + ACCESSIBLE_NODE_LABELS.length() + 1;
+ int labelEndIndx = key.indexOf('.', labelStartIdx);
+ String labelName = key.substring(labelStartIdx, labelEndIndx);
+ configuredNodeLabels.add(labelName);
+ }
+ }
+
+ // always add NO_LABEL
+ configuredNodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+
+ return configuredNodeLabels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f9f9ca9..f8a6af3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -173,8 +173,8 @@ public class LeafQueue extends AbstractCSQueue {
maxAMResourcePerQueuePercent =
conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
- if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
- this.defaultLabelExpression)) {
+ if (!SchedulerUtils.checkQueueLabelExpression(
+ this.accessibleLabels, this.defaultLabelExpression, null)) {
throw new IOException("Invalid default label expression of "
+ " queue="
+ getQueueName()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 8abaeb6..0fd598c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -64,8 +64,10 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -83,12 +85,15 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
public class TestSchedulerUtils {
private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class);
+ private RMContext rmContext = getMockRMContext();
+
@Test (timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -206,6 +211,8 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y"));
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -213,22 +220,44 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression(" ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is a subset of queue labels");
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
+ }
+
+ // same as above, but cluster node labels don't contains label being
+ // requested. should fail
+ try {
+ // set queue accessible node labesl to [x, y]
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), ResourceRequest.ANY, resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler, rmContext);
+
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
// queue has labels, failed cases (when ask a label not included by queue)
@@ -236,6 +265,8 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y"));
Resource resource = Resources.createResource(
0,
@@ -244,9 +275,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("z");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// we don't allow specify more than two node labels in a single expression
@@ -255,6 +289,8 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y"));
Resource resource = Resources.createResource(
0,
@@ -263,9 +299,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x && y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// queue doesn't have label, succeed (when request no label)
@@ -280,15 +319,15 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression(" ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is empty");
@@ -299,6 +338,9 @@ public class TestSchedulerUtils {
// set queue accessible node labels to empty
queueAccessibleNodeLabels.clear();
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x"));
+
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -306,9 +348,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x"));
}
// queue is "*", always succeeded
@@ -317,6 +362,9 @@ public class TestSchedulerUtils {
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y", "z"));
+
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -324,18 +372,39 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("z");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when queue can access any labels");
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y", "z"));
+ }
+
+ // same as above, but cluster node labels don't contains label, should fail
+ try {
+ // set queue accessible node labels to empty
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), ResourceRequest.ANY, resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler, rmContext);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
// we don't allow resource name other than ANY and specify label
@@ -343,6 +412,9 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x", "y"));
+
Resource resource = Resources.createResource(
0,
@@ -351,9 +423,12 @@ public class TestSchedulerUtils {
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// we don't allow resource name other than ANY and specify label even if
@@ -363,6 +438,8 @@ public class TestSchedulerUtils {
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays
.asList(CommonNodeLabelsManager.ANY));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x"));
Resource resource = Resources.createResource(
0,
@@ -371,9 +448,12 @@ public class TestSchedulerUtils {
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x"));
}
}
@@ -395,7 +475,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Zero memory should be accepted");
}
@@ -409,7 +489,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Zero vcores should be accepted");
}
@@ -424,7 +504,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Max memory should be accepted");
}
@@ -439,7 +519,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Max vcores should not be accepted");
}
@@ -453,7 +533,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("Negative memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -468,7 +548,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("Negative vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -484,7 +564,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("More than max memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -501,7 +581,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("More than max vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -632,4 +712,12 @@ public class TestSchedulerUtils {
Assert.assertNull(applications.get(appId));
return app;
}
+
+ private static RMContext getMockRMContext() {
+ RMContext rmContext = mock(RMContext.class);
+ RMNodeLabelsManager nlm = new MemoryRMNodeLabelsManager();
+ nlm.init(new Configuration(false));
+ when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+ return rmContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index fc03581..d0c5d95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -501,7 +501,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(capacityScheduler);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -529,7 +529,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -557,7 +557,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -585,7 +585,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf =
@@ -659,4 +659,100 @@ public class TestQueueParsing {
DELTA);
capacityScheduler.stop();
}
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabels(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfiguration(csConf);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabels(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfigurationWithLabels(csConf);
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
+ "red", 24);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabels(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfigurationWithLabels(csConf);
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
+ "red", 24);
+ csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
+ ImmutableSet.of(RMNodeLabelsManager.ANY));
+ csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
+ ImmutableSet.of(RMNodeLabelsManager.ANY));
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
}
[3/8] hadoop git commit: YARN-3092. Created a common ResourceUsage
class to track labeled resource usages in Capacity Scheduler. Contributed by
Wangda Tan (cherry picked from commit
6f9fe76918bbc79109653edc6cde85df05148ba3)
Posted by vi...@apache.org.
YARN-3092. Created a common ResourceUsage class to track labeled resource usages in Capacity Scheduler. Contributed by Wangda Tan
(cherry picked from commit 6f9fe76918bbc79109653edc6cde85df05148ba3)
(cherry picked from commit 61b4116b4b3c0eec8f514f079debd88bc757b28e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0ad5538
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0ad5538
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0ad5538
Branch: refs/heads/branch-2.6.1
Commit: b0ad55384191122af160e3268ef448f5b2bc0780
Parents: 419e18c
Author: Jian He <ji...@apache.org>
Authored: Mon Jan 26 15:21:22 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:19 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/ResourceUsage.java | 332 +++++++++++++++++++
.../scheduler/TestResourceUsage.java | 138 ++++++++
3 files changed, 473 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0ad5538/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index be9e2d5..975e4e1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -16,6 +16,9 @@ Release 2.6.1 - UNRELEASED
YARN-1809. Synchronize RM and TimeLineServer Web-UIs. (Zhijie Shen and
Xuan Gong via jianhe)
+ YARN-3092. Created a common ResourceUsage class to track labeled resource
+ usages in Capacity Scheduler. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0ad5538/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
new file mode 100644
index 0000000..5a4cced
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -0,0 +1,332 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * Resource Usage by Labels for following fields by label - AM resource (to
+ * enforce max-am-resource-by-label after YARN-2637) - Used resource (includes
+ * AM resource usage) - Reserved resource - Pending resource - Headroom
+ *
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class ResourceUsage {
+ private ReadLock readLock;
+ private WriteLock writeLock;
+ private Map<String, UsageByLabel> usages;
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public ResourceUsage() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ usages = new HashMap<String, UsageByLabel>();
+ }
+
+ // Usage enum here to make implement cleaner
+ private enum ResourceType {
+ USED(0), PENDING(1), AMUSED(2), RESERVED(3), HEADROOM(4);
+
+ private int value;
+
+ private ResourceType(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return this.value;
+ }
+ }
+
+ private static class UsageByLabel {
+ // usage by label, contains all UsageType
+ private Resource[] resArr;
+
+ public UsageByLabel(String label) {
+ resArr = new Resource[ResourceType.values().length];
+ for (int i = 0; i < resArr.length; i++) {
+ resArr[i] = Resource.newInstance(0, 0);
+ }
+ }
+
+ public Resource get(ResourceType type) {
+ return resArr[type.getValue()];
+ }
+
+ public void set(ResourceType type, Resource res) {
+ resArr[type.getValue()] = res;
+ }
+
+ public void inc(ResourceType type, Resource res) {
+ Resources.addTo(resArr[type.getValue()], res);
+ }
+
+ public void dec(ResourceType type, Resource res) {
+ Resources.subtractFrom(resArr[type.getValue()], res);
+ }
+ }
+
+ /*
+ * Used
+ */
+ public Resource getUsed() {
+ return getUsed(NL);
+ }
+
+ public Resource getUsed(String label) {
+ return internalGet(label, ResourceType.USED);
+ }
+
+ public void incUsed(String label, Resource res) {
+ internalInc(label, ResourceType.USED, res);
+ }
+
+ public void incUsed(Resource res) {
+ incUsed(NL, res);
+ }
+
+ public void decUsed(Resource res) {
+ decUsed(NL, res);
+ }
+
+ public void decUsed(String label, Resource res) {
+ internalDec(label, ResourceType.USED, res);
+ }
+
+ public void setUsed(Resource res) {
+ setUsed(NL, res);
+ }
+
+ public void setUsed(String label, Resource res) {
+ internalSet(label, ResourceType.USED, res);
+ }
+
+ /*
+ * Pending
+ */
+ public Resource getPending() {
+ return getPending(NL);
+ }
+
+ public Resource getPending(String label) {
+ return internalGet(label, ResourceType.PENDING);
+ }
+
+ public void incPending(String label, Resource res) {
+ internalInc(label, ResourceType.PENDING, res);
+ }
+
+ public void incPending(Resource res) {
+ incPending(NL, res);
+ }
+
+ public void decPending(Resource res) {
+ decPending(NL, res);
+ }
+
+ public void decPending(String label, Resource res) {
+ internalDec(label, ResourceType.PENDING, res);
+ }
+
+ public void setPending(Resource res) {
+ setPending(NL, res);
+ }
+
+ public void setPending(String label, Resource res) {
+ internalSet(label, ResourceType.PENDING, res);
+ }
+
+ /*
+ * Reserved
+ */
+ public Resource getReserved() {
+ return getReserved(NL);
+ }
+
+ public Resource getReserved(String label) {
+ return internalGet(label, ResourceType.RESERVED);
+ }
+
+ public void incReserved(String label, Resource res) {
+ internalInc(label, ResourceType.RESERVED, res);
+ }
+
+ public void incReserved(Resource res) {
+ incReserved(NL, res);
+ }
+
+ public void decReserved(Resource res) {
+ decReserved(NL, res);
+ }
+
+ public void decReserved(String label, Resource res) {
+ internalDec(label, ResourceType.RESERVED, res);
+ }
+
+ public void setReserved(Resource res) {
+ setReserved(NL, res);
+ }
+
+ public void setReserved(String label, Resource res) {
+ internalSet(label, ResourceType.RESERVED, res);
+ }
+
+ /*
+ * Headroom
+ */
+ public Resource getHeadroom() {
+ return getHeadroom(NL);
+ }
+
+ public Resource getHeadroom(String label) {
+ return internalGet(label, ResourceType.HEADROOM);
+ }
+
+ public void incHeadroom(String label, Resource res) {
+ internalInc(label, ResourceType.HEADROOM, res);
+ }
+
+ public void incHeadroom(Resource res) {
+ incHeadroom(NL, res);
+ }
+
+ public void decHeadroom(Resource res) {
+ decHeadroom(NL, res);
+ }
+
+ public void decHeadroom(String label, Resource res) {
+ internalDec(label, ResourceType.HEADROOM, res);
+ }
+
+ public void setHeadroom(Resource res) {
+ setHeadroom(NL, res);
+ }
+
+ public void setHeadroom(String label, Resource res) {
+ internalSet(label, ResourceType.HEADROOM, res);
+ }
+
+ /*
+ * AM-Used
+ */
+ public Resource getAMUsed() {
+ return getAMUsed(NL);
+ }
+
+ public Resource getAMUsed(String label) {
+ return internalGet(label, ResourceType.AMUSED);
+ }
+
+ public void incAMUsed(String label, Resource res) {
+ internalInc(label, ResourceType.AMUSED, res);
+ }
+
+ public void incAMUsed(Resource res) {
+ incAMUsed(NL, res);
+ }
+
+ public void decAMUsed(Resource res) {
+ decAMUsed(NL, res);
+ }
+
+ public void decAMUsed(String label, Resource res) {
+ internalDec(label, ResourceType.AMUSED, res);
+ }
+
+ public void setAMUsed(Resource res) {
+ setAMUsed(NL, res);
+ }
+
+ public void setAMUsed(String label, Resource res) {
+ internalSet(label, ResourceType.AMUSED, res);
+ }
+
+ private static Resource normalize(Resource res) {
+ if (res == null) {
+ return Resources.none();
+ }
+ return res;
+ }
+
+ private Resource internalGet(String label, ResourceType type) {
+ try {
+ readLock.lock();
+ UsageByLabel usage = usages.get(label);
+ if (null == usage) {
+ return Resources.none();
+ }
+ return normalize(usage.get(type));
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private UsageByLabel getAndAddIfMissing(String label) {
+ if (!usages.containsKey(label)) {
+ UsageByLabel u = new UsageByLabel(label);
+ usages.put(label, u);
+ return u;
+ }
+
+ return usages.get(label);
+ }
+
+ private void internalSet(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.set(type, res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void internalInc(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.inc(type, res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void internalDec(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.dec(type, res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0ad5538/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
new file mode 100644
index 0000000..b6dfacb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestResourceUsage {
+ private static final Log LOG = LogFactory.getLog(TestResourceUsage.class);
+ private String suffix;
+
+ @Parameterized.Parameters
+ public static Collection<String[]> getParameters() {
+ return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
+ { "Headroom" }, { "Reserved" }, { "AMUsed" } });
+ }
+
+ public TestResourceUsage(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private static void dec(ResourceUsage obj, String suffix, Resource res,
+ String label) throws Exception {
+ executeByName(obj, "dec" + suffix, res, label);
+ }
+
+ private static void inc(ResourceUsage obj, String suffix, Resource res,
+ String label) throws Exception {
+ executeByName(obj, "inc" + suffix, res, label);
+ }
+
+ private static void set(ResourceUsage obj, String suffix, Resource res,
+ String label) throws Exception {
+ executeByName(obj, "set" + suffix, res, label);
+ }
+
+ private static Resource get(ResourceUsage obj, String suffix, String label)
+ throws Exception {
+ return executeByName(obj, "get" + suffix, null, label);
+ }
+
+ // Use reflection to avoid too much avoid code
+ private static Resource executeByName(ResourceUsage obj, String methodName,
+ Resource arg, String label) throws Exception {
+ // We have 4 kinds of method
+ // 1. getXXX() : Resource
+ // 2. getXXX(label) : Resource
+ // 3. set/inc/decXXX(res) : void
+ // 4. set/inc/decXXX(label, res) : void
+ if (methodName.startsWith("get")) {
+ Resource result;
+ if (label == null) {
+ // 1.
+ Method method = ResourceUsage.class.getDeclaredMethod(methodName);
+ result = (Resource) method.invoke(obj);
+ } else {
+ // 2.
+ Method method =
+ ResourceUsage.class.getDeclaredMethod(methodName, String.class);
+ result = (Resource) method.invoke(obj, label);
+ }
+ return result;
+ } else {
+ if (label == null) {
+ // 3.
+ Method method =
+ ResourceUsage.class.getDeclaredMethod(methodName, Resource.class);
+ method.invoke(obj, arg);
+ } else {
+ // 4.
+ Method method =
+ ResourceUsage.class.getDeclaredMethod(methodName, String.class,
+ Resource.class);
+ method.invoke(obj, label, arg);
+ }
+ return null;
+ }
+ }
+
+ private void internalTestModifyAndRead(String label) throws Exception {
+ ResourceUsage usage = new ResourceUsage();
+ Resource res;
+
+ // First get returns 0 always
+ res = get(usage, suffix, label);
+ check(0, 0, res);
+
+ // Add 1,1 should returns 1,1
+ inc(usage, suffix, Resource.newInstance(1, 1), label);
+ check(1, 1, get(usage, suffix, label));
+
+ // Set 2,2
+ set(usage, suffix, Resource.newInstance(2, 2), label);
+ check(2, 2, get(usage, suffix, label));
+
+ // dec 2,2
+ dec(usage, suffix, Resource.newInstance(2, 2), label);
+ check(0, 0, get(usage, suffix, label));
+ }
+
+ void check(int mem, int cpu, Resource res) {
+ Assert.assertEquals(mem, res.getMemory());
+ Assert.assertEquals(cpu, res.getVirtualCores());
+ }
+
+ @Test
+ public void testModifyAndRead() throws Exception {
+ LOG.info("Test - " + suffix);
+ internalTestModifyAndRead(null);
+ internalTestModifyAndRead("label");
+ }
+}
[4/8] hadoop git commit: YARN-3098. Created common QueueCapacities
class in Capacity Scheduler to track capacities-by-labels of queues.
Contributed by Wangda Tan (cherry picked from commit
21d80b3dd90a8e33e51701887c8d9369ed4ab17d)
Posted by vi...@apache.org.
YARN-3098. Created common QueueCapacities class in Capacity Scheduler to track capacities-by-labels of queues. Contributed by Wangda Tan
(cherry picked from commit 21d80b3dd90a8e33e51701887c8d9369ed4ab17d)
(cherry picked from commit c0b1311a93614becc4a255af48fb7b697d491b80)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c94f071
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c94f071
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c94f071
Branch: refs/heads/branch-2.6.1
Commit: 4c94f0714012e1a7cfce1ea7b60d3c86f0b5cce1
Parents: d9281fb
Author: Jian He <ji...@apache.org>
Authored: Tue Feb 3 11:43:12 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:20 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/ResourceUsage.java | 80 +++-----
.../scheduler/capacity/QueueCapacities.java | 191 +++++++++++++++++++
.../scheduler/capacity/TestQueueCapacities.java | 127 ++++++++++++
4 files changed, 351 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bac13ff..58c2139 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -19,6 +19,9 @@ Release 2.6.1 - UNRELEASED
YARN-3092. Created a common ResourceUsage class to track labeled resource
usages in Capacity Scheduler. (Wangda Tan via jianhe)
+ YARN-3098. Created common QueueCapacities class in Capacity Scheduler to
+ track capacities-by-labels of queues. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 5a4cced..c651878 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -56,14 +56,10 @@ public class ResourceUsage {
private enum ResourceType {
USED(0), PENDING(1), AMUSED(2), RESERVED(3), HEADROOM(4);
- private int value;
+ private int idx;
private ResourceType(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return this.value;
+ this.idx = value;
}
}
@@ -77,22 +73,6 @@ public class ResourceUsage {
resArr[i] = Resource.newInstance(0, 0);
}
}
-
- public Resource get(ResourceType type) {
- return resArr[type.getValue()];
- }
-
- public void set(ResourceType type, Resource res) {
- resArr[type.getValue()] = res;
- }
-
- public void inc(ResourceType type, Resource res) {
- Resources.addTo(resArr[type.getValue()], res);
- }
-
- public void dec(ResourceType type, Resource res) {
- Resources.subtractFrom(resArr[type.getValue()], res);
- }
}
/*
@@ -103,11 +83,11 @@ public class ResourceUsage {
}
public Resource getUsed(String label) {
- return internalGet(label, ResourceType.USED);
+ return _get(label, ResourceType.USED);
}
public void incUsed(String label, Resource res) {
- internalInc(label, ResourceType.USED, res);
+ _inc(label, ResourceType.USED, res);
}
public void incUsed(Resource res) {
@@ -119,7 +99,7 @@ public class ResourceUsage {
}
public void decUsed(String label, Resource res) {
- internalDec(label, ResourceType.USED, res);
+ _dec(label, ResourceType.USED, res);
}
public void setUsed(Resource res) {
@@ -127,7 +107,7 @@ public class ResourceUsage {
}
public void setUsed(String label, Resource res) {
- internalSet(label, ResourceType.USED, res);
+ _set(label, ResourceType.USED, res);
}
/*
@@ -138,11 +118,11 @@ public class ResourceUsage {
}
public Resource getPending(String label) {
- return internalGet(label, ResourceType.PENDING);
+ return _get(label, ResourceType.PENDING);
}
public void incPending(String label, Resource res) {
- internalInc(label, ResourceType.PENDING, res);
+ _inc(label, ResourceType.PENDING, res);
}
public void incPending(Resource res) {
@@ -154,7 +134,7 @@ public class ResourceUsage {
}
public void decPending(String label, Resource res) {
- internalDec(label, ResourceType.PENDING, res);
+ _dec(label, ResourceType.PENDING, res);
}
public void setPending(Resource res) {
@@ -162,7 +142,7 @@ public class ResourceUsage {
}
public void setPending(String label, Resource res) {
- internalSet(label, ResourceType.PENDING, res);
+ _set(label, ResourceType.PENDING, res);
}
/*
@@ -173,11 +153,11 @@ public class ResourceUsage {
}
public Resource getReserved(String label) {
- return internalGet(label, ResourceType.RESERVED);
+ return _get(label, ResourceType.RESERVED);
}
public void incReserved(String label, Resource res) {
- internalInc(label, ResourceType.RESERVED, res);
+ _inc(label, ResourceType.RESERVED, res);
}
public void incReserved(Resource res) {
@@ -189,7 +169,7 @@ public class ResourceUsage {
}
public void decReserved(String label, Resource res) {
- internalDec(label, ResourceType.RESERVED, res);
+ _dec(label, ResourceType.RESERVED, res);
}
public void setReserved(Resource res) {
@@ -197,7 +177,7 @@ public class ResourceUsage {
}
public void setReserved(String label, Resource res) {
- internalSet(label, ResourceType.RESERVED, res);
+ _set(label, ResourceType.RESERVED, res);
}
/*
@@ -208,11 +188,11 @@ public class ResourceUsage {
}
public Resource getHeadroom(String label) {
- return internalGet(label, ResourceType.HEADROOM);
+ return _get(label, ResourceType.HEADROOM);
}
public void incHeadroom(String label, Resource res) {
- internalInc(label, ResourceType.HEADROOM, res);
+ _inc(label, ResourceType.HEADROOM, res);
}
public void incHeadroom(Resource res) {
@@ -224,7 +204,7 @@ public class ResourceUsage {
}
public void decHeadroom(String label, Resource res) {
- internalDec(label, ResourceType.HEADROOM, res);
+ _dec(label, ResourceType.HEADROOM, res);
}
public void setHeadroom(Resource res) {
@@ -232,7 +212,7 @@ public class ResourceUsage {
}
public void setHeadroom(String label, Resource res) {
- internalSet(label, ResourceType.HEADROOM, res);
+ _set(label, ResourceType.HEADROOM, res);
}
/*
@@ -243,11 +223,11 @@ public class ResourceUsage {
}
public Resource getAMUsed(String label) {
- return internalGet(label, ResourceType.AMUSED);
+ return _get(label, ResourceType.AMUSED);
}
public void incAMUsed(String label, Resource res) {
- internalInc(label, ResourceType.AMUSED, res);
+ _inc(label, ResourceType.AMUSED, res);
}
public void incAMUsed(Resource res) {
@@ -259,7 +239,7 @@ public class ResourceUsage {
}
public void decAMUsed(String label, Resource res) {
- internalDec(label, ResourceType.AMUSED, res);
+ _dec(label, ResourceType.AMUSED, res);
}
public void setAMUsed(Resource res) {
@@ -267,7 +247,7 @@ public class ResourceUsage {
}
public void setAMUsed(String label, Resource res) {
- internalSet(label, ResourceType.AMUSED, res);
+ _set(label, ResourceType.AMUSED, res);
}
private static Resource normalize(Resource res) {
@@ -277,14 +257,14 @@ public class ResourceUsage {
return res;
}
- private Resource internalGet(String label, ResourceType type) {
+ private Resource _get(String label, ResourceType type) {
try {
readLock.lock();
UsageByLabel usage = usages.get(label);
if (null == usage) {
return Resources.none();
}
- return normalize(usage.get(type));
+ return normalize(usage.resArr[type.idx]);
} finally {
readLock.unlock();
}
@@ -300,31 +280,31 @@ public class ResourceUsage {
return usages.get(label);
}
- private void internalSet(String label, ResourceType type, Resource res) {
+ private void _set(String label, ResourceType type, Resource res) {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.set(type, res);
+ usage.resArr[type.idx] = res;
} finally {
writeLock.unlock();
}
}
- private void internalInc(String label, ResourceType type, Resource res) {
+ private void _inc(String label, ResourceType type, Resource res) {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.inc(type, res);
+ Resources.addTo(usage.resArr[type.idx], res);
} finally {
writeLock.unlock();
}
}
- private void internalDec(String label, ResourceType type, Resource res) {
+ private void _dec(String label, ResourceType type, Resource res) {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.dec(type, res);
+ Resources.subtractFrom(usage.resArr[type.idx], res);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
new file mode 100644
index 0000000..a0e6d8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+public class QueueCapacities {
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+ private static final float LABEL_DOESNT_EXIST_CAP = 0f;
+ private Map<String, Capacities> capacitiesMap;
+ private ReadLock readLock;
+ private WriteLock writeLock;
+
+ public QueueCapacities() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ capacitiesMap = new HashMap<String, Capacities>();
+ }
+
+ // Usage enum here to make implement cleaner
+ private enum CapacityType {
+ USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5);
+
+ private int idx;
+
+ private CapacityType(int idx) {
+ this.idx = idx;
+ }
+ }
+
+ private static class Capacities {
+ private float[] capacitiesArr;
+
+ public Capacities() {
+ capacitiesArr = new float[CapacityType.values().length];
+ }
+ }
+
+ private float _get(String label, CapacityType type) {
+ try {
+ readLock.lock();
+ Capacities cap = capacitiesMap.get(label);
+ if (null == cap) {
+ return LABEL_DOESNT_EXIST_CAP;
+ }
+ return cap.capacitiesArr[type.idx];
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private void _set(String label, CapacityType type, float value) {
+ try {
+ writeLock.lock();
+ Capacities cap = capacitiesMap.get(label);
+ if (null == cap) {
+ cap = new Capacities();
+ capacitiesMap.put(label, cap);
+ }
+ cap.capacitiesArr[type.idx] = value;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /* Used Capacity Getter and Setter */
+ public float getUsedCapacity() {
+ return _get(NL, CapacityType.USED_CAP);
+ }
+
+ public float getUsedCapacity(String label) {
+ return _get(label, CapacityType.USED_CAP);
+ }
+
+ public void setUsedCapacity(float value) {
+ _set(NL, CapacityType.USED_CAP, value);
+ }
+
+ public void setUsedCapacity(String label, float value) {
+ _set(label, CapacityType.USED_CAP, value);
+ }
+
+ /* Absolute Used Capacity Getter and Setter */
+ public float getAbsoluteUsedCapacity() {
+ return _get(NL, CapacityType.ABS_USED_CAP);
+ }
+
+ public float getAbsoluteUsedCapacity(String label) {
+ return _get(label, CapacityType.ABS_USED_CAP);
+ }
+
+ public void setAbsoluteUsedCapacity(float value) {
+ _set(NL, CapacityType.ABS_USED_CAP, value);
+ }
+
+ public void setAbsoluteUsedCapacity(String label, float value) {
+ _set(label, CapacityType.ABS_USED_CAP, value);
+ }
+
+ /* Capacity Getter and Setter */
+ public float getCapacity() {
+ return _get(NL, CapacityType.CAP);
+ }
+
+ public float getCapacity(String label) {
+ return _get(label, CapacityType.CAP);
+ }
+
+ public void setCapacity(float value) {
+ _set(NL, CapacityType.CAP, value);
+ }
+
+ public void setCapacity(String label, float value) {
+ _set(label, CapacityType.CAP, value);
+ }
+
+ /* Absolute Capacity Getter and Setter */
+ public float getAbsoluteCapacity() {
+ return _get(NL, CapacityType.ABS_CAP);
+ }
+
+ public float getAbsoluteCapacity(String label) {
+ return _get(label, CapacityType.ABS_CAP);
+ }
+
+ public void setAbsoluteCapacity(float value) {
+ _set(NL, CapacityType.ABS_CAP, value);
+ }
+
+ public void setAbsoluteCapacity(String label, float value) {
+ _set(label, CapacityType.ABS_CAP, value);
+ }
+
+ /* Maximum Capacity Getter and Setter */
+ public float getMaximumCapacity() {
+ return _get(NL, CapacityType.MAX_CAP);
+ }
+
+ public float getMaximumCapacity(String label) {
+ return _get(label, CapacityType.MAX_CAP);
+ }
+
+ public void setMaximumCapacity(float value) {
+ _set(NL, CapacityType.MAX_CAP, value);
+ }
+
+ public void setMaximumCapacity(String label, float value) {
+ _set(label, CapacityType.MAX_CAP, value);
+ }
+
+ /* Absolute Maximum Capacity Getter and Setter */
+ public float getAbsoluteMaximumCapacity() {
+ return _get(NL, CapacityType.ABS_MAX_CAP);
+ }
+
+ public float getAbsoluteMaximumCapacity(String label) {
+ return _get(label, CapacityType.ABS_MAX_CAP);
+ }
+
+ public void setAbsoluteMaximumCapacity(float value) {
+ _set(NL, CapacityType.ABS_MAX_CAP, value);
+ }
+
+ public void setAbsoluteMaximumCapacity(String label, float value) {
+ _set(label, CapacityType.ABS_MAX_CAP, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
new file mode 100644
index 0000000..89a5311
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestQueueCapacities {
+ private static final Log LOG = LogFactory.getLog(TestQueueCapacities.class);
+ private String suffix;
+
+ @Parameterized.Parameters
+ public static Collection<String[]> getParameters() {
+ return Arrays.asList(new String[][] {
+ { "Capacity" },
+ { "AbsoluteCapacity" },
+ { "UsedCapacity" },
+ { "AbsoluteUsedCapacity" },
+ { "MaximumCapacity" },
+ { "AbsoluteMaximumCapacity" } });
+ }
+
+ public TestQueueCapacities(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private static float get(QueueCapacities obj, String suffix,
+ String label) throws Exception {
+ return executeByName(obj, "get" + suffix, label, -1f);
+ }
+
+ private static void set(QueueCapacities obj, String suffix,
+ String label, float value) throws Exception {
+ executeByName(obj, "set" + suffix, label, value);
+ }
+
+ // Use reflection to avoid too much avoid code
+ private static float executeByName(QueueCapacities obj, String methodName,
+ String label, float value) throws Exception {
+ // We have 4 kinds of method
+ // 1. getXXX() : float
+ // 2. getXXX(label) : float
+ // 3. setXXX(float) : void
+ // 4. setXXX(label, float) : void
+ if (methodName.startsWith("get")) {
+ float result;
+ if (label == null) {
+ // 1.
+ Method method = QueueCapacities.class.getDeclaredMethod(methodName);
+ result = (Float) method.invoke(obj);
+ } else {
+ // 2.
+ Method method =
+ QueueCapacities.class.getDeclaredMethod(methodName, String.class);
+ result = (Float) method.invoke(obj, label);
+ }
+ return result;
+ } else {
+ if (label == null) {
+ // 3.
+ Method method =
+ QueueCapacities.class.getDeclaredMethod(methodName, Float.TYPE);
+ method.invoke(obj, value);
+ } else {
+ // 4.
+ Method method =
+ QueueCapacities.class.getDeclaredMethod(methodName, String.class,
+ Float.TYPE);
+ method.invoke(obj, label, value);
+ }
+ return -1f;
+ }
+ }
+
+ private void internalTestModifyAndRead(String label) throws Exception {
+ QueueCapacities qc = new QueueCapacities();
+
+ // First get returns 0 always
+ Assert.assertEquals(0f, get(qc, suffix, label), 1e-8);
+
+ // Set to 1, and check
+ set(qc, suffix, label, 1f);
+ Assert.assertEquals(1f, get(qc, suffix, label), 1e-8);
+
+ // Set to 2, and check
+ set(qc, suffix, label, 2f);
+ Assert.assertEquals(2f, get(qc, suffix, label), 1e-8);
+ }
+
+ void check(int mem, int cpu, Resource res) {
+ Assert.assertEquals(mem, res.getMemory());
+ Assert.assertEquals(cpu, res.getVirtualCores());
+ }
+
+ @Test
+ public void testModifyAndRead() throws Exception {
+ LOG.info("Test - " + suffix);
+ internalTestModifyAndRead(null);
+ internalTestModifyAndRead("label");
+ }
+}
\ No newline at end of file