You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/08/11 17:32:29 UTC
[49/50] [abbrv] hadoop git commit: YARN-6471. Support to add min/max
resource configuration for a queue. (Sunil G via wangda)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index f6ada4f..5b529d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
@@ -45,7 +44,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSetUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.io.IOException;
@@ -69,6 +68,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
@Private
@Evolving
@@ -163,31 +163,78 @@ public class ParentQueue extends AbstractCSQueue {
writeLock.lock();
// Validate
float childCapacities = 0;
+ Resource minResDefaultLabel = Resources.createResource(0, 0);
for (CSQueue queue : childQueues) {
childCapacities += queue.getCapacity();
+ Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource());
+
+ // If any child queue is using percentage based capacity model vs parent
+ // queues' absolute configuration or vice versa, throw back an
+ // exception.
+ if (!queueName.equals("root") && getCapacity() != 0f
+ && !queue.getQueueResourceQuotas().getConfiguredMinResource()
+ .equals(Resources.none())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource together.");
+ }
}
+
float delta = Math.abs(1.0f - childCapacities); // crude way to check
// allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || (
- (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
- throw new IllegalArgumentException(
- "Illegal" + " capacity of " + childCapacities
- + " for children of queue " + queueName);
+ if ((minResDefaultLabel.equals(Resources.none())
+ && (queueCapacities.getCapacity() > 0) && (delta > PRECISION))
+ || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
+ throw new IllegalArgumentException("Illegal" + " capacity of "
+ + childCapacities + " for children of queue " + queueName);
}
// check label capacities
for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
// check children's labels
float sum = 0;
+ Resource minRes = Resources.createResource(0, 0);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+ scheduler.getClusterResource());
for (CSQueue queue : childQueues) {
sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+
+ // If any child queue of a label is using percentage based capacity
+ // model vs parent queues' absolute configuration or vice versa, throw
+ // back an exception
+ if (!queueName.equals("root") && !this.capacityConfigType
+ .equals(queue.getCapacityConfigType())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + "configuration or absolute resource together for label:"
+ + nodeLabel);
+ }
+
+ // Accumulate all min/max resource configured for all child queues.
+ Resources.addTo(minRes, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource(nodeLabel));
}
- if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
+ if ((minResDefaultLabel.equals(Resources.none()) && capacityByLabel > 0
+ && Math.abs(1.0f - sum) > PRECISION)
|| (capacityByLabel == 0) && (sum > 0)) {
throw new IllegalArgumentException(
"Illegal" + " capacity of " + sum + " for children of queue "
+ queueName + " for label=" + nodeLabel);
}
+
+ // Ensure that for each parent queue: parent.min-resource >=
+ // Σ(child.min-resource).
+ Resource parentMinResource = queueResourceQuotas
+ .getConfiguredMinResource(nodeLabel);
+ if (!parentMinResource.equals(Resources.none()) && Resources.lessThan(
+ resourceCalculator, resourceByLabel, parentMinResource, minRes)) {
+ throw new IllegalArgumentException("Parent Queues" + " capacity: "
+ + parentMinResource + " is less than" + " to its children:"
+ + minRes + " for queue:" + queueName);
+ }
}
this.childQueues.clear();
@@ -690,11 +737,8 @@ public class ParentQueue extends AbstractCSQueue {
child.getQueueResourceUsage().getUsed(nodePartition));
// Get child's max resource
- Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
+ nodePartition, minimumAllocation);
// Child's limit should be capped by child configured max resource
childLimit =
@@ -830,6 +874,14 @@ public class ParentQueue extends AbstractCSQueue {
ResourceLimits resourceLimits) {
try {
writeLock.lock();
+
+ // Update effective capacity in all parent queue.
+ Set<String> configuredNodelabels = csContext.getConfiguration()
+ .getConfiguredNodeLabels(getQueuePath());
+ for (String label : configuredNodelabels) {
+ calculateEffectiveResourcesAndCapacity(label, clusterResource);
+ }
+
// Update all children
for (CSQueue childQueue : childQueues) {
// Get ResourceLimits of child queue before assign containers
@@ -851,6 +903,110 @@ public class ParentQueue extends AbstractCSQueue {
return true;
}
+ private void calculateEffectiveResourcesAndCapacity(String label,
+ Resource clusterResource) {
+
+ // For root queue, ensure that max/min resource is updated to latest
+ // cluster resource.
+ Resource resourceByLabel = labelManager.getResourceByLabel(label,
+ clusterResource);
+ if (getQueueName().equals("root")) {
+ queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel);
+ queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ }
+
+ // Total configured min resources of direct children of queue
+ Resource configuredMinResources = Resource.newInstance(0L, 0);
+ for (CSQueue childQueue : getChildQueues()) {
+ Resources.addTo(configuredMinResources,
+ childQueue.getQueueResourceQuotas().getConfiguredMinResource(label));
+ }
+
+ // Factor to scale down effective resource: When cluster has sufficient
+ // resources, effective_min_resources will be same as configured
+ // min_resources.
+ float effectiveMinRatio = 1;
+ ResourceCalculator rc = this.csContext.getResourceCalculator();
+ if (getQueueName().equals("root")) {
+ if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
+ clusterResource, resourceByLabel, configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ resourceByLabel, configuredMinResources);
+ }
+ } else {
+ if (Resources.lessThan(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources);
+ }
+ }
+
+ // loop and do this for all child queues
+ for (CSQueue childQueue : getChildQueues()) {
+ Resource minResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMinResource(label);
+
+ // Update effective resource (min/max) to each child queue.
+ if (childQueue.getCapacityConfigType()
+ .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(minResource, effectiveMinRatio));
+
+ // Max resource of a queue should be a minimum of {configuredMaxRes,
+ // parentMaxRes}. parentMaxRes could be configured value. But if not
+ // present could also be taken from effective max resource of parent.
+ Resource parentMaxRes = queueResourceQuotas
+ .getConfiguredMaxResource(label);
+ if (parentMaxRes.equals(Resources.none())) {
+ parentMaxRes = parent.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label);
+ }
+
+ // Minimum of {childMaxResource, parentMaxRes}. However if
+ // childMaxResource is empty, consider parent's max resource alone.
+ Resource childMaxResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ Resource effMaxResource = Resources.min(resourceCalculator,
+ resourceByLabel, childMaxResource.equals(Resources.none())
+ ? parentMaxRes
+ : childMaxResource,
+ parentMaxRes);
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.clone(effMaxResource));
+ } else {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label)));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label)));
+
+ childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label),
+ minimumAllocation));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc,
+ resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label),
+ minimumAllocation));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating effective min resource for queue:"
+ + childQueue.getQueueName() + " as effMinResource="
+ + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label)
+ + "and Updating effective max resource as effMaxResource="
+ + childQueue.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label));
+ }
+ }
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
@@ -980,9 +1136,21 @@ public class ParentQueue extends AbstractCSQueue {
* When this happens, we have to preempt killable container (on same or different
* nodes) of parent queue to avoid violating parent's max resource.
*/
- if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
- < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
- killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+ if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition)
+ .equals(Resources.none())) {
+ if (Resources.lessThan(resourceCalculator, clusterResource,
+ queueResourceQuotas.getEffectiveMaxResource(nodePartition),
+ queueUsage.getUsed(nodePartition))) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
+ } else {
+ if (getQueueCapacities()
+ .getAbsoluteMaximumCapacity(nodePartition) < getQueueCapacities()
+ .getAbsoluteUsedCapacity(nodePartition)) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
}
} finally {
writeLock.unlock();
@@ -999,8 +1167,7 @@ public class ParentQueue extends AbstractCSQueue {
Resource partitionResource = labelManager.getResourceByLabel(partition,
null);
- Resource maxResource = Resources.multiply(partitionResource,
- getQueueCapacities().getAbsoluteMaximumCapacity(partition));
+ Resource maxResource = getEffectiveMaxCapacity(partition);
while (Resources.greaterThan(resourceCalculator, partitionResource,
queueUsage.getUsed(partition), maxResource)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 5f7d185..a066a35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,10 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = Resources.multiplyAndNormalizeUp(
- resourceCalculator, partitionResource,
- lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition),
- lQueue.getMinimumAllocation());
+ Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 0544387..4985a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
import java.util.Collections;
@@ -121,6 +123,15 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
// For queue with same used ratio / priority, queue with higher configured
// capacity goes first
if (0 == rc) {
+ Resource minEffRes1 = q1.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ Resource minEffRes2 = q2.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ if (!minEffRes1.equals(Resources.none())
+ && !minEffRes2.equals(Resources.none())) {
+ return minEffRes2.compareTo(minEffRes1);
+ }
+
float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p);
float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p);
return Float.compare(abs2, abs1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 22705cc..86b2fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -62,6 +62,8 @@ public class CapacitySchedulerQueueInfo {
protected long pendingContainers;
protected QueueCapacitiesInfo capacities;
protected ResourcesInfo resources;
+ protected ResourceInfo minEffectiveCapacity;
+ protected ResourceInfo maxEffectiveCapacity;
CapacitySchedulerQueueInfo() {
};
@@ -105,6 +107,11 @@ public class CapacitySchedulerQueueInfo {
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
+
+ minEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMinResource());
+ maxEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMaxResource());
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -200,4 +207,12 @@ public class CapacitySchedulerQueueInfo {
public ResourcesInfo getResources() {
return resources;
}
+
+ public ResourceInfo getMinEffectiveCapacity(){
+ return minEffectiveCapacity;
+ }
+
+ public ResourceInfo getMaxEffectiveCapacity(){
+ return maxEffectiveCapacity;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 2d76127..30cb8d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -117,6 +118,13 @@ public class MockNM {
true, ++responseId);
}
+ public void unRegisterNode() throws Exception {
+ UnRegisterNodeManagerRequest request = Records
+ .newRecord(UnRegisterNodeManagerRequest.class);
+ request.setNodeId(nodeId);
+ resourceTracker.unRegisterNodeManager(request);
+ }
+
public RegisterNodeManagerResponse registerNode() throws Exception {
return registerNode(null, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index e967807..4ccbb92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -803,6 +803,12 @@ public class MockRM extends ResourceManager {
return rmApp;
}
+ public MockNM unRegisterNode(MockNM nm) throws Exception {
+ nm.unRegisterNode();
+ drainEventsImplicitly();
+ return nm;
+ }
+
public MockNM registerNode(String nodeIdStr, int memory) throws Exception {
MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService());
nm.registerNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 4fc0ea4..591d5f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
@@ -641,9 +642,11 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
QueueCapacities qc = new QueueCapacities(0 == myLevel);
ResourceUsage ru = new ResourceUsage();
+ QueueResourceQuotas qr = new QueueResourceQuotas();
when(queue.getQueueCapacities()).thenReturn(qc);
when(queue.getQueueResourceUsage()).thenReturn(ru);
+ when(queue.getQueueResourceQuotas()).thenReturn(qr);
LOG.debug("Setup queue, name=" + queue.getQueueName() + " path="
+ queue.getQueuePath());
@@ -676,7 +679,17 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
qc.setAbsoluteMaximumCapacity(partitionName, absMax);
qc.setAbsoluteUsedCapacity(partitionName, absUsed);
qc.setUsedCapacity(partitionName, used);
+ qr.setEffectiveMaxResource(parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(parseResourceFromString(values[0].trim()));
+ qr.setEffectiveMaxResource(partitionName,
+ parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(partitionName,
+ parseResourceFromString(values[0].trim()));
when(queue.getUsedCapacity()).thenReturn(used);
+ when(queue.getEffectiveCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[0].trim()));
+ when(queue.getEffectiveMaxCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[1].trim()));
ru.setPending(partitionName, pending);
// Setup reserved resource if it contained by input config
Resource reserved = Resources.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index a14a2b1..b881323 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.Clock;
@@ -435,8 +435,8 @@ public class TestProportionalCapacityPreemptionPolicy {
policy.editSchedule();
// queueF(appD) wants resources, Verify that resources come from queueE(appC)
// because it's a sibling and queueB(appA) because queueA is over capacity.
- verify(mDisp, times(28)).handle(argThat(new IsPreemptionRequestFor(appA)));
- verify(mDisp, times(22)).handle(argThat(new IsPreemptionRequestFor(appC)));
+ verify(mDisp, times(27)).handle(argThat(new IsPreemptionRequestFor(appA)));
+ verify(mDisp, times(23)).handle(argThat(new IsPreemptionRequestFor(appC)));
// Need to call setup() again to reset mDisp
setup();
@@ -1170,6 +1170,17 @@ public class TestProportionalCapacityPreemptionPolicy {
when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
boolean preemptionDisabled = mockPreemptionStatus("root");
when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
+ QueueResourceQuotas rootQr = new QueueResourceQuotas();
+ rootQr.setEffectiveMaxResource(Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(abs[0]);
+ rootQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[0]);
+ when(root.getQueueResourceQuotas()).thenReturn(rootQr);
+ when(root.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[0]);
+ when(root.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[0], 0));
for (int i = 1; i < queues.length; ++i) {
final CSQueue q;
@@ -1200,6 +1211,18 @@ public class TestProportionalCapacityPreemptionPolicy {
qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
when(q.getQueueCapacities()).thenReturn(qc);
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ qr.setEffectiveMaxResource(Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(abs[i]);
+ qr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]);
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
+ when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[i]);
+ when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[i], 0));
+
String parentPathName = p.getQueuePath();
parentPathName = (parentPathName == null) ? "root" : parentPathName;
String queuePathName = (parentPathName + "." + queueName).replace("/",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
index 7784549..a1d89d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
@@ -67,9 +67,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY,
"priority_first");
- String labelsConfig = "=100:200,true;";
+ String labelsConfig = "=100:50,true;";
String nodesConfig = // n1 has no label
- "n1= res=100:200";
+ "n1= res=100:50";
String queuesConfig =
// guaranteed,max,used,pending,reserved
"root(=[100:50 100:50 80:40 120:60 0]);" + // root
@@ -105,7 +105,7 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
- verify(mDisp, times(7)).handle(argThat(
+ verify(mDisp, times(3)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
new file mode 100644
index 0000000..5a66281
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -0,0 +1,516 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAbsoluteResourceConfiguration {
+
+ private static final int GB = 1024;
+
+ private static final String QUEUEA = "queueA";
+ private static final String QUEUEB = "queueB";
+ private static final String QUEUEC = "queueC";
+ private static final String QUEUEA1 = "queueA1";
+ private static final String QUEUEA2 = "queueA2";
+ private static final String QUEUEB1 = "queueB1";
+
+ private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEA;
+ private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEB;
+ private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEC;
+ private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1;
+ private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2;
+ private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1;
+
+ private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB,
+ 10);
+ private static final Resource QUEUE_A_MAXRES = Resource.newInstance(200 * GB,
+ 30);
+ private static final Resource QUEUE_A1_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_A2_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_B_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_B1_MINRES = Resource.newInstance(40 * GB,
+ 10);
+ private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB,
+ 30);
+ private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB,
+ 20);
+ private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6);
+ private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000,
+ 20);
+
+ private static Set<String> resourceTypes = new HashSet<>(
+ Arrays.asList("memory", "vcores"));
+
+ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+ csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ csConf.setCapacity(QUEUEA1_FULL, 50f);
+ csConf.setCapacity(QUEUEA2_FULL, 50f);
+ csConf.setCapacity(QUEUEB1_FULL, 100f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ @Test
+ public void testSimpleMinMaxResourceConfigurartionPerQueue() {
+
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true);
+ setupMinMaxResourceConfiguration(csConf);
+
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ }
+
+ @Test
+ public void testEffectiveMinMaxResourceConfigurartionPerQueue()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+
+ @Test
+ public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases 1) Configure percentage based capacity and absolute
+ * resource together. 2) As per above tree structure, ensure all values
+ * could be retrieved. 3) Validate whether min resource cannot be more than
+ * max resources. 4) Validate whether max resource of queue cannot be more
+ * than its parent max resource.
+ */
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // 1. Create a new config with capcity and min/max together. Ensure an
+ // exception is thrown.
+ CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
+ true);
+ setupMinMaxResourceConfiguration(csConf1);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Queue 'queueA' should use either"
+ + " percentage based capacity configuration or absolute resource.",
+ e.getMessage());
+ }
+ rm.stop();
+
+ // 2. Create a new config with min/max alone with a complex queue config.
+ // Check all values could be fetched correctly.
+ CapacitySchedulerConfiguration csConf2 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf2);
+
+ rm = new MockRM(csConf2);
+ rm.start();
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+ cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+ Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+ QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+ QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+ Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+ QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+ QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+ Assert.assertNotNull(qB1);
+ Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // 3. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf3 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf3);
+
+ csConf3.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B_MAXRES);
+ csConf3.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ try {
+ cs.reinitialize(csConf3, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Min resource configuration "
+ + "<memory:153600, vCores:30> is greater than its "
+ + "max value:<memory:40960, vCores:10> in queue:queueB1",
+ e.getMessage());
+ }
+
+ // 4. Create a new config and make sure one queue's max resource is more
+ // than its preant's max resource configured.
+ CapacitySchedulerConfiguration csConf4 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf4);
+
+ csConf4.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_A_MAXRES);
+
+ try {
+ cs.reinitialize(csConf4, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert
+ .assertEquals(
+ "Failed to re-init queues : Max resource configuration "
+ + "<memory:204800, vCores:30> is greater than parents max value:"
+ + "<memory:153600, vCores:30> in queue:queueB1",
+ e.getMessage());
+ }
+ rm.stop();
+ }
+
+ @Test
+ public void testComplexValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases: 1) Parent and its child queues must use either
+ * percentage based or absolute resource configuration. 2) Parent's min
+ * resource must be more than sum of child's min resource.
+ */
+
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // 1. Explicitly set percentage based config for parent queues. This will
+ // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
+ // resource.
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+
+ // Also unset resource based config.
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ try {
+ cs.reinitialize(csConf, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Parent queue 'queueA' "
+ + "and child queue 'queueA1' should use either percentage based"
+ + " capacity configuration or absolute resource together.",
+ e.getMessage());
+ }
+
+ // 2. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf1 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf1);
+
+ // Configure QueueA with lesser resource than its children.
+ csConf1.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A1_MINRES);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals("Failed to re-init queues : Parent Queues capacity: "
+ + "<memory:51200, vCores:5> is less than to its children:"
+ + "<memory:102400, vCores:10> for queue:queueA", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEffectiveResourceAfterReducingClusterResource()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+ rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // unregister one NM.
+ rm.unRegisterNode(nm1);
+
+ // After loosing one NM, effective min res of queueA will become just
+ // above half. Hence A's min will be 60Gi and 6 cores and max will be
+ // 128GB and 20 cores.
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 8aca235..24ae244 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -86,6 +87,7 @@ public class TestApplicationLimits {
final static int GB = 1024;
LeafQueue queue;
+ CSQueue root;
private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -100,7 +102,7 @@ public class TestApplicationLimits {
setupQueueConfiguration(csConf);
rmContext = TestUtils.getMockRMContext();
-
+ Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32);
CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
when(csContext.getConfiguration()).thenReturn(csConf);
@@ -110,10 +112,11 @@ public class TestApplicationLimits {
when(csContext.getMaximumResourceCapability()).
thenReturn(Resources.createResource(16*GB, 32));
when(csContext.getClusterResource()).
- thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
+ thenReturn(clusterResource);
when(csContext.getResourceCalculator()).
thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
RMContainerTokenSecretManager containerTokenSecretManager =
new RMContainerTokenSecretManager(conf);
@@ -122,13 +125,17 @@ public class TestApplicationLimits {
containerTokenSecretManager);
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
- CSQueue root = CapacitySchedulerQueueManager
+ root = CapacitySchedulerQueueManager
.parseQueue(csContext, csConf, null, "root",
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
-
queue = spy(new LeafQueue(csContext, A, root, null));
+ QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A))
+ .getQueueResourceQuotas();
+ doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas();
// Stub out ACL checks
doReturn(true).
@@ -189,6 +196,8 @@ public class TestApplicationLimits {
// when there is only 1 user, and drops to 2G (the userlimit) when there
// is a second user
Resource clusterResource = Resource.newInstance(80 * GB, 40);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue.updateClusterResource(clusterResource, new ResourceLimits(
clusterResource));
@@ -287,6 +296,8 @@ public class TestApplicationLimits {
CSQueue root =
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
"root", queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
LeafQueue queue = (LeafQueue)queues.get(A);
@@ -357,6 +368,8 @@ public class TestApplicationLimits {
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
clusterResource = Resources.createResource(100 * 16 * GB);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
@@ -378,6 +391,8 @@ public class TestApplicationLimits {
root = CapacitySchedulerQueueManager.parseQueue(
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
@@ -393,7 +408,7 @@ public class TestApplicationLimits {
final String user_0 = "user_0";
final String user_1 = "user_1";
final String user_2 = "user_2";
-
+
assertEquals(Resource.newInstance(16 * GB, 1),
queue.calculateAndGetAMResourceLimit());
assertEquals(Resource.newInstance(8 * GB, 1),
@@ -578,6 +593,7 @@ public class TestApplicationLimits {
thenReturn(Resources.createResource(16*GB));
when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
// Say cluster has 100 nodes of 16G each
Resource clusterResource = Resources.createResource(100 * 16 * GB);
@@ -586,6 +602,8 @@ public class TestApplicationLimits {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -693,6 +711,8 @@ public class TestApplicationLimits {
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.createResource(90*16*GB);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Any change is cluster resource needs to enforce user-limit recomputation.
// In existing code, LeafQueue#updateClusterResource handled this. However
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index 0aac2ef..d73f1c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -600,6 +601,7 @@ public class TestApplicationLimitsByPartition {
RMContext spyRMContext = spy(rmContext);
when(spyRMContext.getNodeLabelManager()).thenReturn(mgr);
when(csContext.getRMContext()).thenReturn(spyRMContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
mgr.activateNode(NodeId.newInstance("h0", 0),
Resource.newInstance(160 * GB, 16)); // default Label
@@ -615,6 +617,8 @@ public class TestApplicationLimitsByPartition {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 64e0df4..cc9a3d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4268,7 +4268,7 @@ public class TestCapacityScheduler {
null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
- assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
+ assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
cs.getQueue("p2").getUsedResources().getMemorySize());
//Free a container from X1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95a81934/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index e34665d..b6b0361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -242,6 +242,8 @@ public class TestChildQueueOrder {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org