You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/06/04 19:23:09 UTC
hadoop git commit: YARN-3733. Fix DominantRC#compare() does not work
as expected if cluster resource is empty. (Rohith Sharmaks via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk dbed757cb -> ebd797c48
YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty. (Rohith Sharmaks via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebd797c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebd797c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebd797c4
Branch: refs/heads/trunk
Commit: ebd797c48fe236b404cf3a125ac9d1f7714e291e
Parents: dbed757
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jun 4 10:22:57 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Jun 4 10:22:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../resource/DominantResourceCalculator.java | 15 +++++
.../capacity/TestCapacityScheduler.java | 58 +++++++++++++++++++-
3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebd797c4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4400365..c90674e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -623,6 +623,9 @@ Release 2.7.1 - UNRELEASED
YARN-3585. NodeManager cannot exit on SHUTDOWN event triggered and NM
recovery is enabled (Rohith Sharmaks via jlowe)
+ YARN-3733. Fix DominantRC#compare() does not work as expected if
+ cluster resource is empty. (Rohith Sharmaks via wangda)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebd797c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6f5b40e..2ee95ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -53,6 +53,21 @@ public class DominantResourceCalculator extends ResourceCalculator {
return 0;
}
+ if (isInvalidDivisor(clusterResource)) {
+ if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs
+ .getVirtualCores())
+ || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs
+ .getVirtualCores())) {
+ return 0;
+ } else if (lhs.getMemory() > rhs.getMemory()
+ || lhs.getVirtualCores() > rhs.getVirtualCores()) {
+ return 1;
+ } else if (lhs.getMemory() < rhs.getMemory()
+ || lhs.getVirtualCores() < rhs.getVirtualCores()) {
+ return -1;
+ }
+ }
+
float l = getResourceAsValue(clusterResource, lhs, true);
float r = getResourceAsValue(clusterResource, rhs, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebd797c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 0361424..3827f85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -130,6 +130,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@@ -1281,9 +1282,15 @@ public class TestCapacityScheduler {
private MockRM setUpMove() {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ return setUpMove(conf);
+ }
+
+ private MockRM setUpMove(Configuration config) {
+ CapacitySchedulerConfiguration conf =
+ new CapacitySchedulerConfiguration(config);
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
+ ResourceScheduler.class);
MockRM rm = new MockRM(conf);
rm.start();
return rm;
@@ -2952,6 +2959,55 @@ public class TestCapacityScheduler {
Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
+ @Test(timeout = 30000)
+ public void testAMLimitUsage() throws Exception {
+
+ CapacitySchedulerConfiguration config =
+ new CapacitySchedulerConfiguration();
+
+ config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DefaultResourceCalculator.class.getName());
+ verifyAMLimitForLeafQueue(config);
+
+ config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class.getName());
+ verifyAMLimitForLeafQueue(config);
+
+ }
+
+ private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config)
+ throws Exception {
+ MockRM rm = setUpMove(config);
+
+ String queueName = "a1";
+ String userName = "user_0";
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ LeafQueue queueA =
+ (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
+ Resource amResourceLimit = queueA.getAMResourceLimit();
+
+ Resource amResource =
+ Resource.newInstance(amResourceLimit.getMemory() + 1,
+ amResourceLimit.getVirtualCores() + 1);
+
+ rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+
+ rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+
+ // When AM limit is exceeded, 1 applications will be activated.Rest all
+ // applications will be in pending
+ Assert.assertEquals("PendingApplications should be 1", 1,
+ queueA.getNumPendingApplications());
+ Assert.assertEquals("Active applications should be 1", 1,
+ queueA.getNumActiveApplications());
+
+ Assert.assertEquals("User PendingApplications should be 1", 1, queueA
+ .getUser(userName).getPendingApplications());
+ Assert.assertEquals("User Active applications should be 1", 1, queueA
+ .getUser(userName).getActiveApplications());
+ rm.stop();
+ }
+
private void setMaxAllocMb(Configuration conf, int maxAllocMb) {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
maxAllocMb);