You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/07/12 04:53:52 UTC
[36/50] [abbrv] hadoop git commit: YARN-6428. Queue AM limit is not
honored in CS always. Contributed by Bibin A Chundatt.
YARN-6428. Queue AM limit is not honored in CS always. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/626d730b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/626d730b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/626d730b
Branch: refs/heads/YARN-5972
Commit: 626d730bfc4c1f6ea54e8a246a0a676cdb74937f
Parents: 9d27880
Author: Naganarasimha <na...@apache.org>
Authored: Sun Jul 9 17:26:09 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sun Jul 9 17:26:09 2017 +0530
----------------------------------------------------------------------
.../resource/DominantResourceCalculator.java | 22 ++++-------
.../capacity/TestCapacityScheduler.java | 40 ++++++++++++++++++++
2 files changed, 47 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/626d730b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index ea9b927..7697e1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -218,27 +218,19 @@ public class DominantResourceCalculator extends ResourceCalculator {
public Resource multiplyAndNormalizeUp(Resource r, double by,
Resource stepFactor) {
return Resources.createResource(
- roundUp(
- (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()),
- roundUp(
- (int)Math.ceil(r.getVirtualCores() * by),
- stepFactor.getVirtualCores())
- );
+ roundUp((long) Math.ceil((float) (r.getMemorySize() * by)),
+ stepFactor.getMemorySize()),
+ roundUp((int) Math.ceil((float) (r.getVirtualCores() * by)),
+ stepFactor.getVirtualCores()));
}
@Override
public Resource multiplyAndNormalizeDown(Resource r, double by,
Resource stepFactor) {
return Resources.createResource(
- roundDown(
- (int)(r.getMemorySize() * by),
- stepFactor.getMemorySize()
- ),
- roundDown(
- (int)(r.getVirtualCores() * by),
- stepFactor.getVirtualCores()
- )
- );
+ roundDown((long) (r.getMemorySize() * by), stepFactor.getMemorySize()),
+ roundDown((int) (r.getVirtualCores() * by),
+ stepFactor.getVirtualCores()));
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/626d730b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 3c6e6df..ba0f906 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4417,4 +4417,44 @@ public class TestCapacityScheduler {
Assert.assertEquals(b1.getState(), QueueState.RUNNING);
Assert.assertTrue(!b1.getChildQueues().isEmpty());
}
+
+ @Test(timeout = 30000)
+ public void testAMLimitDouble() throws Exception {
+ CapacitySchedulerConfiguration config =
+ new CapacitySchedulerConfiguration();
+ config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class.getName());
+ CapacitySchedulerConfiguration conf =
+ new CapacitySchedulerConfiguration(config);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setInt("yarn.scheduler.minimum-allocation-mb", 512);
+ conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1);
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ rm.registerNode("127.0.0.1:1234", 10 * GB);
+ rm.registerNode("127.0.0.1:1235", 10 * GB);
+ rm.registerNode("127.0.0.1:1236", 10 * GB);
+ rm.registerNode("127.0.0.1:1237", 10 * GB);
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ waitforNMRegistered(scheduler, 4, 5);
+ LeafQueue queueA =
+ (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default");
+ Resource amResourceLimit = queueA.getAMResourceLimit();
+ Assert.assertEquals(4096, amResourceLimit.getMemorySize());
+ Assert.assertEquals(4, amResourceLimit.getVirtualCores());
+ rm.stop();
+ }
+
+ private void waitforNMRegistered(ResourceScheduler scheduler, int nodecount,
+ int timesec) throws InterruptedException {
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < timesec * 1000) {
+ if (scheduler.getNumClusterNodes() < nodecount) {
+ Thread.sleep(100);
+ } else {
+ break;
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org