You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2019/08/26 15:39:17 UTC
[hadoop] branch trunk updated: YARN-8917. Absolute (maximum)
capacity of level3+ queues is wrongly calculated for absolute resource.
Contributed by Tao Yang.
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 689d2e6 YARN-8917. Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource. Contributed by Tao Yang.
689d2e6 is described below
commit 689d2e61058b5f719c6cbe9897a72b19b44a29a3
Author: Rohith Sharma K S <ro...@apache.org>
AuthorDate: Mon Aug 26 21:06:15 2019 +0530
YARN-8917. Absolute (maximum) capacity of level3+ queues is wrongly calculated for absolute resource. Contributed by Tao Yang.
---
.../scheduler/capacity/ParentQueue.java | 8 ++--
.../TestAbsoluteResourceConfiguration.java | 49 ++++++++++++++++++++++
2 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index c56369c..fb747fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1102,14 +1102,14 @@ public class ParentQueue extends AbstractCSQueue {
// 3. Update absolute capacity as a float based on parent's minResource and
// cluster resource.
childQueue.getQueueCapacities().setAbsoluteCapacity(label,
- (float) childQueue.getQueueCapacities().getCapacity()
- / getQueueCapacities().getAbsoluteCapacity(label));
+ childQueue.getQueueCapacities().getCapacity(label)
+ * getQueueCapacities().getAbsoluteCapacity(label));
// 4. Update absolute max-capacity as a float based on parent's maxResource
// and cluster resource.
childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
- (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
- / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+ childQueue.getQueueCapacities().getMaximumCapacity(label)
+ * getQueueCapacities().getAbsoluteMaximumCapacity(label));
// Re-visit max applications for a queue based on absolute capacity if
// needed.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 298e1ab..fe4c880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -33,6 +33,7 @@ import org.junit.Test;
public class TestAbsoluteResourceConfiguration {
private static final int GB = 1024;
+ private static final float DELTA = 0.001f;
private static final String QUEUEA = "queueA";
private static final String QUEUEB = "queueB";
@@ -544,6 +545,10 @@ public class TestAbsoluteResourceConfiguration {
QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEA is not correct",
+ 0.4, qA.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct",
+ 0.8, qA.getAbsoluteMaximumCapacity(), DELTA);
ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB);
Assert.assertNotNull(qB);
@@ -555,6 +560,10 @@ public class TestAbsoluteResourceConfiguration {
QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEB is not correct",
+ 0.2, qB.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct",
+ 0.6, qB.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
Assert.assertNotNull(qC);
@@ -566,18 +575,30 @@ public class TestAbsoluteResourceConfiguration {
QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEC is not correct",
+ 0.2, qC.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct",
+ 0.6, qC.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct",
+ 0.2, qA1.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct",
+ 0.8, qA1.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct",
+ 0.2, qA2.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct",
+ 0.8, qA2.getAbsoluteMaximumCapacity(), DELTA);
LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
@@ -588,6 +609,10 @@ public class TestAbsoluteResourceConfiguration {
QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct",
+ 0.16, qB1.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct",
+ 0.6, qB1.getAbsoluteMaximumCapacity(), DELTA);
// add new NM.
rm.registerNode("127.0.0.3:1234", 125 * GB, 20);
@@ -599,31 +624,55 @@ public class TestAbsoluteResourceConfiguration {
QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEA is not correct",
+ 0.266, qA.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEA is not correct",
+ 0.533, qA.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEB is not correct",
+ 0.133, qB.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEB is not correct",
+ 0.4, qB.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEC is not correct",
+ 0.133, qC.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEC is not correct",
+ 0.4, qC.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEB1 is not correct",
+ 0.106, qB1.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEB1 is not correct",
+ 0.4, qB1.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEA1 is not correct",
+ 0.133, qA1.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEA1 is not correct",
+ 0.533, qA1.getAbsoluteMaximumCapacity(), DELTA);
Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+ Assert.assertEquals("Absolute capacity for QUEUEA2 is not correct",
+ 0.133, qA2.getAbsoluteCapacity(), DELTA);
+ Assert.assertEquals("Absolute Max capacity for QUEUEA2 is not correct",
+ 0.533, qA2.getAbsoluteMaximumCapacity(), DELTA);
rm.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org