You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2017/12/12 23:58:46 UTC
[11/50] hadoop git commit: YARN-7575. NPE in scheduler UI when
max-capacity is not configured. Contributed by Sunil G.
YARN-7575. NPE in scheduler UI when max-capacity is not configured. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4db4a4a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4db4a4a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4db4a4a1
Branch: refs/heads/HDFS-7240
Commit: 4db4a4a165d3936d59f5e67947c6bfbc9c3270a5
Parents: daa1cdd
Author: Sunil G <su...@apache.org>
Authored: Wed Nov 29 14:50:54 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800
----------------------------------------------------------------------
.../webapp/CapacitySchedulerPage.java | 9 +-
.../TestAbsoluteResourceConfiguration.java | 116 +++++++++++++++++++
2 files changed, 121 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db4a4a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 3f68e07..4933d34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -161,10 +161,11 @@ class CapacitySchedulerPage extends RmView {
.__("Configured Capacity:",
capacities.getConfiguredMinResource().toString())
.__("Configured Max Capacity:",
- capacities.getConfiguredMaxResource().getResource()
- .equals(Resources.none())
- ? "unlimited"
- : capacities.getConfiguredMaxResource().toString())
+ (capacities.getConfiguredMaxResource() == null
+ || capacities.getConfiguredMaxResource().getResource()
+ .equals(Resources.none()))
+ ? "unlimited"
+ : capacities.getConfiguredMaxResource().toString())
.__("Effective Capacity:",
appendPercent(capacities.getEffectiveMinResource().toString(),
capacities.getCapacity() / 100))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4db4a4a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 80d16bb..5c4a12e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -511,4 +511,120 @@ public class TestAbsoluteResourceConfiguration {
rm.stop();
}
+
+ @Test
+ public void testEffectiveResourceAfterIncreasinClusterResource()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+ rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ ParentQueue qA = (ParentQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ ParentQueue qB = (ParentQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+ Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+ QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+ QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+ Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+ QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+ QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+ Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+ // add new NM.
+ rm.registerNode("127.0.0.3:1234", 125 * GB, 20);
+
+ // There will be no change in effective resource when nodes are added.
+ // Since configured capacity was based on initial node capacity, a
+ // re configurations is needed to use this added capacity.
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+ QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+ QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+ QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+ QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org