You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/12/09 00:35:38 UTC
[hadoop] 01/02: Revert "YARN-10012. Guaranteed and max capacity
queue metrics for custom resources"
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit cd4779f3170231bfbef1ff7575b015940f671854
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Sun Dec 8 16:34:46 2019 -0800
Revert "YARN-10012. Guaranteed and max capacity queue metrics for custom resources"
This reverts commit 92bce918dc5d03560169642b71636800680a3292.
---
.../CSQueueMetricsForCustomResources.java | 50 ------
.../resourcemanager/scheduler/QueueMetrics.java | 52 +++---
.../scheduler/capacity/CSQueueMetrics.java | 53 ------
.../yarn/server/resourcemanager/MockNodes.java | 7 -
.../TestQueueMetricsForCustomResources.java | 4 +-
.../scheduler/capacity/TestCapacityScheduler.java | 180 +++------------------
6 files changed, 39 insertions(+), 307 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
deleted file mode 100644
index 77fe42b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-
-import java.util.Map;
-
-/**
- * This class is a main entry-point for any kind of CSQueueMetrics for
- * custom resources.
- * It provides increase and decrease methods for all types of metrics.
- */
-public class CSQueueMetricsForCustomResources
- extends QueueMetricsForCustomResources {
- private final QueueMetricsCustomResource guaranteedCapacity =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource maxCapacity =
- new QueueMetricsCustomResource();
-
- public void setGuaranteedCapacity(Resource res) {
- guaranteedCapacity.set(res);
- }
-
- public void setMaxCapacity(Resource res) {
- maxCapacity.set(res);
- }
-
- public Map<String, Long> getGuaranteedCapacity() {
- return guaranteedCapacity.getValues();
- }
-
- public Map<String, Long> getMaxCapacity() {
- return maxCapacity.getValues();
- }
-}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index 6a428b5..1b926a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -159,7 +159,11 @@ public class QueueMetrics implements MetricsSource {
this.conf = conf;
runningTime = buildBuckets(conf);
- createQueueMetricsForCustomResources();
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ this.queueMetricsForCustomResources =
+ new QueueMetricsForCustomResources();
+ registerCustomResources();
+ }
}
protected QueueMetrics tag(MetricsInfo info, String value) {
@@ -445,33 +449,25 @@ public class QueueMetrics implements MetricsSource {
}
}
- protected Map<String, Long> initAndGetCustomResources() {
- Map<String, Long> customResources = new HashMap<String, Long>();
- ResourceInformation[] resources = ResourceUtils.getResourceTypesArray();
-
- for (int i = 2; i < resources.length; i++) {
- ResourceInformation resource = resources[i];
- customResources.put(resource.getName(), Long.valueOf(0));
- }
- return customResources;
- }
-
- protected void createQueueMetricsForCustomResources() {
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- this.queueMetricsForCustomResources =
- new QueueMetricsForCustomResources();
- registerCustomResources();
- }
- }
-
/**
* Register all custom resources metrics as part of initialization. As and
* when this metric object construction happens for any queue, all custom
* resource metrics value would be initialized with '0' like any other
* mandatory resources metrics
*/
- protected void registerCustomResources() {
- Map<String, Long> customResources = initAndGetCustomResources();
+ private void registerCustomResources() {
+ Map<String, Long> customResources =
+ new HashMap<String, Long>();
+ ResourceInformation[] resources =
+ ResourceUtils.getResourceTypesArray();
+
+ for (int i =
+ 2; i < resources.length; i++) {
+ ResourceInformation resource =
+ resources[i];
+ customResources.put(resource.getName(), Long.valueOf(0));
+ }
+
registerCustomResources(customResources, ALLOCATED_RESOURCE_METRIC_PREFIX,
ALLOCATED_RESOURCE_METRIC_DESC);
registerCustomResources(customResources, AVAILABLE_RESOURCE_METRIC_PREFIX,
@@ -485,7 +481,7 @@ public class QueueMetrics implements MetricsSource {
AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC);
}
- protected void registerCustomResources(Map<String, Long> customResources,
+ private void registerCustomResources(Map<String, Long> customResources,
String metricPrefix, String metricDesc) {
for (Entry<String, Long> entry : customResources.entrySet()) {
String resourceName = entry.getKey();
@@ -1011,14 +1007,4 @@ public class QueueMetrics implements MetricsSource {
}
}
}
-
- @VisibleForTesting
- public QueueMetricsForCustomResources getQueueMetricsForCustomResources() {
- return this.queueMetricsForCustomResources;
- }
-
- public void setQueueMetricsForCustomResources(
- QueueMetricsForCustomResources metrics) {
- this.queueMetricsForCustomResources = metrics;
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index e9a0aaf..c50a1b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
@@ -30,10 +28,8 @@ import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@Metrics(context = "yarn")
public class CSQueueMetrics extends QueueMetrics {
@@ -68,36 +64,11 @@ public class CSQueueMetrics extends QueueMetrics {
@Metric("Maximum capacity in percentage relative to total partition")
private MutableGaugeFloat maxAbsoluteCapacity;
- private static final String GUARANTEED_CAPACITY_METRIC_PREFIX =
- "GuaranteedCapacity.";
- private static final String GUARANTEED_CAPACITY_METRIC_DESC =
- "GuaranteedCapacity of NAME";
-
- private static final String MAX_CAPACITY_METRIC_PREFIX =
- "MaxCapacity.";
- private static final String MAX_CAPACITY_METRIC_DESC =
- "MaxCapacity of NAME";
-
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
super(ms, queueName, parent, enableUserMetrics, conf);
}
- /**
- * Register all custom resources metrics as part of initialization. As and
- * when this metric object construction happens for any queue, all custom
- * resource metrics value would be initialized with '0' like any other
- * mandatory resources metrics
- */
- protected void registerCustomResources() {
- Map<String, Long> customResources = initAndGetCustomResources();
- registerCustomResources(customResources, GUARANTEED_CAPACITY_METRIC_PREFIX,
- GUARANTEED_CAPACITY_METRIC_DESC);
- registerCustomResources(customResources, MAX_CAPACITY_METRIC_PREFIX,
- MAX_CAPACITY_METRIC_DESC);
- super.registerCustomResources();
- }
-
public long getAMResourceLimitMB() {
return AMResourceLimitMB.value();
}
@@ -184,14 +155,6 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
guaranteedMB.set(res.getMemorySize());
guaranteedVCores.set(res.getVirtualCores());
- if (getQueueMetricsForCustomResources() != null) {
- ((CSQueueMetricsForCustomResources) getQueueMetricsForCustomResources())
- .setGuaranteedCapacity(res);
- registerCustomResources(
- ((CSQueueMetricsForCustomResources)
- getQueueMetricsForCustomResources()).getGuaranteedCapacity(),
- GUARANTEED_CAPACITY_METRIC_PREFIX, GUARANTEED_CAPACITY_METRIC_DESC);
- }
}
}
@@ -207,22 +170,6 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
maxCapacityMB.set(res.getMemorySize());
maxCapacityVCores.set(res.getVirtualCores());
- if (getQueueMetricsForCustomResources() != null) {
- ((CSQueueMetricsForCustomResources) getQueueMetricsForCustomResources())
- .setMaxCapacity(res);
- registerCustomResources(
- ((CSQueueMetricsForCustomResources)
- getQueueMetricsForCustomResources()).getMaxCapacity(),
- MAX_CAPACITY_METRIC_PREFIX, MAX_CAPACITY_METRIC_DESC);
- }
- }
- }
-
- @Override
- protected void createQueueMetricsForCustomResources() {
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- setQueueMetricsForCustomResources(new CSQueueMetricsForCustomResources());
- registerCustomResources();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 600edfc..3b72ca1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -93,12 +92,6 @@ public class MockNodes {
return rs;
}
- public static Resource newResource(long memory, int vCores,
- Map<String, String> customResources) {
- return ResourceTypesTestHelper.newResource(memory, vCores, customResources);
- }
-
-
public static Resource newUsedResource(Resource total) {
Resource rs = recordFactory.newRecordInstance(Resource.class);
rs.setMemorySize((int)(Math.random() * total.getMemorySize()));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
index 78f2a43..a18f2d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
@@ -93,8 +93,8 @@ public class TestQueueMetricsForCustomResources {
public static final long GB = 1024; // MB
private static final Configuration CONF = new Configuration();
- public static final String CUSTOM_RES_1 = "custom_res_1";
- public static final String CUSTOM_RES_2 = "custom_res_2";
+ private static final String CUSTOM_RES_1 = "custom_res_1";
+ private static final String CUSTOM_RES_2 = "custom_res_2";
public static final String USER = "alice";
private Resource defaultResource;
private MetricsSystem ms;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index f71a273..84dffa3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -93,7 +93,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -122,7 +121,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
+
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -141,7 +140,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -149,7 +147,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState;
@@ -206,7 +203,6 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
@Before
public void setUp() throws Exception {
- ResourceUtils.resetResourceTypes(new Configuration());
resourceManager = new ResourceManager() {
@Override
protected RMNodeLabelsManager createNodeLabelManager() {
@@ -5723,109 +5719,23 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
@Test
public void testCSQueueMetrics() throws Exception {
+ CapacityScheduler cs = new CapacityScheduler();
+ cs.setConf(new YarnConfiguration());
+ cs.setRMContext(resourceManager.getRMContext());
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ cs.init(conf);
+ cs.start();
- // Initialize resource map
- Map<String, ResourceInformation> riMap = new HashMap<>();
-
- // Initialize mandatory resources
- ResourceInformation memory =
- ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
- ResourceInformation.MEMORY_MB.getUnits(),
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
- ResourceInformation vcores =
- ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
- ResourceInformation.VCORES.getUnits(),
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
- riMap.put(ResourceInformation.MEMORY_URI, memory);
- riMap.put(ResourceInformation.VCORES_URI, vcores);
- riMap.put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
- ResourceInformation.newInstance(
- TestQueueMetricsForCustomResources.CUSTOM_RES_1, "", 1, 10));
-
- ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
-
- CapacitySchedulerConfiguration csConf =
- new CapacitySchedulerConfiguration();
- csConf.setResourceComparator(DominantResourceCalculator.class);
-
- csConf.set(YarnConfiguration.RESOURCE_TYPES,
- TestQueueMetricsForCustomResources.CUSTOM_RES_1);
-
- setupQueueConfiguration(csConf);
-
- YarnConfiguration conf = new YarnConfiguration(csConf);
-
- // Don't reset resource types since we have already configured resource
- // types
- conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
-
- MockRM rm = new MockRM(conf);
- rm.start();
-
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
-
- RMNode n1 = MockNodes.newNodeInfo(0,
- MockNodes.newResource(50 * GB, 50,
- ImmutableMap.<String, String> builder()
- .put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
- String.valueOf(1000))
- .build()),
- 1, "n1");
- RMNode n2 = MockNodes.newNodeInfo(0,
- MockNodes.newResource(50 * GB, 50,
- ImmutableMap.<String, String> builder()
- .put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
- String.valueOf(2000))
- .build()),
- 2, "n2");
+ RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 1, "n1");
+ RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 2, "n2");
cs.handle(new NodeAddedSchedulerEvent(n1));
cs.handle(new NodeAddedSchedulerEvent(n2));
- Map<String, Long> guaranteedCapA11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(94, guaranteedCapA11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(3000, maxCapA11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
-
assertEquals(10240, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB());
assertEquals(71680, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(102400, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(102400, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
- Map<String, Long> guaranteedCapA =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(314, guaranteedCapA
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(3000, maxCapA
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> guaranteedCapB1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(2126, guaranteedCapB1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapB1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(3000, maxCapB1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
// Remove a node, metrics should be updated
cs.handle(new NodeRemovedSchedulerEvent(n2));
@@ -5833,31 +5743,6 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
- Map<String, Long> guaranteedCapA1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
-
- assertEquals(104, guaranteedCapA1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(1000, maxCapA1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> guaranteedCapB11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(708, guaranteedCapB11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapB11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(1000, maxCapB11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
.getMetrics()).getGuaranteedCapacity(), DELTA);
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
@@ -5876,49 +5761,20 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
.getMaxAbsoluteCapacity(), DELTA);
// Add child queue to a, and reinitialize. Metrics should be updated
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a",
- new String[] {"a1", "a2", "a3"});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 29.5f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.5f);
- csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3",
- 50.0f);
-
- cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null,
- null, new RMContainerTokenSecretManager(csConf),
- new NMTokenSecretManagerInRM(csConf),
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 30.0f);
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.0f);
+ conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 50.0f);
+
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), null));
assertEquals(1024, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getGuaranteedMB());
assertEquals(2048, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getMaxCapacityMB());
assertEquals(25600, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getMaxCapacityMB());
-
- Map<String, Long> guaranteedCapA2 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a2")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(30, guaranteedCapA2
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA2 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a2")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(1000, maxCapA2
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
-
- Map<String, Long> guaranteedCapA3 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a3")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(42, guaranteedCapA3
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA3 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a3")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(500, maxCapA3
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- rm.stop();
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org