You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/12/09 00:36:30 UTC
[hadoop] branch branch-3.2 updated (b5235f1 -> 9228e3f)
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a change to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from b5235f1 YARN-10012. Guaranteed and max capacity queue metrics for custom resources
new 8badcd9 Revert "YARN-10012. Guaranteed and max capacity queue metrics for custom resources"
new 9228e3f YARN-10012. Guaranteed and max capacity queue metrics for custom resources. Contributed by Manikandan R
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: Revert "YARN-10012. Guaranteed and max capacity
queue metrics for custom resources"
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8badcd989e4749c05ac35e8295fd1ab94f6dff08
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Sun Dec 8 16:36:00 2019 -0800
Revert "YARN-10012. Guaranteed and max capacity queue metrics for custom resources"
This reverts commit b5235f1ed05f6506aa2c40825b2b647475a4d91a.
---
.../CSQueueMetricsForCustomResources.java | 50 ------
.../resourcemanager/scheduler/QueueMetrics.java | 52 +++---
.../scheduler/capacity/CSQueueMetrics.java | 53 ------
.../yarn/server/resourcemanager/MockNodes.java | 7 -
.../TestQueueMetricsForCustomResources.java | 4 +-
.../scheduler/capacity/TestCapacityScheduler.java | 180 +++------------------
6 files changed, 39 insertions(+), 307 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
deleted file mode 100644
index 77fe42b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-
-import org.apache.hadoop.yarn.api.records.Resource;
-
-import java.util.Map;
-
-/**
- * This class is a main entry-point for any kind of CSQueueMetrics for
- * custom resources.
- * It provides increase and decrease methods for all types of metrics.
- */
-public class CSQueueMetricsForCustomResources
- extends QueueMetricsForCustomResources {
- private final QueueMetricsCustomResource guaranteedCapacity =
- new QueueMetricsCustomResource();
- private final QueueMetricsCustomResource maxCapacity =
- new QueueMetricsCustomResource();
-
- public void setGuaranteedCapacity(Resource res) {
- guaranteedCapacity.set(res);
- }
-
- public void setMaxCapacity(Resource res) {
- maxCapacity.set(res);
- }
-
- public Map<String, Long> getGuaranteedCapacity() {
- return guaranteedCapacity.getValues();
- }
-
- public Map<String, Long> getMaxCapacity() {
- return maxCapacity.getValues();
- }
-}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index b05a0ae..094444f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -161,7 +161,11 @@ public class QueueMetrics implements MetricsSource {
this.conf = conf;
runningTime = buildBuckets(conf);
- createQueueMetricsForCustomResources();
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ this.queueMetricsForCustomResources =
+ new QueueMetricsForCustomResources();
+ registerCustomResources();
+ }
}
protected QueueMetrics tag(MetricsInfo info, String value) {
@@ -447,33 +451,25 @@ public class QueueMetrics implements MetricsSource {
}
}
- protected Map<String, Long> initAndGetCustomResources() {
- Map<String, Long> customResources = new HashMap<String, Long>();
- ResourceInformation[] resources = ResourceUtils.getResourceTypesArray();
-
- for (int i = 2; i < resources.length; i++) {
- ResourceInformation resource = resources[i];
- customResources.put(resource.getName(), Long.valueOf(0));
- }
- return customResources;
- }
-
- protected void createQueueMetricsForCustomResources() {
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- this.queueMetricsForCustomResources =
- new QueueMetricsForCustomResources();
- registerCustomResources();
- }
- }
-
/**
* Register all custom resources metrics as part of initialization. As and
* when this metric object construction happens for any queue, all custom
* resource metrics value would be initialized with '0' like any other
* mandatory resources metrics
*/
- protected void registerCustomResources() {
- Map<String, Long> customResources = initAndGetCustomResources();
+ private void registerCustomResources() {
+ Map<String, Long> customResources =
+ new HashMap<String, Long>();
+ ResourceInformation[] resources =
+ ResourceUtils.getResourceTypesArray();
+
+ for (int i =
+ 2; i < resources.length; i++) {
+ ResourceInformation resource =
+ resources[i];
+ customResources.put(resource.getName(), Long.valueOf(0));
+ }
+
registerCustomResources(customResources, ALLOCATED_RESOURCE_METRIC_PREFIX,
ALLOCATED_RESOURCE_METRIC_DESC);
registerCustomResources(customResources, AVAILABLE_RESOURCE_METRIC_PREFIX,
@@ -487,7 +483,7 @@ public class QueueMetrics implements MetricsSource {
AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC);
}
- protected void registerCustomResources(Map<String, Long> customResources,
+ private void registerCustomResources(Map<String, Long> customResources,
String metricPrefix, String metricDesc) {
for (Entry<String, Long> entry : customResources.entrySet()) {
String resourceName = entry.getKey();
@@ -970,14 +966,4 @@ public class QueueMetrics implements MetricsSource {
public long getAggregateVcoresPreempted() {
return aggregateVcoresPreempted.value();
}
-
- @VisibleForTesting
- public QueueMetricsForCustomResources getQueueMetricsForCustomResources() {
- return this.queueMetricsForCustomResources;
- }
-
- public void setQueueMetricsForCustomResources(
- QueueMetricsForCustomResources metrics) {
- this.queueMetricsForCustomResources = metrics;
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index e9a0aaf..c50a1b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
@@ -30,10 +28,8 @@ import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@Metrics(context = "yarn")
public class CSQueueMetrics extends QueueMetrics {
@@ -68,36 +64,11 @@ public class CSQueueMetrics extends QueueMetrics {
@Metric("Maximum capacity in percentage relative to total partition")
private MutableGaugeFloat maxAbsoluteCapacity;
- private static final String GUARANTEED_CAPACITY_METRIC_PREFIX =
- "GuaranteedCapacity.";
- private static final String GUARANTEED_CAPACITY_METRIC_DESC =
- "GuaranteedCapacity of NAME";
-
- private static final String MAX_CAPACITY_METRIC_PREFIX =
- "MaxCapacity.";
- private static final String MAX_CAPACITY_METRIC_DESC =
- "MaxCapacity of NAME";
-
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
super(ms, queueName, parent, enableUserMetrics, conf);
}
- /**
- * Register all custom resources metrics as part of initialization. As and
- * when this metric object construction happens for any queue, all custom
- * resource metrics value would be initialized with '0' like any other
- * mandatory resources metrics
- */
- protected void registerCustomResources() {
- Map<String, Long> customResources = initAndGetCustomResources();
- registerCustomResources(customResources, GUARANTEED_CAPACITY_METRIC_PREFIX,
- GUARANTEED_CAPACITY_METRIC_DESC);
- registerCustomResources(customResources, MAX_CAPACITY_METRIC_PREFIX,
- MAX_CAPACITY_METRIC_DESC);
- super.registerCustomResources();
- }
-
public long getAMResourceLimitMB() {
return AMResourceLimitMB.value();
}
@@ -184,14 +155,6 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
guaranteedMB.set(res.getMemorySize());
guaranteedVCores.set(res.getVirtualCores());
- if (getQueueMetricsForCustomResources() != null) {
- ((CSQueueMetricsForCustomResources) getQueueMetricsForCustomResources())
- .setGuaranteedCapacity(res);
- registerCustomResources(
- ((CSQueueMetricsForCustomResources)
- getQueueMetricsForCustomResources()).getGuaranteedCapacity(),
- GUARANTEED_CAPACITY_METRIC_PREFIX, GUARANTEED_CAPACITY_METRIC_DESC);
- }
}
}
@@ -207,22 +170,6 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
maxCapacityMB.set(res.getMemorySize());
maxCapacityVCores.set(res.getVirtualCores());
- if (getQueueMetricsForCustomResources() != null) {
- ((CSQueueMetricsForCustomResources) getQueueMetricsForCustomResources())
- .setMaxCapacity(res);
- registerCustomResources(
- ((CSQueueMetricsForCustomResources)
- getQueueMetricsForCustomResources()).getMaxCapacity(),
- MAX_CAPACITY_METRIC_PREFIX, MAX_CAPACITY_METRIC_DESC);
- }
- }
- }
-
- @Override
- protected void createQueueMetricsForCustomResources() {
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- setQueueMetricsForCustomResources(new CSQueueMetricsForCustomResources());
- registerCustomResources();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index e8d352f..a871993 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -93,12 +92,6 @@ public class MockNodes {
return rs;
}
- public static Resource newResource(long memory, int vCores,
- Map<String, String> customResources) {
- return ResourceTypesTestHelper.newResource(memory, vCores, customResources);
- }
-
-
public static Resource newUsedResource(Resource total) {
Resource rs = recordFactory.newRecordInstance(Resource.class);
rs.setMemorySize((int)(Math.random() * total.getMemorySize()));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
index b6441f3..ed0ae6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
@@ -95,8 +95,8 @@ public class TestQueueMetricsForCustomResources {
public static final long GB = 1024; // MB
private static final Configuration CONF = new Configuration();
- public static final String CUSTOM_RES_1 = "custom_res_1";
- public static final String CUSTOM_RES_2 = "custom_res_2";
+ private static final String CUSTOM_RES_1 = "custom_res_1";
+ private static final String CUSTOM_RES_2 = "custom_res_2";
public static final String USER = "alice";
private Resource defaultResource;
private MetricsSystem ms;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 7ef8b9e..73f6fc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -79,7 +79,6 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
@@ -109,7 +108,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
+
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -128,7 +127,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -136,7 +134,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState;
@@ -193,7 +190,6 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
@Before
public void setUp() throws Exception {
- ResourceUtils.resetResourceTypes(new Configuration());
resourceManager = new ResourceManager() {
@Override
protected RMNodeLabelsManager createNodeLabelManager() {
@@ -5182,109 +5178,23 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
@Test
public void testCSQueueMetrics() throws Exception {
+ CapacityScheduler cs = new CapacityScheduler();
+ cs.setConf(new YarnConfiguration());
+ cs.setRMContext(resourceManager.getRMContext());
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ cs.init(conf);
+ cs.start();
- // Initialize resource map
- Map<String, ResourceInformation> riMap = new HashMap<>();
-
- // Initialize mandatory resources
- ResourceInformation memory =
- ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
- ResourceInformation.MEMORY_MB.getUnits(),
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
- ResourceInformation vcores =
- ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
- ResourceInformation.VCORES.getUnits(),
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
- riMap.put(ResourceInformation.MEMORY_URI, memory);
- riMap.put(ResourceInformation.VCORES_URI, vcores);
- riMap.put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
- ResourceInformation.newInstance(
- TestQueueMetricsForCustomResources.CUSTOM_RES_1, "", 1, 10));
-
- ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
-
- CapacitySchedulerConfiguration csConf =
- new CapacitySchedulerConfiguration();
- csConf.setResourceComparator(DominantResourceCalculator.class);
-
- csConf.set(YarnConfiguration.RESOURCE_TYPES,
- TestQueueMetricsForCustomResources.CUSTOM_RES_1);
-
- setupQueueConfiguration(csConf);
-
- YarnConfiguration conf = new YarnConfiguration(csConf);
-
- // Don't reset resource types since we have already configured resource
- // types
- conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
-
- MockRM rm = new MockRM(conf);
- rm.start();
-
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
-
- RMNode n1 = MockNodes.newNodeInfo(0,
- MockNodes.newResource(50 * GB, 50,
- ImmutableMap.<String, String> builder()
- .put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
- String.valueOf(1000))
- .build()),
- 1, "n1");
- RMNode n2 = MockNodes.newNodeInfo(0,
- MockNodes.newResource(50 * GB, 50,
- ImmutableMap.<String, String> builder()
- .put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
- String.valueOf(2000))
- .build()),
- 2, "n2");
+ RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 1, "n1");
+ RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 2, "n2");
cs.handle(new NodeAddedSchedulerEvent(n1));
cs.handle(new NodeAddedSchedulerEvent(n2));
- Map<String, Long> guaranteedCapA11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(94, guaranteedCapA11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(3000, maxCapA11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
-
assertEquals(10240, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB());
assertEquals(71680, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(102400, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(102400, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
- Map<String, Long> guaranteedCapA =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(314, guaranteedCapA
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(3000, maxCapA
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> guaranteedCapB1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(2126, guaranteedCapB1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapB1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(3000, maxCapB1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
// Remove a node, metrics should be updated
cs.handle(new NodeRemovedSchedulerEvent(n2));
@@ -5292,31 +5202,6 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
- Map<String, Long> guaranteedCapA1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
-
- assertEquals(104, guaranteedCapA1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA1 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(1000, maxCapA1
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> guaranteedCapB11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(708, guaranteedCapB11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapB11 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(1000, maxCapB11
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
.getMetrics()).getGuaranteedCapacity(), DELTA);
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
@@ -5335,49 +5220,20 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
.getMaxAbsoluteCapacity(), DELTA);
// Add child queue to a, and reinitialize. Metrics should be updated
- csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a",
- new String[] {"a1", "a2", "a3"});
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 29.5f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.5f);
- csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3",
- 50.0f);
-
- cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null,
- null, new RMContainerTokenSecretManager(csConf),
- new NMTokenSecretManagerInRM(csConf),
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 30.0f);
+ conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.0f);
+ conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 50.0f);
+
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), null));
assertEquals(1024, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getGuaranteedMB());
assertEquals(2048, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getMaxCapacityMB());
assertEquals(25600, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getMaxCapacityMB());
-
- Map<String, Long> guaranteedCapA2 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a2")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(30, guaranteedCapA2
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA2 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a2")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(1000, maxCapA2
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
-
- Map<String, Long> guaranteedCapA3 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a3")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getGuaranteedCapacity();
- assertEquals(42, guaranteedCapA3
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- Map<String, Long> maxCapA3 =
- ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a3")
- .getMetrics()).getQueueMetricsForCustomResources())
- .getMaxCapacity();
- assertEquals(500, maxCapA3
- .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
- rm.stop();
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: YARN-10012. Guaranteed and max capacity queue
metrics for custom resources. Contributed by Manikandan R
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 9228e3f0ad4e94cedd59c5b02a87ce6c267db5b5
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Sun Dec 8 15:52:17 2019 -0800
YARN-10012. Guaranteed and max capacity queue metrics for custom resources. Contributed by Manikandan R
(cherry picked from commit 92bce918dc5d03560169642b71636800680a3292)
---
.../CSQueueMetricsForCustomResources.java | 50 ++++++
.../resourcemanager/scheduler/QueueMetrics.java | 52 +++---
.../scheduler/capacity/CSQueueMetrics.java | 53 ++++++
.../yarn/server/resourcemanager/MockNodes.java | 7 +
.../TestQueueMetricsForCustomResources.java | 4 +-
.../scheduler/capacity/TestCapacityScheduler.java | 180 ++++++++++++++++++---
6 files changed, 307 insertions(+), 39 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
new file mode 100644
index 0000000..77fe42b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/CSQueueMetricsForCustomResources.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+
+import java.util.Map;
+
+/**
+ * This class is a main entry-point for any kind of CSQueueMetrics for
+ * custom resources.
+ * It provides increase and decrease methods for all types of metrics.
+ */
+public class CSQueueMetricsForCustomResources
+ extends QueueMetricsForCustomResources {
+ private final QueueMetricsCustomResource guaranteedCapacity =
+ new QueueMetricsCustomResource();
+ private final QueueMetricsCustomResource maxCapacity =
+ new QueueMetricsCustomResource();
+
+ public void setGuaranteedCapacity(Resource res) {
+ guaranteedCapacity.set(res);
+ }
+
+ public void setMaxCapacity(Resource res) {
+ maxCapacity.set(res);
+ }
+
+ public Map<String, Long> getGuaranteedCapacity() {
+ return guaranteedCapacity.getValues();
+ }
+
+ public Map<String, Long> getMaxCapacity() {
+ return maxCapacity.getValues();
+ }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
index 094444f..b05a0ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
@@ -161,11 +161,7 @@ public class QueueMetrics implements MetricsSource {
this.conf = conf;
runningTime = buildBuckets(conf);
- if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
- this.queueMetricsForCustomResources =
- new QueueMetricsForCustomResources();
- registerCustomResources();
- }
+ createQueueMetricsForCustomResources();
}
protected QueueMetrics tag(MetricsInfo info, String value) {
@@ -451,25 +447,33 @@ public class QueueMetrics implements MetricsSource {
}
}
+ protected Map<String, Long> initAndGetCustomResources() {
+ Map<String, Long> customResources = new HashMap<String, Long>();
+ ResourceInformation[] resources = ResourceUtils.getResourceTypesArray();
+
+ for (int i = 2; i < resources.length; i++) {
+ ResourceInformation resource = resources[i];
+ customResources.put(resource.getName(), Long.valueOf(0));
+ }
+ return customResources;
+ }
+
+ protected void createQueueMetricsForCustomResources() {
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ this.queueMetricsForCustomResources =
+ new QueueMetricsForCustomResources();
+ registerCustomResources();
+ }
+ }
+
/**
* Register all custom resources metrics as part of initialization. As and
* when this metric object construction happens for any queue, all custom
* resource metrics value would be initialized with '0' like any other
* mandatory resources metrics
*/
- private void registerCustomResources() {
- Map<String, Long> customResources =
- new HashMap<String, Long>();
- ResourceInformation[] resources =
- ResourceUtils.getResourceTypesArray();
-
- for (int i =
- 2; i < resources.length; i++) {
- ResourceInformation resource =
- resources[i];
- customResources.put(resource.getName(), Long.valueOf(0));
- }
-
+ protected void registerCustomResources() {
+ Map<String, Long> customResources = initAndGetCustomResources();
registerCustomResources(customResources, ALLOCATED_RESOURCE_METRIC_PREFIX,
ALLOCATED_RESOURCE_METRIC_DESC);
registerCustomResources(customResources, AVAILABLE_RESOURCE_METRIC_PREFIX,
@@ -483,7 +487,7 @@ public class QueueMetrics implements MetricsSource {
AGGREGATE_PREEMPTED_SECONDS_METRIC_DESC);
}
- private void registerCustomResources(Map<String, Long> customResources,
+ protected void registerCustomResources(Map<String, Long> customResources,
String metricPrefix, String metricDesc) {
for (Entry<String, Long> entry : customResources.entrySet()) {
String resourceName = entry.getKey();
@@ -966,4 +970,14 @@ public class QueueMetrics implements MetricsSource {
public long getAggregateVcoresPreempted() {
return aggregateVcoresPreempted.value();
}
+
+ @VisibleForTesting
+ public QueueMetricsForCustomResources getQueueMetricsForCustomResources() {
+ return this.queueMetricsForCustomResources;
+ }
+
+ public void setQueueMetricsForCustomResources(
+ QueueMetricsForCustomResources metrics) {
+ this.queueMetricsForCustomResources = metrics;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index c50a1b1..e9a0aaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metric;
@@ -28,8 +30,10 @@ import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@Metrics(context = "yarn")
public class CSQueueMetrics extends QueueMetrics {
@@ -64,11 +68,36 @@ public class CSQueueMetrics extends QueueMetrics {
@Metric("Maximum capacity in percentage relative to total partition")
private MutableGaugeFloat maxAbsoluteCapacity;
+ private static final String GUARANTEED_CAPACITY_METRIC_PREFIX =
+ "GuaranteedCapacity.";
+ private static final String GUARANTEED_CAPACITY_METRIC_DESC =
+ "GuaranteedCapacity of NAME";
+
+ private static final String MAX_CAPACITY_METRIC_PREFIX =
+ "MaxCapacity.";
+ private static final String MAX_CAPACITY_METRIC_DESC =
+ "MaxCapacity of NAME";
+
CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
boolean enableUserMetrics, Configuration conf) {
super(ms, queueName, parent, enableUserMetrics, conf);
}
+ /**
+ * Register all custom resources metrics as part of initialization. As and
+ * when this metric object construction happens for any queue, all custom
+ * resource metrics value would be initialized with '0' like any other
+ * mandatory resources metrics
+ */
+ protected void registerCustomResources() {
+ Map<String, Long> customResources = initAndGetCustomResources();
+ registerCustomResources(customResources, GUARANTEED_CAPACITY_METRIC_PREFIX,
+ GUARANTEED_CAPACITY_METRIC_DESC);
+ registerCustomResources(customResources, MAX_CAPACITY_METRIC_PREFIX,
+ MAX_CAPACITY_METRIC_DESC);
+ super.registerCustomResources();
+ }
+
public long getAMResourceLimitMB() {
return AMResourceLimitMB.value();
}
@@ -155,6 +184,14 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
guaranteedMB.set(res.getMemorySize());
guaranteedVCores.set(res.getVirtualCores());
+ if (getQueueMetricsForCustomResources() != null) {
+ ((CSQueueMetricsForCustomResources) getQueueMetricsForCustomResources())
+ .setGuaranteedCapacity(res);
+ registerCustomResources(
+ ((CSQueueMetricsForCustomResources)
+ getQueueMetricsForCustomResources()).getGuaranteedCapacity(),
+ GUARANTEED_CAPACITY_METRIC_PREFIX, GUARANTEED_CAPACITY_METRIC_DESC);
+ }
}
}
@@ -170,6 +207,22 @@ public class CSQueueMetrics extends QueueMetrics {
if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
maxCapacityMB.set(res.getMemorySize());
maxCapacityVCores.set(res.getVirtualCores());
+ if (getQueueMetricsForCustomResources() != null) {
+ ((CSQueueMetricsForCustomResources) getQueueMetricsForCustomResources())
+ .setMaxCapacity(res);
+ registerCustomResources(
+ ((CSQueueMetricsForCustomResources)
+ getQueueMetricsForCustomResources()).getMaxCapacity(),
+ MAX_CAPACITY_METRIC_PREFIX, MAX_CAPACITY_METRIC_DESC);
+ }
+ }
+ }
+
+ @Override
+ protected void createQueueMetricsForCustomResources() {
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
+ setQueueMetricsForCustomResources(new CSQueueMetricsForCustomResources());
+ registerCustomResources();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index a871993..e8d352f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -92,6 +93,12 @@ public class MockNodes {
return rs;
}
+ public static Resource newResource(long memory, int vCores,
+ Map<String, String> customResources) {
+ return ResourceTypesTestHelper.newResource(memory, vCores, customResources);
+ }
+
+
public static Resource newUsedResource(Resource total) {
Resource rs = recordFactory.newRecordInstance(Resource.class);
rs.setMemorySize((int)(Math.random() * total.getMemorySize()));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
index ed0ae6e..b6441f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetricsForCustomResources.java
@@ -95,8 +95,8 @@ public class TestQueueMetricsForCustomResources {
public static final long GB = 1024; // MB
private static final Configuration CONF = new Configuration();
- private static final String CUSTOM_RES_1 = "custom_res_1";
- private static final String CUSTOM_RES_2 = "custom_res_2";
+ public static final String CUSTOM_RES_1 = "custom_res_1";
+ public static final String CUSTOM_RES_2 = "custom_res_2";
public static final String USER = "alice";
private Resource defaultResource;
private MetricsSystem ms;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 73f6fc1..7ef8b9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -79,6 +79,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
@@ -108,7 +109,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-
+import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -127,6 +128,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdate
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.CSQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -134,6 +136,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestQueueMetricsForCustomResources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AllocationState;
@@ -190,6 +193,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
@Before
public void setUp() throws Exception {
+ ResourceUtils.resetResourceTypes(new Configuration());
resourceManager = new ResourceManager() {
@Override
protected RMNodeLabelsManager createNodeLabelManager() {
@@ -5178,23 +5182,109 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
@Test
public void testCSQueueMetrics() throws Exception {
- CapacityScheduler cs = new CapacityScheduler();
- cs.setConf(new YarnConfiguration());
- cs.setRMContext(resourceManager.getRMContext());
- CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- setupQueueConfiguration(conf);
- cs.init(conf);
- cs.start();
- RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 1, "n1");
- RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(50 * GB), 2, "n2");
+ // Initialize resource map
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ ResourceInformation memory =
+ ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores =
+ ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
+ ResourceInformation.newInstance(
+ TestQueueMetricsForCustomResources.CUSTOM_RES_1, "", 1, 10));
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ csConf.setResourceComparator(DominantResourceCalculator.class);
+
+ csConf.set(YarnConfiguration.RESOURCE_TYPES,
+ TestQueueMetricsForCustomResources.CUSTOM_RES_1);
+
+ setupQueueConfiguration(csConf);
+
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+
+ // Don't reset resource types since we have already configured resource
+ // types
+ conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ MockRM rm = new MockRM(conf);
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ RMNode n1 = MockNodes.newNodeInfo(0,
+ MockNodes.newResource(50 * GB, 50,
+ ImmutableMap.<String, String> builder()
+ .put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
+ String.valueOf(1000))
+ .build()),
+ 1, "n1");
+ RMNode n2 = MockNodes.newNodeInfo(0,
+ MockNodes.newResource(50 * GB, 50,
+ ImmutableMap.<String, String> builder()
+ .put(TestQueueMetricsForCustomResources.CUSTOM_RES_1,
+ String.valueOf(2000))
+ .build()),
+ 2, "n2");
cs.handle(new NodeAddedSchedulerEvent(n1));
cs.handle(new NodeAddedSchedulerEvent(n2));
+ Map<String, Long> guaranteedCapA11 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a1")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+ assertEquals(94, guaranteedCapA11
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapA11 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a1")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(3000, maxCapA11
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+
assertEquals(10240, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getGuaranteedMB());
assertEquals(71680, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(102400, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(102400, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
+ Map<String, Long> guaranteedCapA =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+ assertEquals(314, guaranteedCapA
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapA =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(3000, maxCapA
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> guaranteedCapB1 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+ assertEquals(2126, guaranteedCapB1
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapB1 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(3000, maxCapB1
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
// Remove a node, metrics should be updated
cs.handle(new NodeRemovedSchedulerEvent(n2));
@@ -5202,6 +5292,31 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
+ Map<String, Long> guaranteedCapA1 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+
+ assertEquals(104, guaranteedCapA1
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapA1 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(1000, maxCapA1
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> guaranteedCapB11 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+ assertEquals(708, guaranteedCapB11
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapB11 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("b1")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(1000, maxCapB11
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
.getMetrics()).getGuaranteedCapacity(), DELTA);
assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
@@ -5220,20 +5335,49 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
.getMaxAbsoluteCapacity(), DELTA);
// Add child queue to a, and reinitialize. Metrics should be updated
- conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} );
- conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 30.0f);
- conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.0f);
- conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 50.0f);
-
- cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
- null, new RMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInRM(conf),
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a",
+ new String[] {"a1", "a2", "a3"});
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a2", 29.5f);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3", 40.5f);
+ csConf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a3",
+ 50.0f);
+
+ cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
new ClientToAMTokenSecretManagerInRM(), null));
assertEquals(1024, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getGuaranteedMB());
assertEquals(2048, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getGuaranteedMB());
assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a2").getMetrics()).getMaxCapacityMB());
assertEquals(25600, ((CSQueueMetrics)cs.getQueue("a3").getMetrics()).getMaxCapacityMB());
+
+ Map<String, Long> guaranteedCapA2 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a2")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+ assertEquals(30, guaranteedCapA2
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapA2 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a2")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(1000, maxCapA2
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+
+ Map<String, Long> guaranteedCapA3 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a3")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getGuaranteedCapacity();
+ assertEquals(42, guaranteedCapA3
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ Map<String, Long> maxCapA3 =
+ ((CSQueueMetricsForCustomResources) ((CSQueueMetrics) cs.getQueue("a3")
+ .getMetrics()).getQueueMetricsForCustomResources())
+ .getMaxCapacity();
+ assertEquals(500, maxCapA3
+ .get(TestQueueMetricsForCustomResources.CUSTOM_RES_1).longValue());
+ rm.stop();
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org