You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by eb...@apache.org on 2019/12/20 19:46:58 UTC
[hadoop] branch branch-2.10 updated: YARN-10009. In Capacity
Scheduler,
DRC can treat minimum user limit percent as a max when custom resource is
defined. Contributed by Eric Payne.
This is an automated email from the ASF dual-hosted git repository.
ebadger pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new cb5b80d YARN-10009. In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined. Contributed by Eric Payne.
cb5b80d is described below
commit cb5b80d6cbf95a817376058e2a62c850f55787e6
Author: Eric Badger <eb...@verizonmedia.com>
AuthorDate: Fri Dec 20 19:40:55 2019 +0000
YARN-10009. In Capacity Scheduler, DRC can treat minimum user limit percent as a max when custom resource is defined. Contributed by Eric Payne.
---
.../util/resource/DominantResourceCalculator.java | 12 +-
...estCapacitySchedulerWithMultiResourceTypes.java | 130 +++++++++++++++++++++
2 files changed, 138 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index edd3415..3e0d68b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -415,10 +415,14 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource divideAndCeil(Resource numerator, float denominator) {
- return Resources.createResource(
- divideAndCeil(numerator.getMemorySize(), denominator),
- divideAndCeil(numerator.getVirtualCores(), denominator)
- );
+ Resource ret = Resource.newInstance(numerator);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation resourceInformation = ret.getResourceInformation(i);
+ resourceInformation
+ .setValue(divideAndCeil(resourceInformation.getValue(), denominator));
+ }
+ return ret;
}
@Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index 1a30e1d..0c06d17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import com.google.common.collect.ImmutableMap;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -28,19 +32,36 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
import org.junit.Assert;
+import static org.junit.Assert.assertEquals;
import org.junit.Test;
+import com.google.common.collect.ImmutableMap;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -187,4 +208,113 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
rm.close();
}
+
+ @Test(timeout = 300000)
+ public void testConsumeAllExtendedResourcesWithSmallMinUserLimitPct()
+ throws Exception {
+ int GB = 1024;
+
+ // Initialize resource map for 3 types.
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ ResourceInformation res1 = ResourceInformation.newInstance("res_1",
+ "", 0, 10);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put("res_1", res1);
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.set("yarn.resource-types", "res_1");
+ csconf.set("yarn.resource-types.res_1.minimum-allocation", "0");
+ csconf.set("yarn.resource-types.res_1.maximum-allocation", "10");
+ csconf.setResourceComparator(DominantResourceCalculator.class);
+
+ YarnConfiguration yarnConf = new YarnConfiguration(csconf);
+ // Don't reset resource types since we have already configured resource
+ // types
+ yarnConf.setBoolean(TestResourceUtils.TEST_CONF_RESET_RESOURCE_TYPES,
+ false);
+ yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ MockRM rm = new MockRM(yarnConf);
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ LeafQueue qb = (LeafQueue)cs.getQueue("default");
+ // Setting minimum user limit percent should not affect max user resource
+ // limit using extended resources with DRF (see YARN-10009).
+ qb.setUserLimit(25);
+
+ // add app 1
+ ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
+ ApplicationAttemptId appAttemptId =
+ BuilderUtils.newApplicationAttemptId(appId, 1);
+
+ RMAppAttemptMetrics attemptMetric =
+ new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
+ RMAppImpl app = mock(RMAppImpl.class);
+ when(app.getApplicationId()).thenReturn(appId);
+ RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
+ Container container = mock(Container.class);
+ when(attempt.getMasterContainer()).thenReturn(container);
+ ApplicationSubmissionContext submissionContext = mock(
+ ApplicationSubmissionContext.class);
+ when(attempt.getSubmissionContext()).thenReturn(submissionContext);
+ when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
+ when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
+ when(app.getCurrentAppAttempt()).thenReturn(attempt);
+
+ rm.getRMContext().getRMApps().put(appId, app);
+
+ SchedulerEvent addAppEvent =
+ new AppAddedSchedulerEvent(appId, "default", "user1");
+ cs.handle(addAppEvent);
+ SchedulerEvent addAttemptEvent =
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
+ cs.handle(addAttemptEvent);
+
+ // add nodes to cluster. Cluster has 20GB, 20 vcores, 80 res_1s.
+ HashMap<String, Long> resMap = new HashMap<String, Long>();
+ resMap.put("res_1", 80L);
+ Resource newResource = Resource.newInstance(2048 * GB, 100, resMap);
+ RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
+ cs.handle(new NodeAddedSchedulerEvent(node));
+
+ FiCaSchedulerApp fiCaApp1 =
+ cs.getSchedulerApplications().get(app.getApplicationId())
+ .getCurrentAppAttempt();
+
+ // allocate 8 containers for app1 with 1GB memory, 1 vcore, 10 res_1s
+ for (int i = 0; i < 8; i++) {
+ fiCaApp1.updateResourceRequests(Collections.singletonList(
+ ResourceRequest.newBuilder()
+ .capability(TestUtils.createResource(1 * GB, 1,
+ ImmutableMap.of("res_1", 10)))
+ .numContainers(1)
+ .resourceName("*")
+ .build()));
+ cs.handle(new NodeUpdateSchedulerEvent(node));
+ }
+ assertEquals(8*GB, fiCaApp1.getCurrentConsumption().getMemorySize());
+ assertEquals(80,
+ fiCaApp1.getCurrentConsumption()
+ .getResourceInformation("res_1").getValue());
+
+ rm.close();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org