You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/02/16 22:00:36 UTC
[14/21] hadoop git commit: YARN-7292. Retrospect Resource Profile
Behavior for overriding capability. Contributed by Wangda Tan.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java
deleted file mode 100644
index cbad3f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestProfileCapability.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.ProfileCapability;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Test profile capability behavior.
- */
-public class TestProfileCapability {
- @Before
- public void setup() {
- // Initialize resource map
- Map<String, ResourceInformation> riMap = new HashMap<>();
-
- // Initialize mandatory resources
- riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
- riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
-
- for (int i = 0; i < 5; i++) {
- String resourceName = "res-" + i;
- riMap.put(resourceName, ResourceInformation
- .newInstance(resourceName, "", 0, ResourceTypes.COUNTABLE, 0,
- Integer.MAX_VALUE));
- }
-
- ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
- }
-
- @Test
- public void testConvertProfileCapabilityToResource() {
- Resource profile1 = Resource.newInstance(1, 1);
- profile1.setResourceValue("res-0", 1);
- profile1.setResourceValue("res-1", 1);
-
- Resource profile2 = Resource.newInstance(2, 2);
- profile2.setResourceValue("res-0", 2);
- profile2.setResourceValue("res-1", 2);
-
- Resource profile3 = Resource.newInstance(3, 3);
- profile3.setResourceValue("res-0", 3);
- profile3.setResourceValue("res-1", 3);
-
- Map<String, Resource> profiles = ImmutableMap.of("profile1", profile1,
- "profile2", profile2, "profile3", profile3, "default", profile1);
-
- // Test case 1, set override value to (1, 1, 0), since we only allow
- // overwrite for positive value, it is still profile1.
- ProfileCapability pc = ProfileCapability.newInstance("profile1",
- Resource.newInstance(1, 1));
- Assert.assertEquals(profile1, ProfileCapability.toResource(pc, profiles));
-
- // Test case 2, similarly, negative value won't be respected.
- pc = ProfileCapability.newInstance("profile1",
- Resource.newInstance(1, -1));
- Assert.assertEquals(profile1, ProfileCapability.toResource(pc, profiles));
-
- // Test case 3, do overwrite for memory and vcores, the result is (3,3,1,1)
- Resource expected = Resource.newInstance(3, 3);
- expected.setResourceValue("res-0", 1);
- expected.setResourceValue("res-1", 1);
- pc = ProfileCapability.newInstance("profile1",
- Resource.newInstance(3, 3));
- Assert.assertEquals(expected, ProfileCapability.toResource(pc, profiles));
-
- // Test case 3, do overwrite for mem and res-1, the result is (3,1,3,1)
- expected = Resource.newInstance(3, 1);
- expected.setResourceValue("res-0", 3);
- expected.setResourceValue("res-1", 1);
-
- Resource overwrite = Resource.newInstance(3, 0);
- overwrite.setResourceValue("res-0", 3);
- overwrite.setResourceValue("res-1", 0);
-
- pc = ProfileCapability.newInstance("profile1", overwrite);
- Assert.assertEquals(expected, ProfileCapability.toResource(pc, profiles));
-
- // Test case 4, when null profile is specified, use default.
- pc = ProfileCapability.newInstance("", null);
- Assert.assertEquals(profile1, ProfileCapability.toResource(pc, profiles));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 83f912f..4ba1bdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -384,7 +384,6 @@ public class BuilderUtils {
request.setNumContainers(r.getNumContainers());
request.setNodeLabelExpression(r.getNodeLabelExpression());
request.setExecutionTypeRequest(r.getExecutionTypeRequest());
- request.setProfileCapability(r.getProfileCapability());
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 4934243..33451295 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
@@ -67,7 +66,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt
.RMAppAttemptState;
@@ -627,37 +625,6 @@ public class RMServerUtils {
}
}
- public static void convertProfileToResourceCapability(ResourceRequest ask,
- Configuration conf, ResourceProfilesManager resourceProfilesManager)
- throws YarnException {
-
- if (LOG_HANDLE.isDebugEnabled()) {
- LOG_HANDLE
- .debug("Converting profile to resource capability for ask " + ask);
- }
-
- boolean profilesEnabled =
- conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!profilesEnabled) {
- if (ask.getProfileCapability() != null && !ask.getProfileCapability()
- .getProfileCapabilityOverride().equals(Resources.none())) {
- ask.setCapability(
- ask.getProfileCapability().getProfileCapabilityOverride());
- }
- } else {
- if (ask.getProfileCapability() != null) {
- ask.setCapability(ProfileCapability
- .toResource(ask.getProfileCapability(),
- resourceProfilesManager.getResourceProfiles()));
- }
- }
- if (LOG_HANDLE.isDebugEnabled()) {
- LOG_HANDLE
- .debug("Converted profile to resource capability for ask " + ask);
- }
- }
-
public static Long getOrDefault(Map<String, Long> map, String key,
Long defaultValue) {
if (map.containsKey(key)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 0080a29..27563d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -40,12 +40,10 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
@@ -270,14 +268,6 @@ public class SchedulerUtils {
private static void validateResourceRequest(ResourceRequest resReq,
Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
- try {
- RMServerUtils.convertProfileToResourceCapability(resReq,
- rmContext.getYarnConfiguration(),
- rmContext.getResourceProfilesManager());
- } catch (YarnException ye) {
- throw new InvalidResourceRequestException(ye);
- }
-
Resource requestedResource = resReq.getCapability();
for (int i = 0; i < ResourceUtils.getNumberOfKnownResourceTypes(); i++) {
ResourceInformation reqRI = requestedResource.getResourceInformation(i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index ceaf236..90e4be8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRespons
import org.apache.hadoop.yarn.api.protocolrecords
.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.AllocateRequestPBImpl;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -52,7 +51,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -64,8 +62,6 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.MockResourceProfileManager;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -73,13 +69,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java
deleted file mode 100644
index b75f925..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/MockResourceProfileManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.resource;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Mock ResourceProfileManager for unit test.
- */
-public class MockResourceProfileManager extends ResourceProfilesManagerImpl {
- private Map<String, Resource> profiles;
- private boolean featureEnabled;
-
- public MockResourceProfileManager(Map<String, Resource> profiles) {
- this.profiles = new HashMap<>();
- this.profiles.putAll(profiles);
-
- // Set minimum / maximum allocation so test doesn't need to add them
- // every time.
- this.profiles.put(ResourceProfilesManagerImpl.MINIMUM_PROFILE,
- ResourceUtils.getResourceTypesMinimumAllocation());
- this.profiles.put(ResourceProfilesManagerImpl.MAXIMUM_PROFILE,
- ResourceUtils.getResourceTypesMaximumAllocation());
- }
-
- @Override
- public void init(Configuration config) throws IOException {
- this.featureEnabled = config.getBoolean(
- YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- }
-
- @Override
- public Resource getProfile(String profile) throws YarnException {
- if (!featureEnabled) {
- throw new YARNFeatureNotEnabledException("");
- }
- return profiles.get(profile);
- }
-
- @Override
- public Map<String, Resource> getResourceProfiles()
- throws YARNFeatureNotEnabledException {
- if (!featureEnabled) {
- throw new YARNFeatureNotEnabledException("");
- }
- return profiles;
- }
-
- @Override
- public void reloadProfiles() throws IOException {
- throw new IOException("Not supported");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aae62991/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index ea29f7f..3a8d33a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -18,27 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import com.google.common.collect.ImmutableMap;
-import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.ProfileCapability;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.MockResourceProfileManager;
-import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -46,7 +32,6 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -55,101 +40,6 @@ import java.util.Map;
*/
public class TestCapacitySchedulerWithMultiResourceTypes {
private static String RESOURCE_1 = "res1";
- private final int GB = 1024;
-
- @Test
- public void testBasicCapacitySchedulerWithProfile() throws Exception {
-
- // Initialize resource map
- Map<String, ResourceInformation> riMap = new HashMap<>();
-
- // Initialize mandatory resources
- ResourceInformation memory = ResourceInformation.newInstance(
- ResourceInformation.MEMORY_MB.getName(),
- ResourceInformation.MEMORY_MB.getUnits(),
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
- ResourceInformation vcores = ResourceInformation.newInstance(
- ResourceInformation.VCORES.getName(),
- ResourceInformation.VCORES.getUnits(),
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
- riMap.put(ResourceInformation.MEMORY_URI, memory);
- riMap.put(ResourceInformation.VCORES_URI, vcores);
- riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
- ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
-
- ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
-
- CapacitySchedulerConfiguration csconf =
- new CapacitySchedulerConfiguration();
- csconf.setMaximumApplicationMasterResourcePerQueuePercent("root", 100.0f);
- csconf.setMaximumAMResourcePercentPerPartition("root", "", 100.0f);
- csconf.setMaximumApplicationMasterResourcePerQueuePercent("root.default",
- 100.0f);
- csconf.setMaximumAMResourcePercentPerPartition("root.default", "", 100.0f);
- csconf.setResourceComparator(DominantResourceCalculator.class);
-
- YarnConfiguration conf = new YarnConfiguration(csconf);
- // Don't reset resource types since we have already configured resource
- // types
- conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
- conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
-
- final MockResourceProfileManager mrpm = new MockResourceProfileManager(
- ImmutableMap.of("res-1", TestUtils
- .createResource(2 * GB, 2, ImmutableMap.of(RESOURCE_1, 2))));
-
- MockRM rm = new MockRM(conf) {
- @Override
- protected ResourceProfilesManager createResourceProfileManager() {
- return mrpm;
- }
- };
- rm.start();
-
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- LeafQueue leafQueue = (LeafQueue) cs.getQueue("default");
-
- MockNM nm1 = rm.registerNode("h1:1234",
- TestUtils.createResource(8 * GB, 8, ImmutableMap.of(RESOURCE_1, 8)));
-
- RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
- MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
-
- Assert.assertEquals(Resource.newInstance(1 * GB, 1),
- leafQueue.getUsedResources());
-
- RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
-
- // Now request resource:
- am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
- Resource.newInstance(1 * GB, 1)).numContainers(1).resourceName("*")
- .profileCapability(ProfileCapability
- .newInstance("res-1",
- Resource.newInstance(2 * GB, 2))).build()),
- null);
-
- // Do node heartbeats 1 time and check container allocated.
- cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
-
- // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2>
- Assert.assertEquals(
- TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)),
- leafQueue.getUsedResources());
-
- // Acquire container
- AllocateResponse amResponse = am1.allocate(null, null);
- Assert.assertFalse(amResponse.getAllocatedContainers().isEmpty());
- ContainerTokenIdentifier containerTokenIdentifier =
- BuilderUtils.newContainerTokenIdentifier(
- amResponse.getAllocatedContainers().get(0).getContainerToken());
- Assert.assertEquals(
- TestUtils.createResource(2 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
- containerTokenIdentifier.getResource());
- }
@Test
public void testMaximumAllocationRefreshWithMultipleResourceTypes() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org