You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/14 17:44:14 UTC
[11/50] [abbrv] hadoop git commit: YARN-8202. DefaultAMSProcessor
should properly check units of requested custom resource types against
minimum/maximum allocation (snemeth via rkanter)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8fa7cb6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 90e4be8..9696741 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -22,9 +22,13 @@ import static java.lang.Thread.sleep;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES;
+
+import static org.junit.Assert.fail;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
@@ -61,6 +65,7 @@ import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -75,6 +80,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .FairSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
@@ -365,7 +373,7 @@ public class TestApplicationMasterService {
am2.addContainerToBeReleased(cId);
try {
am2.schedule();
- Assert.fail("Exception was expected!!");
+ fail("Exception was expected!!");
} catch (InvalidContainerReleaseException e) {
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.append(cId.toString());
@@ -460,7 +468,7 @@ public class TestApplicationMasterService {
FinalApplicationStatus.FAILED, "", "");
try {
am1.unregisterAppAttempt(req, false);
- Assert.fail("ApplicationMasterNotRegisteredException should be thrown");
+ fail("ApplicationMasterNotRegisteredException should be thrown");
} catch (ApplicationMasterNotRegisteredException e) {
Assert.assertNotNull(e);
Assert.assertNotNull(e.getMessage());
@@ -468,7 +476,7 @@ public class TestApplicationMasterService {
"Application Master is trying to unregister before registering for:"
));
} catch (Exception e) {
- Assert.fail("ApplicationMasterNotRegisteredException should be thrown");
+ fail("ApplicationMasterNotRegisteredException should be thrown");
}
am1.registerAppAttempt();
@@ -627,9 +635,7 @@ public class TestApplicationMasterService {
Assert.assertEquals("UPDATE_OUTSTANDING_ERROR",
response.getUpdateErrors().get(0).getReason());
} finally {
- if (rm != null) {
- rm.close();
- }
+ rm.close();
}
}
@@ -709,34 +715,48 @@ public class TestApplicationMasterService {
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
- CapacitySchedulerConfiguration csconf =
- new CapacitySchedulerConfiguration();
- csconf.setResourceComparator(DominantResourceCalculator.class);
+ final YarnConfiguration yarnConf;
+ if (schedulerCls.getCanonicalName()
+ .equals(CapacityScheduler.class.getCanonicalName())) {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ csConf.setResourceComparator(DominantResourceCalculator.class);
+ yarnConf = new YarnConfiguration(csConf);
+ } else if (schedulerCls.getCanonicalName()
+ .equals(FairScheduler.class.getCanonicalName())) {
+ FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
+ yarnConf = new YarnConfiguration(fsConf);
+ } else {
+ throw new IllegalStateException(
+ "Scheduler class is of wrong type: " + schedulerCls);
+ }
- YarnConfiguration conf = new YarnConfiguration(csconf);
// Don't reset resource types since we have already configured resource
// types
- conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, schedulerCls,
+ yarnConf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES,
+ false);
+ yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, schedulerCls,
ResourceScheduler.class);
- conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
+ yarnConf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
- MockRM rm = new MockRM(conf);
+ MockRM rm = new MockRM(yarnConf);
rm.start();
MockNM nm1 = rm.registerNode("199.99.99.1:1234", TestUtils
.createResource(DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, null));
- RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
// Now request resource, memory > allowed
boolean exception = false;
try {
- am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
- Resource.newInstance(9 * GB, 1)).numContainers(1).resourceName("*")
- .build()), null);
+ am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
+ .capability(Resource.newInstance(9 * GB, 1))
+ .numContainers(1)
+ .resourceName("*")
+ .build()), null);
} catch (InvalidResourceRequestException e) {
exception = true;
}
@@ -744,10 +764,12 @@ public class TestApplicationMasterService {
exception = false;
try {
- // Now request resource, vcore > allowed
- am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
- Resource.newInstance(8 * GB, 18)).numContainers(1).resourceName("*")
- .build()), null);
+ // Now request resource, vcores > allowed
+ am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
+ .capability(Resource.newInstance(8 * GB, 18))
+ .numContainers(1)
+ .resourceName("*")
+ .build()), null);
} catch (InvalidResourceRequestException e) {
exception = true;
}
@@ -756,6 +778,73 @@ public class TestApplicationMasterService {
rm.close();
}
+ @Test
+ public void testValidateRequestCapacityAgainstMinMaxAllocationWithDifferentUnits()
+ throws Exception {
+
+ // Initialize resource map for 2 types.
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+
+ // Initialize mandatory resources
+ ResourceInformation memory =
+ ResourceInformation.newInstance(ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores =
+ ResourceInformation.newInstance(ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ ResourceInformation res1 =
+ ResourceInformation.newInstance("res_1", "G", 0, 4);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put("res_1", res1);
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ FairSchedulerConfiguration fsConf =
+ new FairSchedulerConfiguration();
+
+ YarnConfiguration yarnConf = new YarnConfiguration(fsConf);
+ // Don't reset resource types since we have already configured resource
+ // types
+ yarnConf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES,
+ false);
+ yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+ yarnConf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
+
+ MockRM rm = new MockRM(yarnConf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("199.99.99.1:1234",
+ ResourceTypesTestHelper.newResource(
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ ImmutableMap.<String, String> builder()
+ .put("res_1", "5G").build()));
+
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+
+ // Now request res_1, 500M < 5G so it should be allowed
+ try {
+ am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
+ .capability(ResourceTypesTestHelper.newResource(4 * GB, 1,
+ ImmutableMap.<String, String> builder()
+ .put("res_1", "500M")
+ .build()))
+ .numContainers(1).resourceName("*").build()), null);
+ } catch (InvalidResourceRequestException e) {
+ fail(
+ "Allocate request should be accepted but exception was thrown: " + e);
+ }
+
+ rm.close();
+ }
+
@Test(timeout = 300000)
public void testValidateRequestCapacityAgainstMinMaxAllocationFor3rdResourceTypes()
throws Exception {
@@ -774,11 +863,11 @@ public class TestApplicationMasterService {
ResourceInformation.VCORES.getUnits(),
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
- ResourceInformation res_1 = ResourceInformation.newInstance("res_1",
+ ResourceInformation res1 = ResourceInformation.newInstance("res_1",
ResourceInformation.VCORES.getUnits(), 0, 4);
riMap.put(ResourceInformation.MEMORY_URI, memory);
riMap.put(ResourceInformation.VCORES_URI, vcores);
- riMap.put("res_1", res_1);
+ riMap.put("res_1", res1);
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
@@ -786,15 +875,16 @@ public class TestApplicationMasterService {
new CapacitySchedulerConfiguration();
csconf.setResourceComparator(DominantResourceCalculator.class);
- YarnConfiguration conf = new YarnConfiguration(csconf);
+ YarnConfiguration yarnConf = new YarnConfiguration(csconf);
// Don't reset resource types since we have already configured resource
// types
- conf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, false);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ yarnConf.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES,
+ false);
+ yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
- conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
+ yarnConf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, false);
- MockRM rm = new MockRM(conf);
+ MockRM rm = new MockRM(yarnConf);
rm.start();
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -805,18 +895,21 @@ public class TestApplicationMasterService {
DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
ImmutableMap.of("res_1", 4)));
- RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
- Assert.assertEquals(Resource.newInstance(1 * GB, 1),
+ Assert.assertEquals(Resource.newInstance(GB, 1),
leafQueue.getUsedResources());
// Now request resource, memory > allowed
boolean exception = false;
try {
- am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
- TestUtils.createResource(9 * GB, 1, ImmutableMap.of("res_1", 1)))
- .numContainers(1).resourceName("*").build()), null);
+ am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
+ .capability(TestUtils.createResource(9 * GB, 1,
+ ImmutableMap.of("res_1", 1)))
+ .numContainers(1)
+ .resourceName("*")
+ .build()), null);
} catch (InvalidResourceRequestException e) {
exception = true;
}
@@ -824,11 +917,13 @@ public class TestApplicationMasterService {
exception = false;
try {
- // Now request resource, vcore > allowed
- am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
- TestUtils.createResource(8 * GB, 18, ImmutableMap.of("res_1", 1)))
- .numContainers(1).resourceName("*")
- .build()), null);
+ // Now request resource, vcores > allowed
+ am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
+ .capability(
+ TestUtils.createResource(8 * GB, 18, ImmutableMap.of("res_1", 1)))
+ .numContainers(1)
+ .resourceName("*")
+ .build()), null);
} catch (InvalidResourceRequestException e) {
exception = true;
}
@@ -837,10 +932,12 @@ public class TestApplicationMasterService {
exception = false;
try {
// Now request resource, res_1 > allowed
- am1.allocate(Arrays.asList(ResourceRequest.newBuilder().capability(
- TestUtils.createResource(8 * GB, 1, ImmutableMap.of("res_1", 100)))
- .numContainers(1).resourceName("*")
- .build()), null);
+ am1.allocate(Collections.singletonList(ResourceRequest.newBuilder()
+ .capability(TestUtils.createResource(8 * GB, 1,
+ ImmutableMap.of("res_1", 100)))
+ .numContainers(1)
+ .resourceName("*")
+ .build()), null);
} catch (InvalidResourceRequestException e) {
exception = true;
}
@@ -856,7 +953,7 @@ public class TestApplicationMasterService {
rmContainer.handle(
new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
} else {
- Assert.fail("Cannot find RMContainer");
+ fail("Cannot find RMContainer");
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8fa7cb6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index cb1f794..15cfdb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -26,7 +26,9 @@ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.util.Arrays;
@@ -35,6 +37,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -42,6 +45,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
@@ -63,8 +67,10 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.resourcetypes.ResourceTypesTestHelper;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
@@ -83,20 +89,79 @@ import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import org.junit.rules.ExpectedException;
public class TestSchedulerUtils {
private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class);
-
+ private static Resource configuredMaxAllocation;
+
+ private static class CustomResourceTypesConfigurationProvider
+ extends LocalConfigurationProvider {
+
+ @Override
+ public InputStream getConfigurationInputStream(Configuration bootstrapConf,
+ String name) throws YarnException, IOException {
+ if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
+ return new ByteArrayInputStream(
+ ("<configuration>\n" +
+ " <property>\n" +
+ " <name>yarn.resource-types</name>\n" +
+ " <value>custom-resource-1," +
+ "custom-resource-2,custom-resource-3</value>\n" +
+ " </property>\n" +
+ " <property>\n" +
+ " <name>yarn.resource-types" +
+ ".custom-resource-1.units</name>\n" +
+ " <value>G</value>\n" +
+ " </property>\n" +
+ " <property>\n" +
+ " <name>yarn.resource-types" +
+ ".custom-resource-2.units</name>\n" +
+ " <value>G</value>\n" +
+ " </property>\n" +
+ "</configuration>\n").getBytes());
+ } else {
+ return super.getConfigurationInputStream(bootstrapConf, name);
+ }
+ }
+ }
private RMContext rmContext = getMockRMContext();
+
private static YarnConfiguration conf = new YarnConfiguration();
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ private void initResourceTypes() {
+ Configuration yarnConf = new Configuration();
+ yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ CustomResourceTypesConfigurationProvider.class.getName());
+ ResourceUtils.resetResourceTypes(yarnConf);
+ }
+
+ @Before
+ public void setUp() {
+ initResourceTypes();
+ //this needs to be initialized after initResourceTypes is called
+ configuredMaxAllocation = Resource.newInstance(8192, 4,
+ ImmutableMap.<String,
+ Long>builder()
+ .put("custom-resource-1", Long.MAX_VALUE)
+ .put("custom-resource-2", Long.MAX_VALUE)
+ .put("custom-resource-3", Long.MAX_VALUE)
+ .build());
+ }
+
@Test (timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -150,16 +215,18 @@ public class TestSchedulerUtils {
// multiple of minMemory > maxMemory, then reduce to maxMemory
SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
- assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
+ assertEquals(maxResource.getMemorySize(),
+ ask.getCapability().getMemorySize());
// ask is more than max
maxResource = Resources.createResource(maxMemory, 0);
ask.setCapability(Resources.createResource(maxMemory + 100));
SchedulerUtils.normalizeRequest(ask, resourceCalculator, minResource,
maxResource);
- assertEquals(maxResource.getMemorySize(), ask.getCapability().getMemorySize());
+ assertEquals(maxResource.getMemorySize(),
+ ask.getCapability().getMemorySize());
}
-
+
@Test (timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
ResourceCalculator resourceCalculator = new DominantResourceCalculator();
@@ -201,10 +268,11 @@ public class TestSchedulerUtils {
Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
QueueInfo queueInfo = mock(QueueInfo.class);
when(queueInfo.getQueueName()).thenReturn("queue");
- when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
+ when(queueInfo.getAccessibleNodeLabels())
+ .thenReturn(queueAccessibleNodeLabels);
when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean()))
.thenReturn(queueInfo);
-
+
Resource maxResource = Resources.createResource(
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
@@ -363,7 +431,7 @@ public class TestSchedulerUtils {
rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
Arrays.asList("x"));
}
- Assert.assertTrue("InvalidLabelResourceRequestException excpeted",
+ Assert.assertTrue("InvalidLabelResourceRequestException expected",
invalidlabelexception);
// queue is "*", always succeeded
try {
@@ -610,11 +678,9 @@ public class TestSchedulerUtils {
// more than max vcores
try {
- Resource resource =
- Resources
- .createResource(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
+ Resource resource = Resources.createResource(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES + 1);
ResourceRequest resReq =
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
@@ -648,10 +714,10 @@ public class TestSchedulerUtils {
waitForLaunchedState(attempt);
// Create a client to the RM.
- final Configuration conf = rm.getConfig();
- final YarnRPC rpc = YarnRPC.create(conf);
+ final Configuration yarnConf = rm.getConfig();
+ final YarnRPC rpc = YarnRPC.create(yarnConf);
- UserGroupInformation currentUser =
+ UserGroupInformation currentUser =
UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
Credentials credentials = containerManager.getContainerCredentials();
final InetSocketAddress rmBindAddress =
@@ -665,7 +731,7 @@ public class TestSchedulerUtils {
@Override
public ApplicationMasterProtocol run() {
return (ApplicationMasterProtocol) rpc.getProxy(
- ApplicationMasterProtocol.class, rmBindAddress, conf);
+ ApplicationMasterProtocol.class, rmBindAddress, yarnConf);
}
});
@@ -775,6 +841,127 @@ public class TestSchedulerUtils {
}
}
+ @Test
+ public void testCustomResourceRequestedUnitIsSmallerThanAvailableUnit()
+ throws InvalidResourceRequestException {
+ Resource requestedResource =
+ ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "11"));
+
+ Resource availableResource =
+ ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "0G"));
+
+ exception.expect(InvalidResourceRequestException.class);
+ exception.expectMessage(InvalidResourceRequestExceptionMessageGenerator
+ .create().withRequestedResourceType("custom-resource-1")
+ .withRequestedResource(requestedResource)
+ .withAvailableAllocation(availableResource)
+ .withMaxAllocation(configuredMaxAllocation).build());
+
+ SchedulerUtils.checkResourceRequestAgainstAvailableResource(
+ requestedResource, availableResource);
+ }
+
+ @Test
+ public void testCustomResourceRequestedUnitIsSmallerThanAvailableUnit2() {
+ Resource requestedResource =
+ ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "11"));
+
+ Resource availableResource =
+ ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "1G"));
+
+ try {
+ SchedulerUtils.checkResourceRequestAgainstAvailableResource(
+ requestedResource, availableResource);
+ } catch (InvalidResourceRequestException e) {
+ fail(String.format(
+ "Resource request should be accepted. Requested: %s, available: %s",
+ requestedResource, availableResource));
+ }
+ }
+
+ @Test
+ public void testCustomResourceRequestedUnitIsGreaterThanAvailableUnit()
+ throws InvalidResourceRequestException {
+ Resource requestedResource =
+ ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "1M"));
+
+ Resource availableResource = ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.<String, String> builder().put("custom-resource-1", "120k")
+ .build());
+
+ exception.expect(InvalidResourceRequestException.class);
+ exception.expectMessage(InvalidResourceRequestExceptionMessageGenerator
+ .create().withRequestedResourceType("custom-resource-1")
+ .withRequestedResource(requestedResource)
+ .withAvailableAllocation(availableResource)
+ .withMaxAllocation(configuredMaxAllocation).build());
+ SchedulerUtils.checkResourceRequestAgainstAvailableResource(
+ requestedResource, availableResource);
+ }
+
+ @Test
+ public void testCustomResourceRequestedUnitIsGreaterThanAvailableUnit2() {
+ Resource requestedResource = ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.<String, String> builder().put("custom-resource-1", "11M")
+ .build());
+
+ Resource availableResource =
+ ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "1G"));
+
+ try {
+ SchedulerUtils.checkResourceRequestAgainstAvailableResource(
+ requestedResource, availableResource);
+ } catch (InvalidResourceRequestException e) {
+ fail(String.format(
+ "Resource request should be accepted. Requested: %s, available: %s",
+ requestedResource, availableResource));
+ }
+ }
+
+ @Test
+ public void testCustomResourceRequestedUnitIsSameAsAvailableUnit() {
+ Resource requestedResource = ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "11M"));
+
+ Resource availableResource = ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "100M"));
+
+ try {
+ SchedulerUtils.checkResourceRequestAgainstAvailableResource(
+ requestedResource, availableResource);
+ } catch (InvalidResourceRequestException e) {
+ fail(String.format(
+ "Resource request should be accepted. Requested: %s, available: %s",
+ requestedResource, availableResource));
+ }
+ }
+
+ @Test
+ public void testCustomResourceRequestedUnitIsSameAsAvailableUnit2()
+ throws InvalidResourceRequestException {
+ Resource requestedResource = ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "110M"));
+
+ Resource availableResource = ResourceTypesTestHelper.newResource(1, 1,
+ ImmutableMap.of("custom-resource-1", "100M"));
+
+ exception.expect(InvalidResourceRequestException.class);
+ exception.expectMessage(InvalidResourceRequestExceptionMessageGenerator
+ .create().withRequestedResourceType("custom-resource-1")
+ .withRequestedResource(requestedResource)
+ .withAvailableAllocation(availableResource)
+ .withMaxAllocation(configuredMaxAllocation).build());
+
+ SchedulerUtils.checkResourceRequestAgainstAvailableResource(
+ requestedResource, availableResource);
+ }
+
public static void waitSchedulerApplicationAttemptStopped(
AbstractYarnScheduler ys,
ApplicationAttemptId attemptId) throws InterruptedException {
@@ -801,8 +988,7 @@ public class TestSchedulerUtils {
public static SchedulerApplication<SchedulerApplicationAttempt>
verifyAppAddedAndRemovedFromScheduler(
Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
- EventHandler<SchedulerEvent> handler, String queueName)
- throws Exception {
+ EventHandler<SchedulerEvent> handler, String queueName) {
ApplicationId appId =
ApplicationId.newInstance(System.currentTimeMillis(), 1);
@@ -832,4 +1018,60 @@ public class TestSchedulerUtils {
when(rmContext.getNodeLabelManager()).thenReturn(nlm);
return rmContext;
}
+
+ private static class InvalidResourceRequestExceptionMessageGenerator {
+
+ private StringBuilder sb;
+ private Resource requestedResource;
+ private Resource availableAllocation;
+ private Resource configuredMaxAllowedAllocation;
+ private String resourceType;
+
+ InvalidResourceRequestExceptionMessageGenerator(StringBuilder sb) {
+ this.sb = sb;
+ }
+
+ public static InvalidResourceRequestExceptionMessageGenerator create() {
+ return new InvalidResourceRequestExceptionMessageGenerator(
+ new StringBuilder());
+ }
+
+ InvalidResourceRequestExceptionMessageGenerator withRequestedResource(
+ Resource r) {
+ this.requestedResource = r;
+ return this;
+ }
+
+ InvalidResourceRequestExceptionMessageGenerator withRequestedResourceType(
+ String rt) {
+ this.resourceType = rt;
+ return this;
+ }
+
+ InvalidResourceRequestExceptionMessageGenerator withAvailableAllocation(
+ Resource r) {
+ this.availableAllocation = r;
+ return this;
+ }
+
+ InvalidResourceRequestExceptionMessageGenerator withMaxAllocation(
+ Resource r) {
+ this.configuredMaxAllowedAllocation = r;
+ return this;
+ }
+
+ public String build() {
+ return sb
+ .append("Invalid resource request, requested resource type=[")
+ .append(resourceType).append("]")
+ .append(" < 0 or greater than maximum allowed allocation. ")
+ .append("Requested resource=").append(requestedResource).append(", ")
+ .append("maximum allowed allocation=").append(availableAllocation)
+ .append(", please note that maximum allowed allocation is calculated "
+ + "by scheduler based on maximum resource of " +
+ "registered NodeManagers, which might be less than " +
+ "configured maximum allocation=")
+ .append(configuredMaxAllowedAllocation).toString();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org