You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2016/12/13 20:16:08 UTC
[12/13] hadoop git commit: YARN-5587. Add support for resource
profiles. (vvasudev via asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 9890296..39d772a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -23,8 +23,10 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
@@ -40,6 +42,7 @@ public class ResourceRequestPBImpl extends ResourceRequest {
private Priority priority = null;
private Resource capability = null;
private ExecutionTypeRequest executionTypeRequest = null;
+ private ProfileCapability profile = null;
public ResourceRequestPBImpl() {
@@ -52,7 +55,7 @@ public class ResourceRequestPBImpl extends ResourceRequest {
}
public ResourceRequestProto getProto() {
- mergeLocalToProto();
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
@@ -69,6 +72,9 @@ public class ResourceRequestPBImpl extends ResourceRequest {
builder.setExecutionTypeRequest(
ProtoUtils.convertToProtoFormat(this.executionTypeRequest));
}
+ if (this.profile != null) {
+ builder.setProfile(converToProtoFormat(this.profile));
+ }
}
private void mergeLocalToProto() {
@@ -229,7 +235,8 @@ public class ResourceRequestPBImpl extends ResourceRequest {
+ ", Location: " + getResourceName()
+ ", Relax Locality: " + getRelaxLocality()
+ ", Execution Type Request: " + getExecutionTypeRequest()
- + ", Node Label Expression: " + getNodeLabelExpression() + "}";
+ + ", Node Label Expression: " + getNodeLabelExpression()
+ + ", Resource Profile: " + getProfileCapability() + "}";
}
@Override
@@ -250,4 +257,34 @@ public class ResourceRequestPBImpl extends ResourceRequest {
}
builder.setNodeLabelExpression(nodeLabelExpression);
}
+
+ @Override
+ public void setProfileCapability(ProfileCapability profileCapability) {
+ maybeInitBuilder();
+ if (profile == null) {
+ builder.clearProfile();
+ }
+ this.profile = profileCapability;
+ }
+
+ @Override
+ public ProfileCapability getProfileCapability() {
+ if (profile != null) {
+ return profile;
+ }
+ ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasProfile()) {
+ return null;
+ }
+ return new ProfileCapabilityPBImpl(p.getProfile());
+ }
+
+ private ProfileCapabilityProto converToProtoFormat(
+ ProfileCapability profileCapability) {
+ ProfileCapabilityPBImpl tmp = new ProfileCapabilityPBImpl();
+ tmp.setProfileName(profileCapability.getProfileName());
+ tmp.setProfileCapabilityOverride(
+ profileCapability.getProfileCapabilityOverride());
+ return tmp.getProto();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 938e462..86cf872 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
@@ -51,6 +52,8 @@ public class ResourceUtils {
public static final String UNITS = ".units";
public static final String TYPE = ".type";
+ public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
+ public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
private static final String VCORES = ResourceInformation.VCORES.getName();
@@ -122,6 +125,86 @@ public class ResourceUtils {
}
}
+ private static void setMinimumAllocationForMandatoryResources(
+ Map<String, ResourceInformation> res, Configuration conf) {
+ String[][] resourceTypesKeys =
+ {
+ { ResourceInformation.MEMORY_MB.getName(),
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+ ResourceInformation.MEMORY_MB.getName()
+ },
+ { ResourceInformation.VCORES.getName(),
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
+ ResourceInformation.VCORES.getName()
+ }
+ };
+ for (String[] arr : resourceTypesKeys) {
+ String resourceTypesKey =
+ YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
+ long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
+ long minimumConf = conf.getLong(arr[1], -1);
+ long minimum;
+ if (minimumResourceTypes != -1) {
+ minimum = minimumResourceTypes;
+ if (minimumConf != -1) {
+ LOG.warn("Using minimum allocation for memory specified in "
+ + "resource-types config file with key "
+ + minimumResourceTypes + ", ignoring minimum specified using "
+ + arr[1]);
+ }
+ } else {
+ minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+ }
+ ResourceInformation ri = res.get(arr[3]);
+ ri.setMinimumAllocation(minimum);
+ }
+ }
+
+ private static void setMaximumAllocationForMandatoryResources(
+ Map<String, ResourceInformation> res, Configuration conf) {
+ String[][] resourceTypesKeys =
+ {
+ {
+ ResourceInformation.MEMORY_MB.getName(),
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
+ ResourceInformation.MEMORY_MB.getName()
+ },
+ {
+ ResourceInformation.VCORES.getName(),
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
+ ResourceInformation.VCORES.getName()
+ }
+ };
+ for (String[] arr : resourceTypesKeys) {
+ String resourceTypesKey =
+ YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
+ long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
+ long maximumConf = conf.getLong(arr[1], -1);
+ long maximum;
+ if (maximumResourceTypes != -1) {
+ maximum = maximumResourceTypes;
+ if (maximumConf != -1) {
+ LOG.warn("Using maximum allocation for memory specified in "
+ + "resource-types config file with key "
+ + maximumResourceTypes + ", ignoring maximum specified using "
+ + arr[1]);
+ }
+ } else {
+ maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+ }
+ ResourceInformation ri = res.get(arr[3]);
+ ri.setMaximumAllocation(maximum);
+ }
+ }
+
@VisibleForTesting
static void initializeResourcesMap(Configuration conf,
Map<String, ResourceInformation> resourceInformationMap) {
@@ -135,6 +218,12 @@ public class ResourceUtils {
String resourceTypeName = conf.get(
YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
ResourceTypes.COUNTABLE.toString());
+ Long minimumAllocation = conf.getLong(
+ YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+ + MINIMUM_ALLOCATION, 0L);
+ Long maximumAllocation = conf.getLong(
+ YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+ + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
if (resourceName == null || resourceName.isEmpty()
|| resourceUnits == null || resourceTypeName == null) {
throw new YarnRuntimeException(
@@ -154,11 +243,14 @@ public class ResourceUtils {
"Error in config, key '" + resourceName + "' specified twice");
}
resourceInformationMap.put(resourceName, ResourceInformation
- .newInstance(resourceName, resourceUnits, 0L, resourceType));
+ .newInstance(resourceName, resourceUnits, 0L, resourceType,
+ minimumAllocation, maximumAllocation));
}
}
checkMandatatoryResources(resourceInformationMap);
addManadtoryResources(resourceInformationMap);
+ setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
+ setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
}
@@ -173,6 +265,12 @@ public class ResourceUtils {
}
private static Map<String, ResourceInformation> getResourceTypes(
+ Configuration conf) {
+ return getResourceTypes(conf,
+ YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ }
+
+ private static Map<String, ResourceInformation> getResourceTypes(
Configuration conf, String resourceFile) {
if (lock == null) {
synchronized (ResourceUtils.class) {
@@ -205,6 +303,12 @@ public class ResourceUtils {
ConfigurationProvider provider =
ConfigurationProviderFactory.getConfigurationProvider(conf);
+ try {
+ provider.init(conf);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
if (ris == null) {
if (conf.getResource(resourceFile) == null) {
@@ -241,6 +345,12 @@ public class ResourceUtils {
lock = null;
}
+ @VisibleForTesting
+ public static void resetResourceTypes(Configuration conf) {
+ lock = null;
+ getResourceTypes(conf);
+ }
+
public static String getUnits(String resourceValue) {
String units;
for (int i = 0; i < resourceValue.length(); i++) {
@@ -326,4 +436,53 @@ public class ResourceUtils {
nodeLock = null;
}
+ public static Resource getResourceTypesMinimumAllocation() {
+ Map<String, ResourceInformation> resourceTypes = getResourceTypes();
+ Resource ret = Resource.newInstance(0, 0);
+ for (Map.Entry<String, ResourceInformation> entry : resourceTypes
+ .entrySet()) {
+ String name = entry.getKey();
+ if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+ ret.setMemorySize(entry.getValue().getMinimumAllocation());
+ continue;
+ }
+ if (name.equals(ResourceInformation.VCORES.getName())) {
+ Long tmp = entry.getValue().getMinimumAllocation();
+ if (tmp > Integer.MAX_VALUE) {
+ tmp = (long) Integer.MAX_VALUE;
+ }
+ ret.setVirtualCores(tmp.intValue());
+ continue;
+ }
+ ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
+ }
+ return ret;
+ }
+
+ /**
+ * Get a Resource object with for the maximum allocation possible.
+ * @return a Resource object with the maximum allocation for the scheduler
+ */
+ public static Resource getResourceTypesMaximumAllocation() {
+ Map<String, ResourceInformation> resourceTypes = getResourceTypes();
+ Resource ret = Resource.newInstance(0, 0);
+ for (Map.Entry<String, ResourceInformation> entry : resourceTypes
+ .entrySet()) {
+ String name = entry.getKey();
+ if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+ ret.setMemorySize(entry.getValue().getMaximumAllocation());
+ continue;
+ }
+ if (name.equals(ResourceInformation.VCORES.getName())) {
+ Long tmp = entry.getValue().getMaximumAllocation();
+ if (tmp > Integer.MAX_VALUE) {
+ tmp = (long) Integer.MAX_VALUE;
+ }
+ ret.setVirtualCores(tmp.intValue());
+ continue;
+ }
+ ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
+ }
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 63986da..c5e60b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -75,12 +75,12 @@ public class Resources {
}
@Override
+ @SuppressWarnings("deprecation")
public void setMemory(int memory) {
throw new RuntimeException(name + " cannot be modified!");
}
@Override
- @SuppressWarnings("deprecation")
public void setMemorySize(long memory) {
throw new RuntimeException(name + " cannot be modified!");
}
@@ -193,13 +193,7 @@ public class Resources {
}
public static Resource clone(Resource res) {
- Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : res.getResources()
- .entrySet()) {
- ret.setResourceInformation(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
- }
- return ret;
+ return Resource.newInstance(res);
}
public static Resource addTo(Resource lhs, Resource rhs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 3d7b2b1..d518641 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -120,6 +121,7 @@ public class ApplicationMasterService extends AbstractService implements
private final ConcurrentMap<ApplicationAttemptId, AllocateResponseLock> responseMap =
new ConcurrentHashMap<ApplicationAttemptId, AllocateResponseLock>();
protected final RMContext rmContext;
+ private ResourceProfilesManager resourceProfilesManager;
public ApplicationMasterService(String name, RMContext rmContext,
YarnScheduler scheduler) {
@@ -127,6 +129,7 @@ public class ApplicationMasterService extends AbstractService implements
this.amLivelinessMonitor = rmContext.getAMLivelinessMonitor();
this.rScheduler = scheduler;
this.rmContext = rmContext;
+ this.resourceProfilesManager = rmContext.getResourceProfilesManager();
}
public ApplicationMasterService(RMContext rmContext,
@@ -301,6 +304,12 @@ public class ApplicationMasterService extends AbstractService implements
response.setSchedulerResourceTypes(rScheduler
.getSchedulingResourceTypes());
+ if (getConfig().getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
+ response
+ .setResourceProfiles(resourceProfilesManager.getResourceProfiles());
+ }
+
return response;
}
}
@@ -455,6 +464,9 @@ public class ApplicationMasterService extends AbstractService implements
List<ResourceRequest> ask = request.getAskList();
List<ContainerId> release = request.getReleaseList();
+ RMServerUtils.convertProfileToResourceCapability(ask, getConfig(),
+ resourceProfilesManager);
+
ResourceBlacklistRequest blacklistRequest =
request.getResourceBlacklistRequest();
List<String> blacklistAdditions =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 74898ca..1dfe005 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
@@ -58,6 +60,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt
.RMAppAttemptState;
@@ -80,6 +83,8 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*/
public class RMServerUtils {
+ private static final Log LOG_HANDLE = LogFactory.getLog(RMServerUtils.class);
+
private static final String UPDATE_OUTSTANDING_ERROR =
"UPDATE_OUTSTANDING_ERROR";
private static final String INCORRECT_CONTAINER_VERSION_ERROR =
@@ -523,4 +528,49 @@ public class RMServerUtils {
}
return newApplicationTimeout;
}
+
+ public static void convertProfileToResourceCapability(
+ List<ResourceRequest> asks, Configuration conf,
+ ResourceProfilesManager resourceProfilesManager) throws YarnException {
+ boolean profilesEnabled =
+ conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+ if (!profilesEnabled) {
+ return;
+ }
+ for (ResourceRequest req : asks) {
+ convertProfileToResourceCapability(req, conf, resourceProfilesManager);
+ }
+ }
+
+ public static void convertProfileToResourceCapability(ResourceRequest ask,
+ Configuration conf, ResourceProfilesManager resourceProfilesManager)
+ throws YarnException {
+
+ if (LOG_HANDLE.isDebugEnabled()) {
+ LOG_HANDLE
+ .debug("Converting profile to resource capability for ask " + ask);
+ }
+
+ boolean profilesEnabled =
+ conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+ if (!profilesEnabled) {
+ if (ask.getProfileCapability() != null && !ask.getProfileCapability()
+ .getProfileCapabilityOverride().equals(Resources.none())) {
+ ask.setCapability(
+ ask.getProfileCapability().getProfileCapabilityOverride());
+ }
+ } else {
+ if (ask.getProfileCapability() != null) {
+ ask.setCapability(ProfileCapability
+ .toResource(ask.getProfileCapability(),
+ resourceProfilesManager.getResourceProfiles()));
+ }
+ }
+ if (LOG_HANDLE.isDebugEnabled()) {
+ LOG_HANDLE
+ .debug("Converted profile to resource capability for ask " + ask);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index 4bef333..15479e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -88,6 +88,10 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
while (iterator.hasNext()) {
Map.Entry entry = (Map.Entry) iterator.next();
String key = entry.getKey().toString();
+ if (key.isEmpty()) {
+ throw new IOException(
+ "Name of resource profile cannot be an empty string");
+ }
if (entry.getValue() instanceof Map) {
Map value = (Map) entry.getValue();
// ensure memory and vcores are specified
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index c0cc6b0..27bfe6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntit
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.SettableFuture;
@@ -1052,4 +1053,47 @@ public abstract class AbstractYarnScheduler
normalizeRequest(ask);
}
}
+
+ /**
+ * Get a Resource object with for the minimum allocation possible. If resource
+ * profiles are enabled then the 'minimum' resource profile will be used. If
+ * they are not enabled, use the minimums specified in the config files.
+ *
+ * @return a Resource object with the minimum allocation for the scheduler
+ */
+ public Resource getMinimumAllocation() {
+ boolean profilesEnabled = getConfig()
+ .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+ Resource ret;
+ if (!profilesEnabled) {
+ ret = ResourceUtils.getResourceTypesMinimumAllocation();
+ } else {
+ ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+ }
+ LOG.info("Minimum allocation = " + ret);
+ return ret;
+ }
+
+ /**
+ * Get a Resource object with for the maximum allocation possible. If resource
+ * profiles are enabled then the 'maximum' resource profile will be used. If
+ * they are not enabled, use the maximums specified in the config files.
+ *
+ * @return a Resource object with the maximum allocation for the scheduler
+ */
+
+ public Resource getMaximumAllocation() {
+ boolean profilesEnabled = getConfig()
+ .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
+ Resource ret;
+ if (!profilesEnabled) {
+ ret = ResourceUtils.getResourceTypesMaximumAllocation();
+ } else {
+ ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+ }
+ LOG.info("Maximum allocation = " + ret);
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index e487f69..769a9e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -221,8 +221,7 @@ public class ClusterNodeTracker<N extends SchedulerNode> {
return Resources.createResource(
Math.min(configuredMaxAllocation.getMemorySize(), maxNodeMemory),
- Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores)
- );
+ Math.min(configuredMaxAllocation.getVirtualCores(), maxNodeVCores));
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 6f905b9..05a3359 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -36,10 +36,12 @@ import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -233,6 +235,14 @@ public class SchedulerUtils {
private static void validateResourceRequest(ResourceRequest resReq,
Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
+ try {
+ RMServerUtils.convertProfileToResourceCapability(resReq,
+ rmContext.getYarnConfiguration(),
+ rmContext.getResourceProfilesManager());
+ } catch (YarnException ye) {
+ throw new InvalidResourceRequestException(ye);
+ }
+
if (resReq.getCapability().getMemorySize() < 0 ||
resReq.getCapability().getMemorySize() > maximumResource.getMemorySize()) {
throw new InvalidResourceRequestException("Invalid resource request"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 9a73a65..90d8870 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -299,8 +299,8 @@ public class CapacityScheduler extends
writeLock.lock();
this.conf = loadCapacitySchedulerConfiguration(configuration);
validateConf(this.conf);
- this.minimumAllocation = this.conf.getMinimumAllocation();
- initMaximumResourceCapability(this.conf.getMaximumAllocation());
+ this.minimumAllocation = super.getMinimumAllocation();
+ initMaximumResourceCapability(super.getMaximumAllocation());
this.calculator = this.conf.getResourceCalculator();
this.usePortForNodeName = this.conf.getUsePortForNodeName();
this.applications = new ConcurrentHashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index e790bc2..9e516e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1225,8 +1225,8 @@ public class FairScheduler extends
this.conf = new FairSchedulerConfiguration(conf);
validateConf(this.conf);
authorizer = YarnAuthorizationProvider.getInstance(conf);
- minimumAllocation = this.conf.getMinimumAllocation();
- initMaximumResourceCapability(this.conf.getMaximumAllocation());
+ minimumAllocation = super.getMinimumAllocation();
+ initMaximumResourceCapability(super.getMaximumAllocation());
incrAllocation = this.conf.getIncrementAllocation();
updateReservationThreshold();
continuousSchedulingEnabled = this.conf.isContinuousSchedulingEnabled();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index b5122c0..9a8e62e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -239,17 +239,8 @@ public class FifoScheduler extends
//Use ConcurrentSkipListMap because applications need to be ordered
this.applications =
new ConcurrentSkipListMap<>();
- this.minimumAllocation =
- Resources.createResource(conf.getInt(
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
- initMaximumResourceCapability(
- Resources.createResource(conf.getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
- conf.getInt(
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
+ this.minimumAllocation = super.getMinimumAllocation();
+ initMaximumResourceCapability(super.getMaximumAllocation());
this.usePortForNodeName = conf.getBoolean(
YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index a66b093..f62c15b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -104,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSe
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -150,6 +151,7 @@ public class MockRM extends ResourceManager {
public MockRM(Configuration conf, RMStateStore store,
boolean useNullRMNodeLabelsManager, boolean useRealElector) {
super();
+ ResourceUtils.resetResourceTypes(conf);
this.useNullRMNodeLabelsManager = useNullRMNodeLabelsManager;
this.useRealElector = useRealElector;
init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 03cc3a0..c8655fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -238,6 +238,7 @@ public class TestAppManager{
ResourceScheduler scheduler = mockResourceScheduler();
((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
+ ((RMContextImpl) rmContext).setYarnConfiguration(conf);
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
appMonitor = new TestRMAppManager(rmContext,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index 00466ae..0d1db7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -492,4 +493,38 @@ public class TestApplicationMasterService {
Assert.fail("Cannot find RMContainer");
}
}
+
+ @Test(timeout = 3000000)
+ public void testResourceProfiles() throws Exception {
+
+ MockRM rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+ RMApp app1 = rm.submitApp(2048);
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
+ Assert.assertEquals(0, resp.getResourceProfiles().size());
+ rm.stop();
+ conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
+ conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
+ "profiles/sample-profiles-1.json");
+ rm = new MockRM(conf);
+ rm.start();
+ nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
+ app1 = rm.submitApp(2048);
+ nm1.nodeHeartbeat(true);
+ attempt1 = app1.getCurrentAppAttempt();
+ am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+ resp = am1.registerAppAttempt();
+ Assert.assertEquals(3, resp.getResourceProfiles().size());
+ Assert.assertEquals(Resource.newInstance(1024, 1),
+ resp.getResourceProfiles().get("minimum"));
+ Assert.assertEquals(Resource.newInstance(2048, 2),
+ resp.getResourceProfiles().get("default"));
+ Assert.assertEquals(Resource.newInstance(4096, 4),
+ resp.getResourceProfiles().get("maximum"));
+ rm.stop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 5aa1e2d..4fe22c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -109,6 +109,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Assert;
@@ -208,6 +209,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
128);
+ ResourceUtils.resetResourceTypes(conf);
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -236,6 +238,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
conf.setInt(
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+ ResourceUtils.resetResourceTypes(conf);
scheduler.init(conf);
scheduler.reinitialize(conf, null);
Assert.assertEquals(256, scheduler.getMinimumResourceCapability().getMemorySize());
@@ -253,6 +256,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 512);
conf.setInt(
FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES, 2);
+ ResourceUtils.resetResourceTypes(conf);
scheduler.init(conf);
scheduler.reinitialize(conf, null);
Assert.assertEquals(0, scheduler.getMinimumResourceCapability().getMemorySize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 4f484ad..294c867 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -92,6 +92,7 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -247,6 +248,7 @@ public class MiniYARNCluster extends CompositeService {
YarnConfiguration.DEFAULT_YARN_MINICLUSTER_USE_RPC);
failoverTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
+ ResourceUtils.resetResourceTypes(conf);
if (useRpc && !useFixedPorts) {
throw new YarnRuntimeException("Invalid configuration!" +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org