You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by te...@apache.org on 2017/08/04 16:14:30 UTC
[1/2] hadoop git commit: YARN-6788. [YARN-3926] Improve performance
of resource profile branch (Contributed by Sunil Govindan via Daniel
Templeton)
Repository: hadoop
Updated Branches:
refs/heads/YARN-3926 419bb55cf -> 1b586d700
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
deleted file mode 100644
index 86cf872..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util.resource;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.ConfigurationProvider;
-import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Helper class to read the resource-types to be supported by the system.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ResourceUtils {
-
- public static final String UNITS = ".units";
- public static final String TYPE = ".type";
- public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
- public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
-
- private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
- private static final String VCORES = ResourceInformation.VCORES.getName();
-
- private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
- static {
- DISALLOWED_NAMES.add("memory");
- DISALLOWED_NAMES.add(MEMORY);
- DISALLOWED_NAMES.add(VCORES);
- }
-
- private static volatile Object lock;
- private static Map<String, ResourceInformation> readOnlyResources;
- private static volatile Object nodeLock;
- private static Map<String, ResourceInformation> readOnlyNodeResources;
-
-
- static final Log LOG = LogFactory.getLog(ResourceUtils.class);
-
- private ResourceUtils() {
- }
-
- private static void checkMandatatoryResources(
- Map<String, ResourceInformation> resourceInformationMap)
- throws YarnRuntimeException {
- if (resourceInformationMap.containsKey(MEMORY)) {
- ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
- String memUnits = ResourceInformation.MEMORY_MB.getUnits();
- ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
- if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
- .equals(memType)) {
- throw new YarnRuntimeException(
- "Attempt to re-define mandatory resource 'memory-mb'. It can only"
- + " be of type 'COUNTABLE' and have units 'Mi'.");
- }
- }
-
- if (resourceInformationMap.containsKey(VCORES)) {
- ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
- String vcoreUnits = ResourceInformation.VCORES.getUnits();
- ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
- if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
- .getResourceType().equals(vcoreType)) {
- throw new YarnRuntimeException(
- "Attempt to re-define mandatory resource 'vcores'. It can only be"
- + " of type 'COUNTABLE' and have units ''(no units).");
- }
- }
- }
-
- private static void addManadtoryResources(
- Map<String, ResourceInformation> res) {
- ResourceInformation ri;
- if (!res.containsKey(MEMORY)) {
- LOG.info("Adding resource type - name = " + MEMORY + ", units = "
- + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
- + ResourceTypes.COUNTABLE);
- ri = ResourceInformation
- .newInstance(MEMORY,
- ResourceInformation.MEMORY_MB.getUnits());
- res.put(MEMORY, ri);
- }
- if (!res.containsKey(VCORES)) {
- LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
- + ResourceTypes.COUNTABLE);
- ri =
- ResourceInformation.newInstance(VCORES);
- res.put(VCORES, ri);
- }
- }
-
- private static void setMinimumAllocationForMandatoryResources(
- Map<String, ResourceInformation> res, Configuration conf) {
- String[][] resourceTypesKeys =
- {
- { ResourceInformation.MEMORY_MB.getName(),
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
- ResourceInformation.MEMORY_MB.getName()
- },
- { ResourceInformation.VCORES.getName(),
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
- ResourceInformation.VCORES.getName()
- }
- };
- for (String[] arr : resourceTypesKeys) {
- String resourceTypesKey =
- YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
- long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
- long minimumConf = conf.getLong(arr[1], -1);
- long minimum;
- if (minimumResourceTypes != -1) {
- minimum = minimumResourceTypes;
- if (minimumConf != -1) {
- LOG.warn("Using minimum allocation for memory specified in "
- + "resource-types config file with key "
- + minimumResourceTypes + ", ignoring minimum specified using "
- + arr[1]);
- }
- } else {
- minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
- }
- ResourceInformation ri = res.get(arr[3]);
- ri.setMinimumAllocation(minimum);
- }
- }
-
- private static void setMaximumAllocationForMandatoryResources(
- Map<String, ResourceInformation> res, Configuration conf) {
- String[][] resourceTypesKeys =
- {
- {
- ResourceInformation.MEMORY_MB.getName(),
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
- ResourceInformation.MEMORY_MB.getName()
- },
- {
- ResourceInformation.VCORES.getName(),
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
- ResourceInformation.VCORES.getName()
- }
- };
- for (String[] arr : resourceTypesKeys) {
- String resourceTypesKey =
- YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
- long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
- long maximumConf = conf.getLong(arr[1], -1);
- long maximum;
- if (maximumResourceTypes != -1) {
- maximum = maximumResourceTypes;
- if (maximumConf != -1) {
- LOG.warn("Using maximum allocation for memory specified in "
- + "resource-types config file with key "
- + maximumResourceTypes + ", ignoring maximum specified using "
- + arr[1]);
- }
- } else {
- maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
- }
- ResourceInformation ri = res.get(arr[3]);
- ri.setMaximumAllocation(maximum);
- }
- }
-
- @VisibleForTesting
- static void initializeResourcesMap(Configuration conf,
- Map<String, ResourceInformation> resourceInformationMap) {
-
- String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
-
- if (resourceNames != null && resourceNames.length != 0) {
- for (String resourceName : resourceNames) {
- String resourceUnits = conf.get(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
- String resourceTypeName = conf.get(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
- ResourceTypes.COUNTABLE.toString());
- Long minimumAllocation = conf.getLong(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName
- + MINIMUM_ALLOCATION, 0L);
- Long maximumAllocation = conf.getLong(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName
- + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
- if (resourceName == null || resourceName.isEmpty()
- || resourceUnits == null || resourceTypeName == null) {
- throw new YarnRuntimeException(
- "Incomplete configuration for resource type '" + resourceName
- + "'. One of name, units or type is configured incorrectly.");
- }
- if (DISALLOWED_NAMES.contains(resourceName)) {
- throw new YarnRuntimeException(
- "Resource type cannot be named '" + resourceName
- + "'. That name is disallowed.");
- }
- ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
- LOG.info("Adding resource type - name = " + resourceName + ", units = "
- + resourceUnits + ", type = " + resourceTypeName);
- if (resourceInformationMap.containsKey(resourceName)) {
- throw new YarnRuntimeException(
- "Error in config, key '" + resourceName + "' specified twice");
- }
- resourceInformationMap.put(resourceName, ResourceInformation
- .newInstance(resourceName, resourceUnits, 0L, resourceType,
- minimumAllocation, maximumAllocation));
- }
- }
- checkMandatatoryResources(resourceInformationMap);
- addManadtoryResources(resourceInformationMap);
- setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
- setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
- readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
- }
-
- /**
- * Get the resource types to be supported by the system.
- * @return A map of the resource name to a ResouceInformation object
- * which contains details such as the unit.
- */
- public static Map<String, ResourceInformation> getResourceTypes() {
- return getResourceTypes(null,
- YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
- }
-
- private static Map<String, ResourceInformation> getResourceTypes(
- Configuration conf) {
- return getResourceTypes(conf,
- YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
- }
-
- private static Map<String, ResourceInformation> getResourceTypes(
- Configuration conf, String resourceFile) {
- if (lock == null) {
- synchronized (ResourceUtils.class) {
- if (lock == null) {
- synchronized (ResourceUtils.class) {
- Map<String, ResourceInformation> resources = new HashMap<>();
- if (conf == null) {
- conf = new YarnConfiguration();
- }
- try {
- addResourcesFileToConf(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
- initializeResourcesMap(conf, resources);
- lock = new Object();
- } catch (FileNotFoundException fe) {
- LOG.info("Unable to find '" + resourceFile
- + "'. Falling back to memory and vcores as resources", fe);
- initializeResourcesMap(conf, resources);
- lock = new Object();
- }
- }
- }
- }
- }
- return readOnlyResources;
- }
-
- private static InputStream getConfInputStream(String resourceFile,
- Configuration conf) throws IOException, YarnException {
-
- ConfigurationProvider provider =
- ConfigurationProviderFactory.getConfigurationProvider(conf);
- try {
- provider.init(conf);
- } catch (Exception e) {
- throw new IOException(e);
- }
-
- InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
- if (ris == null) {
- if (conf.getResource(resourceFile) == null) {
- throw new FileNotFoundException("Unable to find " + resourceFile);
- }
- throw new IOException(
- "Unable to open resource types file '" + resourceFile
- + "'. Using provider " + provider);
- }
- return ris;
- }
-
- private static void addResourcesFileToConf(String resourceFile,
- Configuration conf) throws FileNotFoundException {
- try {
- InputStream ris = getConfInputStream(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
- conf.addResource(ris);
- } catch (FileNotFoundException fe) {
- throw fe;
- } catch (IOException ie) {
- LOG.fatal("Exception trying to read resource types configuration '"
- + resourceFile + "'.", ie);
- throw new YarnRuntimeException(ie);
- } catch (YarnException ye) {
- LOG.fatal("YARN Exception trying to read resource types configuration '"
- + resourceFile + "'.", ye);
- throw new YarnRuntimeException(ye);
- }
- }
-
- @VisibleForTesting
- static void resetResourceTypes() {
- lock = null;
- }
-
- @VisibleForTesting
- public static void resetResourceTypes(Configuration conf) {
- lock = null;
- getResourceTypes(conf);
- }
-
- public static String getUnits(String resourceValue) {
- String units;
- for (int i = 0; i < resourceValue.length(); i++) {
- if (Character.isAlphabetic(resourceValue.charAt(i))) {
- units = resourceValue.substring(i);
- if (StringUtils.isAlpha(units)) {
- return units;
- }
- }
- }
- return "";
- }
-
- /**
- * Function to get the resources for a node. This function will look at the
- * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to
- * determine the node resources.
- *
- * @param conf configuration file
- * @return a map to resource name to the ResourceInformation object. The map
- * is guaranteed to have entries for memory and vcores
- */
- public static Map<String, ResourceInformation> getNodeResourceInformation(
- Configuration conf) {
- if (nodeLock == null) {
- synchronized (ResourceUtils.class) {
- if (nodeLock == null) {
- synchronized (ResourceUtils.class) {
- Map<String, ResourceInformation> nodeResources =
- initializeNodeResourceInformation(conf);
- addManadtoryResources(nodeResources);
- checkMandatatoryResources(nodeResources);
- readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
- nodeLock = new Object();
- }
- }
- }
- }
- return readOnlyNodeResources;
- }
-
- private static Map<String, ResourceInformation>
- initializeNodeResourceInformation(Configuration conf) {
- Map<String, ResourceInformation> nodeResources = new HashMap<>();
- try {
- addResourcesFileToConf(
- YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
- for (Map.Entry<String, String> entry : conf) {
- String key = entry.getKey();
- String value = entry.getValue();
- if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
- addResourceInformation(key, value, nodeResources);
- }
- }
- } catch (FileNotFoundException fe) {
- LOG.info("Couldn't find node resources file");
- }
- return nodeResources;
- }
-
- private static void addResourceInformation(String prop, String value,
- Map<String, ResourceInformation> nodeResources) {
- String[] parts = prop.split("\\.");
- LOG.info("Found resource entry " + prop);
- if (parts.length == 4) {
- String resourceType = parts[3];
- if (!nodeResources.containsKey(resourceType)) {
- nodeResources
- .put(resourceType, ResourceInformation.newInstance(resourceType));
- }
- String units = getUnits(value);
- Long resourceValue =
- Long.valueOf(value.substring(0, value.length() - units.length()));
- nodeResources.get(resourceType).setValue(resourceValue);
- nodeResources.get(resourceType).setUnits(units);
- LOG.debug("Setting value for resource type " + resourceType + " to "
- + resourceValue + " with units " + units);
- }
- }
-
- @VisibleForTesting
- synchronized public static void resetNodeResources() {
- nodeLock = null;
- }
-
- public static Resource getResourceTypesMinimumAllocation() {
- Map<String, ResourceInformation> resourceTypes = getResourceTypes();
- Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : resourceTypes
- .entrySet()) {
- String name = entry.getKey();
- if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
- ret.setMemorySize(entry.getValue().getMinimumAllocation());
- continue;
- }
- if (name.equals(ResourceInformation.VCORES.getName())) {
- Long tmp = entry.getValue().getMinimumAllocation();
- if (tmp > Integer.MAX_VALUE) {
- tmp = (long) Integer.MAX_VALUE;
- }
- ret.setVirtualCores(tmp.intValue());
- continue;
- }
- ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
- }
- return ret;
- }
-
- /**
- * Get a Resource object with for the maximum allocation possible.
- * @return a Resource object with the maximum allocation for the scheduler
- */
- public static Resource getResourceTypesMaximumAllocation() {
- Map<String, ResourceInformation> resourceTypes = getResourceTypes();
- Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : resourceTypes
- .entrySet()) {
- String name = entry.getKey();
- if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
- ret.setMemorySize(entry.getValue().getMaximumAllocation());
- continue;
- }
- if (name.equals(ResourceInformation.VCORES.getName())) {
- Long tmp = entry.getValue().getMaximumAllocation();
- if (tmp > Integer.MAX_VALUE) {
- tmp = (long) Integer.MAX_VALUE;
- }
- ret.setVirtualCores(tmp.intValue());
- continue;
- }
- ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index d143e93..f62114d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -18,30 +18,31 @@
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Arrays;
@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" })
@Unstable
public class Resources {
+ private static final Log LOG =
+ LogFactory.getLog(Resources.class);
+
/**
* Helper class to create a resource with a fixed value for all resource
* types. For example, a NONE resource which returns 0 for any resource type.
*/
- static class FixedValueResource extends Resource {
+ static class FixedValueResource extends BaseResource {
- private Map<String, ResourceInformation> resources;
private long resourceValue;
private String name;
@@ -53,7 +54,7 @@ public class Resources {
FixedValueResource(String rName, long value) {
this.resourceValue = value;
this.name = rName;
- resources = initResourceMap();
+ initResourceMap();
}
private int resourceValueToInt() {
@@ -96,31 +97,6 @@ public class Resources {
}
@Override
- public Map<String, ResourceInformation> getResources() {
- return Collections.unmodifiableMap(this.resources);
- }
-
- @Override
- public ResourceInformation getResourceInformation(String resource)
- throws YarnException {
- if (resources.containsKey(resource)) {
- ResourceInformation value = this.resources.get(resource);
- ResourceInformation ret = ResourceInformation.newInstance(value);
- ret.setValue(resourceValue);
- return ret;
- }
- throw new YarnException("" + resource + " not found");
- }
-
- @Override
- public Long getResourceValue(String resource) throws YarnException {
- if (resources.containsKey(resource)) {
- return resourceValue;
- }
- throw new YarnException("" + resource + " not found");
- }
-
- @Override
public void setResourceInformation(String resource,
ResourceInformation resourceInformation)
throws ResourceNotFoundException {
@@ -133,24 +109,24 @@ public class Resources {
throw new RuntimeException(name + " cannot be modified!");
}
- private Map<String, ResourceInformation> initResourceMap() {
- Map<String, ResourceInformation> tmp = new HashMap<>();
- Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+ private void initResourceMap() {
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
if (types != null) {
- for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
- tmp.put(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
- tmp.get(entry.getKey()).setValue(resourceValue);
+ resources = new ResourceInformation[types.length];
+ readOnlyResources = new ResourceInformation[types.length];
+ for (int index = 0; index < types.length; index++) {
+ resources[index] = ResourceInformation.newInstance(types[index]);
+ resources[index].setValue(resourceValue);
+
+ // this is a fix for getVirtualCores returning an int
+ if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES
+ .getName().equals(resources[index].getName())) {
+ resources[index].setValue((long) Integer.MAX_VALUE);
+ }
}
}
- // this is a fix for getVirtualCores returning an int
- if (resourceValue > Integer.MAX_VALUE) {
- tmp.get(ResourceInformation.VCORES.getName())
- .setValue((long) Integer.MAX_VALUE);
- }
- return tmp;
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
}
-
}
public static Resource createResource(int memory) {
@@ -197,17 +173,19 @@ public class Resources {
}
public static Resource addTo(Resource lhs, Resource rhs) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -219,17 +197,19 @@ public class Resources {
}
public static Resource subtractFrom(Resource lhs, Resource rhs) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -263,10 +243,9 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
- ResourceInformation lhsValue = entry.getValue();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
+ ResourceInformation lhsValue = entry;
lhs.setResourceValue(name, (long) (lhsValue.getValue() * by));
}
return lhs;
@@ -282,17 +261,21 @@ public class Resources {
*/
public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = (long) (UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue()) * by);
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (long) (((rhsValue.getUnits()
+ .equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue()))
+ * by);
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -311,10 +294,9 @@ public class Resources {
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- for (Map.Entry<String, ResourceInformation> entry : out.getResources()
- .entrySet()) {
- String name = entry.getKey();
- ResourceInformation lhsValue = entry.getValue();
+ for (ResourceInformation entry : out.getResources()) {
+ String name = entry.getName();
+ ResourceInformation lhsValue = entry;
out.setResourceValue(name, (long) (lhsValue.getValue() * by));
}
return out;
@@ -416,19 +398,21 @@ public class Resources {
}
public static boolean fitsIn(Resource smaller, Resource bigger) {
- for (Map.Entry<String, ResourceInformation> entry : smaller.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : smaller.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = bigger.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
if(lhsValue.getValue() > convertedRhs) {
return false;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
return false;
}
}
@@ -442,19 +426,21 @@ public class Resources {
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
ResourceInformation outInfo =
lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue;
ret.setResourceInformation(name, outInfo);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -463,19 +449,21 @@ public class Resources {
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
ResourceInformation outInfo =
lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue;
ret.setResourceInformation(name, outInfo);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 38554b6..b530150 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -276,13 +276,17 @@ public class TestResourceUtils {
String resourceFile = entry.getKey();
ResourceUtils.resetNodeResources();
File dest;
- File source =
- new File(conf.getClassLoader().getResource(resourceFile).getFile());
+ File source = new File(
+ conf.getClassLoader().getResource(resourceFile).getFile());
dest = new File(source.getParent(), "node-resources.xml");
FileUtils.copyFile(source, dest);
- Map<String, ResourceInformation> actual =
- ResourceUtils.getNodeResourceInformation(conf);
- Assert.assertEquals(entry.getValue().getResources(), actual);
+ Map<String, ResourceInformation> actual = ResourceUtils
+ .getNodeResourceInformation(conf);
+ Assert.assertEquals(actual.size(),
+ entry.getValue().getResources().length);
+ for (ResourceInformation resInfo : entry.getValue().getResources()) {
+ Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+ }
dest.delete();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 1555e55..a8404fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -105,7 +105,7 @@ public class TestResources {
unboundedClone.compareTo(createResource(0, Integer.MAX_VALUE)) > 0);
}
- @Test(timeout=10000)
+ @Test(timeout = 10000)
public void testCompareToWithNoneResource() {
assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0);
assertTrue(Resources.none().compareTo(createResource(1, 0)) < 0);
@@ -114,7 +114,6 @@ public class TestResources {
assertTrue(Resources.none().compareTo(createResource(1, 0, 0)) < 0);
assertTrue(Resources.none().compareTo(createResource(0, 1, 0)) < 0);
assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
- assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
}
@Test(timeout=10000)
@@ -246,7 +245,9 @@ public class TestResources {
}
@Test
- public void testMultiplyAndAddTo() {
+ public void testMultiplyAndAddTo() throws Exception {
+ unsetExtraResourceType();
+ setupExtraResourceType();
assertEquals(createResource(6, 4),
multiplyAndAddTo(createResource(3, 1), createResource(2, 2), 1.5));
assertEquals(createResource(6, 4, 0),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index 7987ded..ab33336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -119,13 +119,13 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
private Resource parseResource(String key, Map value) throws IOException {
Resource resource = Resource.newInstance(0, 0);
Iterator iterator = value.entrySet().iterator();
- Map<String, ResourceInformation> resourceTypes =
- ResourceUtils.getResourceTypes();
+ Map<String, ResourceInformation> resourceTypes = ResourceUtils
+ .getResourceTypes();
while (iterator.hasNext()) {
Map.Entry resourceEntry = (Map.Entry) iterator.next();
String resourceName = resourceEntry.getKey().toString();
- ResourceInformation resourceValue =
- fromString(resourceName, resourceEntry.getValue().toString());
+ ResourceInformation resourceValue = fromString(resourceName,
+ resourceEntry.getValue().toString());
if (resourceName.equals(MEMORY)) {
resource.setMemorySize(resourceValue.getValue());
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index ff18223..c514cb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -176,16 +176,15 @@ public class RMAppAttemptMetrics {
private void updateUsageMap(Resource allocated, long deltaUsedMillis,
Map<String, AtomicLong> targetMap) {
- for (Map.Entry<String, ResourceInformation> entry : allocated.getResources()
- .entrySet()) {
+ for (ResourceInformation entry : allocated.getResources()) {
AtomicLong resourceUsed;
- if (!targetMap.containsKey(entry.getKey())) {
+ if (!targetMap.containsKey(entry.getName())) {
resourceUsed = new AtomicLong(0);
- targetMap.put(entry.getKey(), resourceUsed);
+ targetMap.put(entry.getName(), resourceUsed);
}
- resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis)
+ resourceUsed = targetMap.get(entry.getName());
+ resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis)
/ DateUtils.MILLIS_PER_SECOND);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 811d9dc..b571322 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -984,13 +984,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
for (RMContainer rmContainer : this.liveContainers.values()) {
long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
Resource resource = rmContainer.getContainer().getResource();
- for (Map.Entry<String, ResourceInformation> entry : resource
- .getResources().entrySet()) {
+ for (ResourceInformation entry : resource.getResources()) {
long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue().getValue() * usedMillis
+ .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
+ value += entry.getValue() * usedMillis
/ DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getKey(), value);
+ resourceSecondsMap.put(entry.getName(), value);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 887b854..81491b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import java.util.Arrays;
import java.util.EnumSet;
import javax.xml.bind.annotation.XmlRootElement;
@@ -73,7 +74,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
- return minAllocResource.getResource().getResources().keySet().toString();
+ return Arrays.toString(minAllocResource.getResource().getResources());
}
public int getMaxClusterLevelAppPriority() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8ae630b..b24a309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -249,6 +249,7 @@ public class TestAppManager{
ResourceScheduler scheduler = mockResourceScheduler();
((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
((RMContextImpl) rmContext).setYarnConfiguration(conf);
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-6788. [YARN-3926] Improve performance
of resource profile branch (Contributed by Sunil Govindan via Daniel
Templeton)
Posted by te...@apache.org.
YARN-6788. [YARN-3926] Improve performance of resource profile branch
(Contributed by Sunil Govindan via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b586d70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b586d70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b586d70
Branch: refs/heads/YARN-3926
Commit: 1b586d7001fb3a46fd9c9e1153db9cdd63999ff1
Parents: 419bb55
Author: Daniel Templeton <te...@apache.org>
Authored: Fri Aug 4 08:42:34 2017 -0700
Committer: Daniel Templeton <te...@apache.org>
Committed: Fri Aug 4 08:42:34 2017 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 18 +
.../yarn/api/records/ProfileCapability.java | 8 +-
.../hadoop/yarn/api/records/Resource.java | 234 ++++----
.../yarn/api/records/ResourceInformation.java | 13 +-
.../yarn/api/records/impl/BaseResource.java | 133 +++++
.../yarn/api/records/impl/package-info.java | 22 +
.../hadoop/yarn/util/UnitsConversionUtil.java | 8 +-
.../yarn/util/resource/ResourceUtils.java | 534 +++++++++++++++++++
.../hadoop/yarn/util/resource/package-info.java | 22 +
.../yarn/api/records/impl/pb/ProtoUtils.java | 5 +-
.../api/records/impl/pb/ResourcePBImpl.java | 110 ++--
.../resource/DominantResourceCalculator.java | 67 ++-
.../yarn/util/resource/ResourceUtils.java | 488 -----------------
.../hadoop/yarn/util/resource/Resources.java | 194 ++++---
.../yarn/util/resource/TestResourceUtils.java | 14 +-
.../yarn/util/resource/TestResources.java | 7 +-
.../resource/ResourceProfilesManagerImpl.java | 8 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 11 +-
.../scheduler/SchedulerApplicationAttempt.java | 9 +-
.../webapp/dao/SchedulerInfo.java | 3 +-
.../server/resourcemanager/TestAppManager.java | 1 +
21 files changed, 1039 insertions(+), 870 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 576db93..a2de624 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -583,4 +583,22 @@
<Bug pattern="UL_UNRELEASED_LOCK_EXCEPTION_PATH" />
</Match>
+ <!-- Ignore MS_EXPOSE_REP -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
+ <Method name="getResourceTypesArray" />
+ <Bug pattern="MS_EXPOSE_REP" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
+ <Method name="getResourceNamesArray" />
+ <Bug pattern="MS_EXPOSE_REP" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.api.records.impl.BaseResource" />
+ <Method name="getResources" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
index 1a8d1c3..2cb4670 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java
@@ -162,10 +162,10 @@ public abstract class ProfileCapability {
if (capability.getProfileCapabilityOverride() != null &&
!capability.getProfileCapabilityOverride().equals(none)) {
- for (Map.Entry<String, ResourceInformation> entry : capability
- .getProfileCapabilityOverride().getResources().entrySet()) {
- if (entry.getValue() != null && entry.getValue().getValue() >= 0) {
- resource.setResourceInformation(entry.getKey(), entry.getValue());
+ for (ResourceInformation entry : capability
+ .getProfileCapabilityOverride().getResources()) {
+ if (entry != null && entry.getValue() >= 0) {
+ resource.setResourceInformation(entry.getName(), entry);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 9a8e2ec..a485a57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.Arrays;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -25,13 +27,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Records;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
* <p><code>Resource</code> models a set of computer resources in the
@@ -60,97 +59,49 @@ import java.util.Map;
@Stable
public abstract class Resource implements Comparable<Resource> {
- private static Resource tmpResource = Records.newRecord(Resource.class);
-
- private static class SimpleResource extends Resource {
- private long memory;
- private long vcores;
- private Map<String, ResourceInformation> resourceInformationMap;
-
- SimpleResource(long memory, long vcores) {
- this.memory = memory;
- this.vcores = vcores;
-
- }
- @Override
- public int getMemory() {
- return (int)memory;
- }
- @Override
- public void setMemory(int memory) {
- this.memory = memory;
- }
- @Override
- public long getMemorySize() {
- return memory;
- }
- @Override
- public void setMemorySize(long memory) {
- this.memory = memory;
- }
- @Override
- public int getVirtualCores() {
- return (int)vcores;
- }
- @Override
- public void setVirtualCores(int vcores) {
- this.vcores = vcores;
- }
- @Override
- public Map<String, ResourceInformation> getResources() {
- if (resourceInformationMap == null) {
- resourceInformationMap = new HashMap<>();
- resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(),
- ResourceInformation.newInstance(ResourceInformation.MEMORY_MB));
- resourceInformationMap.put(ResourceInformation.VCORES.getName(),
- ResourceInformation.newInstance(ResourceInformation.VCORES));
- }
- resourceInformationMap.get(ResourceInformation.MEMORY_MB.getName())
- .setValue(this.memory);
- resourceInformationMap.get(ResourceInformation.VCORES.getName())
- .setValue(this.vcores);
- return Collections.unmodifiableMap(resourceInformationMap);
- }
- }
+ protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
+ protected static final String VCORES = ResourceInformation.VCORES.getName();
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
- if (tmpResource.getResources().size() > 2) {
+ if (ResourceUtils.getResourceTypesArray().length > 2) {
Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory);
ret.setVirtualCores(vCores);
return ret;
}
- return new SimpleResource(memory, vCores);
+ return new BaseResource(memory, vCores);
}
@Public
@Stable
public static Resource newInstance(long memory, int vCores) {
- if (tmpResource.getResources().size() > 2) {
+ if (ResourceUtils.getResourceTypesArray().length > 2) {
Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory);
ret.setVirtualCores(vCores);
return ret;
}
- return new SimpleResource(memory, vCores);
+ return new BaseResource(memory, vCores);
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static Resource newInstance(Resource resource) {
- Resource ret = Resource.newInstance(0, 0);
- Resource.copy(resource, ret);
+ Resource ret = Resource.newInstance(resource.getMemorySize(),
+ resource.getVirtualCores());
+ if (ResourceUtils.getResourceTypesArray().length > 2) {
+ Resource.copy(resource, ret);
+ }
return ret;
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static void copy(Resource source, Resource dest) {
- for (Map.Entry<String, ResourceInformation> entry : source.getResources()
- .entrySet()) {
- dest.setResourceInformation(entry.getKey(), entry.getValue());
+ for (ResourceInformation entry : source.getResources()) {
+ dest.setResourceInformation(entry.getName(), entry);
}
}
@@ -251,25 +202,26 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract Map<String, ResourceInformation> getResources();
+ public abstract ResourceInformation[] getResources();
/**
* Get ResourceInformation for a specified resource.
*
* @param resource name of the resource
* @return the ResourceInformation object for the resource
- * @throws YarnException if the resource can't be found
+ * @throws ResourceNotFoundException if the resource can't be found
*/
@Public
@Evolving
public ResourceInformation getResourceInformation(String resource)
- throws YarnException {
- if (getResources().containsKey(resource)) {
- return getResources().get(resource);
+ throws ResourceNotFoundException {
+ Integer index = ResourceUtils.getResourceTypeIndex().get(resource);
+ ResourceInformation[] resources = getResources();
+ if (index != null) {
+ return resources[index];
}
- throw new YarnException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+ throw new ResourceNotFoundException("Unknown resource '" + resource
+ + "'. Known resources are " + Arrays.toString(resources));
}
/**
@@ -278,17 +230,13 @@ public abstract class Resource implements Comparable<Resource> {
*
* @param resource name of the resource
* @return the value for the resource
- * @throws YarnException if the resource can't be found
+ * @throws ResourceNotFoundException if the resource can't be found
*/
@Public
@Evolving
- public Long getResourceValue(String resource) throws YarnException {
- if (getResources().containsKey(resource)) {
- return getResources().get(resource).getValue();
- }
- throw new YarnException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+ public long getResourceValue(String resource)
+ throws ResourceNotFoundException {
+ return getResourceInformation(resource).getValue();
}
/**
@@ -301,23 +249,18 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Evolving
public void setResourceInformation(String resource,
- ResourceInformation resourceInformation) throws ResourceNotFoundException {
- if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+ ResourceInformation resourceInformation)
+ throws ResourceNotFoundException {
+ if (resource.equals(MEMORY)) {
this.setMemorySize(resourceInformation.getValue());
return;
}
- if (resource.equals(ResourceInformation.VCORES.getName())) {
+ if (resource.equals(VCORES)) {
this.setVirtualCores((int) resourceInformation.getValue());
return;
}
- if (getResources().containsKey(resource)) {
- ResourceInformation
- .copy(resourceInformation, getResources().get(resource));
- return;
- }
- throw new ResourceNotFoundException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+ ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ ResourceInformation.copy(resourceInformation, storedResourceInfo);
}
/**
@@ -332,21 +275,17 @@ public abstract class Resource implements Comparable<Resource> {
@Evolving
public void setResourceValue(String resource, Long value)
throws ResourceNotFoundException {
- if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+ if (resource.equals(MEMORY)) {
this.setMemorySize(value);
return;
}
- if (resource.equals(ResourceInformation.VCORES.getName())) {
+ if (resource.equals(VCORES)) {
this.setVirtualCores(value.intValue());
return;
}
- if (getResources().containsKey(resource)) {
- getResources().get(resource).setValue(value);
- return;
- }
- throw new ResourceNotFoundException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+
+ ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ storedResourceInfo.setValue(value);
}
@Override
@@ -356,13 +295,10 @@ public abstract class Resource implements Comparable<Resource> {
int result = (int) (939769357
+ getMemorySize()); // prime * result = 939769357 initially
result = prime * result + getVirtualCores();
- for (Map.Entry<String, ResourceInformation> entry : getResources()
- .entrySet()) {
- if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
- || entry.getKey().equals(ResourceInformation.VCORES.getName())) {
- continue;
+ for (ResourceInformation entry : getResources()) {
+ if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) {
+ result = prime * result + entry.hashCode();
}
- result = prime * result + entry.getValue().hashCode();
}
return result;
}
@@ -379,11 +315,26 @@ public abstract class Resource implements Comparable<Resource> {
return false;
}
Resource other = (Resource) obj;
- if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other
- .getVirtualCores()) {
+ if (getMemorySize() != other.getMemorySize()
+ || getVirtualCores() != other.getVirtualCores()) {
+ return false;
+ }
+
+ ResourceInformation[] myVectors = getResources();
+ ResourceInformation[] otherVectors = other.getResources();
+
+ if (myVectors.length != otherVectors.length) {
return false;
}
- return this.getResources().equals(other.getResources());
+
+ for (int i = 0; i < myVectors.length; i++) {
+ ResourceInformation a = myVectors[i];
+ ResourceInformation b = otherVectors[i];
+ if ((a != b) && ((a == null) || !a.equals(b))) {
+ return false;
+ }
+ }
+ return true;
}
@Override
@@ -391,21 +342,20 @@ public abstract class Resource implements Comparable<Resource> {
StringBuilder sb = new StringBuilder();
sb.append("<memory:").append(getMemorySize()).append(", vCores:")
.append(getVirtualCores());
- for (Map.Entry<String, ResourceInformation> entry : getResources()
- .entrySet()) {
- if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
- && entry.getValue().getUnits()
+ for (ResourceInformation entry : getResources()) {
+ if (entry.getName().equals(MEMORY)
+ && entry.getUnits()
.equals(ResourceInformation.MEMORY_MB.getUnits())) {
continue;
}
- if (entry.getKey().equals(ResourceInformation.VCORES.getName())
- && entry.getValue().getUnits()
+ if (entry.getName().equals(VCORES)
+ && entry.getUnits()
.equals(ResourceInformation.VCORES.getUnits())) {
continue;
}
- sb.append(", ").append(entry.getKey()).append(": ")
- .append(entry.getValue().getValue())
- .append(entry.getValue().getUnits());
+ sb.append(", ").append(entry.getName()).append(": ")
+ .append(entry.getValue())
+ .append(entry.getUnits());
}
sb.append(">");
return sb.toString();
@@ -413,28 +363,30 @@ public abstract class Resource implements Comparable<Resource> {
@Override
public int compareTo(Resource other) {
- Map<String, ResourceInformation> thisResources, otherResources;
- thisResources = this.getResources();
- otherResources = other.getResources();
- long diff = thisResources.size() - otherResources.size();
+ ResourceInformation[] thisResources = this.getResources();
+ ResourceInformation[] otherResources = other.getResources();
+
+ // compare memory and vcores first(in that order) to preserve
+ // existing behaviour
+ long diff = this.getMemorySize() - other.getMemorySize();
if (diff == 0) {
- // compare memory and vcores first(in that order) to preserve
- // existing behaviour
- if (thisResources.keySet().equals(otherResources.keySet())) {
- diff = this.getMemorySize() - other.getMemorySize();
- if (diff == 0) {
- diff = this.getVirtualCores() - other.getVirtualCores();
- }
- if (diff == 0) {
- for (Map.Entry<String, ResourceInformation> entry : thisResources
- .entrySet()) {
- if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
- || entry.getKey()
- .equals(ResourceInformation.VCORES.getName())) {
- continue;
- }
- diff =
- entry.getValue().compareTo(otherResources.get(entry.getKey()));
+ diff = this.getVirtualCores() - other.getVirtualCores();
+ }
+ if (diff == 0) {
+ diff = thisResources.length - otherResources.length;
+ if (diff == 0) {
+ int maxLength = ResourceUtils.getResourceTypesArray().length;
+ for (int i = 0; i < maxLength; i++) {
+ // For memory and vcores, we can skip the loop as it's already
+ // compared.
+ if (i < 2) {
+ continue;
+ }
+
+ ResourceInformation entry = thisResources[i];
+ ResourceInformation otherEntry = otherResources[i];
+ if (entry.getName().equals(otherEntry.getName())) {
+ diff = entry.compareTo(otherEntry);
if (diff != 0) {
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index d75b441..3ab7ccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -242,10 +242,15 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
return false;
}
ResourceInformation r = (ResourceInformation) obj;
- int cmp =
- UnitsConversionUtil.compare(this.units, this.value, r.units, r.value);
- return this.name.equals(r.getName()) && this.resourceType
- .equals(r.getResourceType()) && (cmp == 0);
+ if (!this.name.equals(r.getName())
+ || !this.resourceType.equals(r.getResourceType())) {
+ return false;
+ }
+ if (this.units.equals(r.units)) {
+ return this.value == r.value;
+ }
+ return (UnitsConversionUtil.compare(this.units, this.value, r.units,
+ r.value) == 0);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
new file mode 100644
index 0000000..83db542
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+import java.util.Arrays;
+
+/**
+ * <p>
+ * <code>BaseResource</code> extends Resource to handle base resources such
+ * as memory and CPU.
+ * TODO: We have a long term plan to use AbstractResource when additional
+ * resource types are to be handled as well.
+ * </p>
+ *
+ * <p>
+ * Currently it models both <em>memory</em> and <em>CPU</em>.
+ * </p>
+ *
+ * <p>
+ * The unit for memory is megabytes. CPU is modeled with virtual cores (vcores),
+ * a unit for expressing parallelism. A node's capacity should be configured
+ * with virtual cores equal to its number of physical cores. A container should
+ * be requested with the number of cores it can saturate, i.e. the average
+ * number of threads it expects to have runnable at a time.
+ * </p>
+ *
+ * <p>
+ * Virtual cores take integer values and thus currently CPU-scheduling is very
+ * coarse. A complementary axis for CPU requests that represents processing
+ * power will likely be added in the future to enable finer-grained resource
+ * configuration.
+ * </p>
+ *
+ * @see Resource
+ */
+@Public
+@Unstable
+public class BaseResource extends Resource {
+
+ private ResourceInformation memoryResInfo;
+ private ResourceInformation vcoresResInfo;
+ protected ResourceInformation[] resources = null;
+ protected ResourceInformation[] readOnlyResources = null;
+
+ protected enum MandatoryResources {
+ MEMORY(0), VCORES(1);
+
+ private final int id;
+
+ MandatoryResources(int id) {
+ this.id = id;
+ }
+
+ public int getId() {
+ return this.id;
+ }
+ }
+
+ public BaseResource() {
+ // Base constructor.
+ }
+
+ public BaseResource(long memory, long vcores) {
+ this.memoryResInfo = ResourceInformation.newInstance(MEMORY,
+ ResourceInformation.MEMORY_MB.getUnits(), memory);
+ this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
+
+ resources = new ResourceInformation[MandatoryResources.values().length];
+ readOnlyResources = new ResourceInformation[MandatoryResources
+ .values().length];
+ resources[MandatoryResources.MEMORY.id] = memoryResInfo;
+ resources[MandatoryResources.VCORES.id] = vcoresResInfo;
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public int getMemory() {
+ return (int) memoryResInfo.getValue();
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void setMemory(int memory) {
+ this.memoryResInfo.setValue(memory);
+ }
+
+ @Override
+ public long getMemorySize() {
+ return memoryResInfo.getValue();
+ }
+
+ @Override
+ public void setMemorySize(long memory) {
+ this.memoryResInfo.setValue(memory);
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return (int) vcoresResInfo.getValue();
+ }
+
+ @Override
+ public void setVirtualCores(int vcores) {
+ this.vcoresResInfo.setValue(vcores);
+ }
+
+ @Override
+ public ResourceInformation[] getResources() {
+ return readOnlyResources;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
new file mode 100644
index 0000000..d04e339
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.api.records.impl;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
index c7663de..7b737bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
@@ -186,11 +186,11 @@ public class UnitsConversionUtil {
if (!KNOWN_UNITS.contains(unitB)) {
throw new IllegalArgumentException("Unknown unit '" + unitB + "'");
}
- Converter unitAC = getConverter(unitA);
- Converter unitBC = getConverter(unitB);
if (unitA.equals(unitB)) {
- return Long.valueOf(valueA).compareTo(valueB);
+ return Long.compare(valueA, valueB);
}
+ Converter unitAC = getConverter(unitA);
+ Converter unitBC = getConverter(unitB);
int unitAPos = SORTED_UNITS.indexOf(unitA);
int unitBPos = SORTED_UNITS.indexOf(unitB);
try {
@@ -201,7 +201,7 @@ public class UnitsConversionUtil {
} else {
tmpA = convert(unitA, unitB, valueA);
}
- return Long.valueOf(tmpA).compareTo(tmpB);
+ return Long.compare(tmpA, tmpB);
} catch (IllegalArgumentException ie) {
BigInteger tmpA = BigInteger.valueOf(valueA);
BigInteger tmpB = BigInteger.valueOf(valueB);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
new file mode 100644
index 0000000..49feafa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -0,0 +1,534 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util.resource;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Helper class to read the resource-types to be supported by the system.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class ResourceUtils {
+
+ public static final String UNITS = ".units";
+ public static final String TYPE = ".type";
+ public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
+ public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
+
+ private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
+ private static final String VCORES = ResourceInformation.VCORES.getName();
+
+ private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
+ static {
+ DISALLOWED_NAMES.add("memory");
+ DISALLOWED_NAMES.add(MEMORY);
+ DISALLOWED_NAMES.add(VCORES);
+ }
+
+ private static volatile boolean initializedResources = false;
+ private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
+ new ConcurrentHashMap<String, Integer>();
+ private static volatile Map<String, ResourceInformation> resourceTypes;
+ private static volatile String[] resourceNamesArray;
+ private static volatile ResourceInformation[] resourceTypesArray;
+ private static volatile boolean initializedNodeResources = false;
+ private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
+
+ static final Log LOG = LogFactory.getLog(ResourceUtils.class);
+
+ private ResourceUtils() {
+ }
+
+ private static void checkMandatatoryResources(
+ Map<String, ResourceInformation> resourceInformationMap)
+ throws YarnRuntimeException {
+ if (resourceInformationMap.containsKey(MEMORY)) {
+ ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
+ String memUnits = ResourceInformation.MEMORY_MB.getUnits();
+ ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
+ if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
+ .equals(memType)) {
+ throw new YarnRuntimeException(
+ "Attempt to re-define mandatory resource 'memory-mb'. It can only"
+ + " be of type 'COUNTABLE' and have units 'Mi'.");
+ }
+ }
+
+ if (resourceInformationMap.containsKey(VCORES)) {
+ ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
+ String vcoreUnits = ResourceInformation.VCORES.getUnits();
+ ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
+ if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
+ .getResourceType().equals(vcoreType)) {
+ throw new YarnRuntimeException(
+ "Attempt to re-define mandatory resource 'vcores'. It can only be"
+ + " of type 'COUNTABLE' and have units ''(no units).");
+ }
+ }
+ }
+
+ private static void addManadtoryResources(
+ Map<String, ResourceInformation> res) {
+ ResourceInformation ri;
+ if (!res.containsKey(MEMORY)) {
+ LOG.info("Adding resource type - name = " + MEMORY + ", units = "
+ + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
+ + ResourceTypes.COUNTABLE);
+ ri = ResourceInformation
+ .newInstance(MEMORY,
+ ResourceInformation.MEMORY_MB.getUnits());
+ res.put(MEMORY, ri);
+ }
+ if (!res.containsKey(VCORES)) {
+ LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
+ + ResourceTypes.COUNTABLE);
+ ri =
+ ResourceInformation.newInstance(VCORES);
+ res.put(VCORES, ri);
+ }
+ }
+
+ private static void setMinimumAllocationForMandatoryResources(
+ Map<String, ResourceInformation> res, Configuration conf) {
+ String[][] resourceTypesKeys = {
+ {ResourceInformation.MEMORY_MB.getName(),
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+ ResourceInformation.MEMORY_MB.getName()},
+ {ResourceInformation.VCORES.getName(),
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
+ ResourceInformation.VCORES.getName()}};
+ for (String[] arr : resourceTypesKeys) {
+ String resourceTypesKey =
+ YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
+ long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
+ long minimumConf = conf.getLong(arr[1], -1);
+ long minimum;
+ if (minimumResourceTypes != -1) {
+ minimum = minimumResourceTypes;
+ if (minimumConf != -1) {
+ LOG.warn("Using minimum allocation for memory specified in "
+ + "resource-types config file with key "
+ + minimumResourceTypes + ", ignoring minimum specified using "
+ + arr[1]);
+ }
+ } else {
+ minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+ }
+ ResourceInformation ri = res.get(arr[3]);
+ ri.setMinimumAllocation(minimum);
+ }
+ }
+
+ private static void setMaximumAllocationForMandatoryResources(
+ Map<String, ResourceInformation> res, Configuration conf) {
+ String[][] resourceTypesKeys = {
+ {ResourceInformation.MEMORY_MB.getName(),
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
+ ResourceInformation.MEMORY_MB.getName()},
+ {ResourceInformation.VCORES.getName(),
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ String.valueOf(
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
+ ResourceInformation.VCORES.getName()}};
+ for (String[] arr : resourceTypesKeys) {
+ String resourceTypesKey =
+ YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
+ long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
+ long maximumConf = conf.getLong(arr[1], -1);
+ long maximum;
+ if (maximumResourceTypes != -1) {
+ maximum = maximumResourceTypes;
+ if (maximumConf != -1) {
+ LOG.warn("Using maximum allocation for memory specified in "
+ + "resource-types config file with key "
+ + maximumResourceTypes + ", ignoring maximum specified using "
+ + arr[1]);
+ }
+ } else {
+ maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
+ }
+ ResourceInformation ri = res.get(arr[3]);
+ ri.setMaximumAllocation(maximum);
+ }
+ }
+
+ @VisibleForTesting
+ static void initializeResourcesMap(Configuration conf,
+ Map<String, ResourceInformation> resourceInformationMap) {
+
+ String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
+
+ if (resourceNames != null && resourceNames.length != 0) {
+ for (String resourceName : resourceNames) {
+ String resourceUnits = conf.get(
+ YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
+ String resourceTypeName = conf.get(
+ YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
+ ResourceTypes.COUNTABLE.toString());
+ Long minimumAllocation = conf.getLong(
+ YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+ + MINIMUM_ALLOCATION, 0L);
+ Long maximumAllocation = conf.getLong(
+ YarnConfiguration.RESOURCE_TYPES + "." + resourceName
+ + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
+ if (resourceName == null || resourceName.isEmpty()
+ || resourceUnits == null || resourceTypeName == null) {
+ throw new YarnRuntimeException(
+ "Incomplete configuration for resource type '" + resourceName
+ + "'. One of name, units or type is configured incorrectly.");
+ }
+ if (DISALLOWED_NAMES.contains(resourceName)) {
+ throw new YarnRuntimeException(
+ "Resource type cannot be named '" + resourceName
+ + "'. That name is disallowed.");
+ }
+ ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
+ LOG.info("Adding resource type - name = " + resourceName + ", units = "
+ + resourceUnits + ", type = " + resourceTypeName);
+ if (resourceInformationMap.containsKey(resourceName)) {
+ throw new YarnRuntimeException(
+ "Error in config, key '" + resourceName + "' specified twice");
+ }
+ resourceInformationMap.put(resourceName, ResourceInformation
+ .newInstance(resourceName, resourceUnits, 0L, resourceType,
+ minimumAllocation, maximumAllocation));
+ }
+ }
+ checkMandatatoryResources(resourceInformationMap);
+ addManadtoryResources(resourceInformationMap);
+ setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
+ setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
+ resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
+ updateKnownResources();
+ updateResourceTypeIndex();
+ }
+
+ private static void updateKnownResources() {
+ // Update resource names.
+ resourceNamesArray = new String[resourceTypes.size()];
+ resourceTypesArray = new ResourceInformation[resourceTypes.size()];
+
+ int index = 2;
+ for (ResourceInformation resInfo : resourceTypes.values()) {
+ if (resInfo.getName().equals(MEMORY)) {
+ resourceTypesArray[0] = ResourceInformation
+ .newInstance(resourceTypes.get(MEMORY));
+ resourceNamesArray[0] = MEMORY;
+ } else if (resInfo.getName().equals(VCORES)) {
+ resourceTypesArray[1] = ResourceInformation
+ .newInstance(resourceTypes.get(VCORES));
+ resourceNamesArray[1] = VCORES;
+ } else {
+ resourceTypesArray[index] = ResourceInformation.newInstance(resInfo);
+ resourceNamesArray[index] = resInfo.getName();
+ index++;
+ }
+ }
+ }
+
+ private static void updateResourceTypeIndex() {
+ RESOURCE_NAME_TO_INDEX.clear();
+
+ for (int index = 0; index < resourceTypesArray.length; index++) {
+ ResourceInformation resInfo = resourceTypesArray[index];
+ RESOURCE_NAME_TO_INDEX.put(resInfo.getName(), index);
+ }
+ }
+
+ /**
+ * Get associate index of resource types such memory, cpu etc.
+ * This could help to access each resource types in a resource faster.
+ * @return Index map for all Resource Types.
+ */
+ public static Map<String, Integer> getResourceTypeIndex() {
+ return RESOURCE_NAME_TO_INDEX;
+ }
+
+ /**
+ * Get the resource types to be supported by the system.
+ * @return A map of the resource name to a ResouceInformation object
+ * which contains details such as the unit.
+ */
+ public static Map<String, ResourceInformation> getResourceTypes() {
+ return getResourceTypes(null,
+ YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ }
+
+ /**
+ * Get resource names array, this is mostly for performance perspective. Never
+ * modify returned array.
+ *
+ * @return resourceNamesArray
+ */
+ public static String[] getResourceNamesArray() {
+ getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ return resourceNamesArray;
+ }
+
+ public static ResourceInformation[] getResourceTypesArray() {
+ getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ return resourceTypesArray;
+ }
+
+ private static Map<String, ResourceInformation> getResourceTypes(
+ Configuration conf) {
+ return getResourceTypes(conf,
+ YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ }
+
+ private static Map<String, ResourceInformation> getResourceTypes(
+ Configuration conf, String resourceFile) {
+ if (!initializedResources) {
+ synchronized (ResourceUtils.class) {
+ if (!initializedResources) {
+ Map<String, ResourceInformation> resources = new HashMap<>();
+ if (conf == null) {
+ conf = new YarnConfiguration();
+ }
+ try {
+ addResourcesFileToConf(resourceFile, conf);
+ LOG.debug("Found " + resourceFile + ", adding to configuration");
+ initializeResourcesMap(conf, resources);
+ initializedResources = true;
+ } catch (FileNotFoundException fe) {
+ LOG.info("Unable to find '" + resourceFile
+ + "'. Falling back to memory and vcores as resources", fe);
+ initializeResourcesMap(conf, resources);
+ initializedResources = true;
+ }
+ }
+ }
+ }
+ return resourceTypes;
+ }
+
+ private static InputStream getConfInputStream(String resourceFile,
+ Configuration conf) throws IOException, YarnException {
+
+ ConfigurationProvider provider =
+ ConfigurationProviderFactory.getConfigurationProvider(conf);
+ try {
+ provider.init(conf);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
+ if (ris == null) {
+ if (conf.getResource(resourceFile) == null) {
+ throw new FileNotFoundException("Unable to find " + resourceFile);
+ }
+ throw new IOException(
+ "Unable to open resource types file '" + resourceFile
+ + "'. Using provider " + provider);
+ }
+ return ris;
+ }
+
+ private static void addResourcesFileToConf(String resourceFile,
+ Configuration conf) throws FileNotFoundException {
+ try {
+ InputStream ris = getConfInputStream(resourceFile, conf);
+ LOG.debug("Found " + resourceFile + ", adding to configuration");
+ conf.addResource(ris);
+ } catch (FileNotFoundException fe) {
+ throw fe;
+ } catch (IOException ie) {
+ LOG.fatal("Exception trying to read resource types configuration '"
+ + resourceFile + "'.", ie);
+ throw new YarnRuntimeException(ie);
+ } catch (YarnException ye) {
+ LOG.fatal("YARN Exception trying to read resource types configuration '"
+ + resourceFile + "'.", ye);
+ throw new YarnRuntimeException(ye);
+ }
+ }
+
+ @VisibleForTesting
+ synchronized static void resetResourceTypes() {
+ initializedResources = false;
+ }
+
+ @VisibleForTesting
+ public static void resetResourceTypes(Configuration conf) {
+ synchronized (ResourceUtils.class) {
+ initializedResources = false;
+ }
+ getResourceTypes(conf);
+ }
+
+ public static String getUnits(String resourceValue) {
+ String units;
+ for (int i = 0; i < resourceValue.length(); i++) {
+ if (Character.isAlphabetic(resourceValue.charAt(i))) {
+ units = resourceValue.substring(i);
+ if (StringUtils.isAlpha(units)) {
+ return units;
+ }
+ }
+ }
+ return "";
+ }
+
+ /**
+ * Function to get the resources for a node. This function will look at the
+ * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to
+ * determine the node resources.
+ *
+ * @param conf configuration file
+ * @return a map to resource name to the ResourceInformation object. The map
+ * is guaranteed to have entries for memory and vcores
+ */
+ public static Map<String, ResourceInformation> getNodeResourceInformation(
+ Configuration conf) {
+ if (!initializedNodeResources) {
+ synchronized (ResourceUtils.class) {
+ if (!initializedNodeResources) {
+ Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
+ conf);
+ addManadtoryResources(nodeResources);
+ checkMandatatoryResources(nodeResources);
+ setMinimumAllocationForMandatoryResources(nodeResources, conf);
+ setMaximumAllocationForMandatoryResources(nodeResources, conf);
+ readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
+ initializedNodeResources = true;
+ }
+ }
+ }
+ return readOnlyNodeResources;
+ }
+
+ private static Map<String, ResourceInformation>
+ initializeNodeResourceInformation(Configuration conf) {
+ Map<String, ResourceInformation> nodeResources = new HashMap<>();
+ try {
+ addResourcesFileToConf(
+ YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
+ for (Map.Entry<String, String> entry : conf) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
+ addResourceInformation(key, value, nodeResources);
+ }
+ }
+ } catch (FileNotFoundException fe) {
+ LOG.info("Couldn't find node resources file");
+ }
+ return nodeResources;
+ }
+
+ private static void addResourceInformation(String prop, String value,
+ Map<String, ResourceInformation> nodeResources) {
+ String[] parts = prop.split("\\.");
+ LOG.info("Found resource entry " + prop);
+ if (parts.length == 4) {
+ String resourceType = parts[3];
+ if (!nodeResources.containsKey(resourceType)) {
+ nodeResources
+ .put(resourceType, ResourceInformation.newInstance(resourceType));
+ }
+ String units = getUnits(value);
+ Long resourceValue =
+ Long.valueOf(value.substring(0, value.length() - units.length()));
+ nodeResources.get(resourceType).setValue(resourceValue);
+ nodeResources.get(resourceType).setUnits(units);
+ LOG.debug("Setting value for resource type " + resourceType + " to "
+ + resourceValue + " with units " + units);
+ }
+ }
+
+ @VisibleForTesting
+ synchronized public static void resetNodeResources() {
+ initializedNodeResources = false;
+ }
+
+ public static Resource getResourceTypesMinimumAllocation() {
+ Resource ret = Resource.newInstance(0, 0);
+ for (ResourceInformation entry : resourceTypesArray) {
+ String name = entry.getName();
+ if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+ ret.setMemorySize(entry.getMinimumAllocation());
+ } else if (name.equals(ResourceInformation.VCORES.getName())) {
+ Long tmp = entry.getMinimumAllocation();
+ if (tmp > Integer.MAX_VALUE) {
+ tmp = (long) Integer.MAX_VALUE;
+ }
+ ret.setVirtualCores(tmp.intValue());
+ } else {
+ ret.setResourceValue(name, entry.getMinimumAllocation());
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * Get a Resource object with for the maximum allocation possible.
+ * @return a Resource object with the maximum allocation for the scheduler
+ */
+ public static Resource getResourceTypesMaximumAllocation() {
+ Resource ret = Resource.newInstance(0, 0);
+ for (ResourceInformation entry : resourceTypesArray) {
+ String name = entry.getName();
+ if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
+ ret.setMemorySize(entry.getMaximumAllocation());
+ } else if (name.equals(ResourceInformation.VCORES.getName())) {
+ Long tmp = entry.getMaximumAllocation();
+ if (tmp > Integer.MAX_VALUE) {
+ tmp = (long) Integer.MAX_VALUE;
+ }
+ ret.setVirtualCores(tmp.intValue());
+ continue;
+ } else {
+ ret.setResourceValue(name, entry.getMaximumAllocation());
+ }
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
new file mode 100644
index 0000000..01c7eb0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.util.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 626ff9b..158c2ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -456,9 +456,8 @@ public class ProtoUtils {
List<YarnProtos.StringLongMapProto> pList) {
Resource tmp = Resource.newInstance(0, 0);
Map<String, Long> ret = new HashMap<>();
- for (Map.Entry<String, ResourceInformation> entry : tmp.getResources()
- .entrySet()) {
- ret.put(entry.getKey(), 0L);
+ for (ResourceInformation entry : tmp.getResources()) {
+ ret.put(entry.getName(), 0L);
}
if (pList != null) {
for (YarnProtos.StringLongMapProto p : pList) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 7bc7f5f..561deb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -33,14 +34,13 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.HashMap;
+import java.util.Arrays;
import java.util.Map;
-import java.util.Collections;
@Private
@Unstable
-public class ResourcePBImpl extends Resource {
+public class ResourcePBImpl extends BaseResource {
private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class);
@@ -48,10 +48,6 @@ public class ResourcePBImpl extends Resource {
ResourceProto.Builder builder = null;
boolean viaProto = false;
- private Map<String, ResourceInformation> resources;
- private Map<String, ResourceInformation> readOnlyResources;
-
-
// call via ProtoUtils.convertToProtoFormat(Resource)
static ResourceProto getProto(Resource r) {
final ResourcePBImpl pb;
@@ -72,8 +68,6 @@ public class ResourcePBImpl extends Resource {
public ResourcePBImpl(ResourceProto proto) {
this.proto = proto;
viaProto = true;
- this.readOnlyResources = null;
- this.resources = null;
initResources();
}
@@ -101,11 +95,13 @@ public class ResourcePBImpl extends Resource {
public long getMemorySize() {
// memory should always be present
initResources();
- ResourceInformation ri =
- this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
- return UnitsConversionUtil
- .convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(),
- ri.getValue());
+ ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()];
+
+ if (ri.getUnits().isEmpty()) {
+ return ri.getValue();
+ }
+ return UnitsConversionUtil.convert(ri.getUnits(),
+ ResourceInformation.MEMORY_MB.getUnits(), ri.getValue());
}
@Override
@@ -117,23 +113,20 @@ public class ResourcePBImpl extends Resource {
@Override
public void setMemorySize(long memory) {
maybeInitBuilder();
- getResourceInformation(ResourceInformation.MEMORY_MB.getName())
- .setValue(memory);
+ getResourceInformation(MEMORY).setValue(memory);
}
@Override
public int getVirtualCores() {
// vcores should always be present
initResources();
- return this.getResourceValue(ResourceInformation.VCORES.getName())
- .intValue();
+ return (int) resources[MandatoryResources.VCORES.getId()].getValue();
}
@Override
public void setVirtualCores(int vCores) {
maybeInitBuilder();
- getResourceInformation(ResourceInformation.VCORES.getName())
- .setValue(vCores);
+ getResourceInformation(VCORES).setValue(vCores);
}
private void initResources() {
@@ -142,6 +135,7 @@ public class ResourcePBImpl extends Resource {
}
ResourceProtoOrBuilder p = viaProto ? proto : builder;
initResourcesMap();
+ Map<String, Integer> indexMap = ResourceUtils.getResourceTypeIndex();
for (ResourceInformationProto entry : p.getResourceValueMapList()) {
ResourceTypes type =
entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
@@ -150,14 +144,16 @@ public class ResourcePBImpl extends Resource {
long value = entry.hasValue() ? entry.getValue() : 0L;
ResourceInformation ri = ResourceInformation
.newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
- if (resources.containsKey(ri.getName())) {
- resources.get(ri.getName()).setResourceType(ri.getResourceType());
- resources.get(ri.getName()).setUnits(ri.getUnits());
- resources.get(ri.getName()).setValue(value);
- } else {
+ Integer index = indexMap.get(entry.getKey());
+ if (index == null) {
LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping");
+ } else {
+ resources[index].setResourceType(ri.getResourceType());
+ resources[index].setUnits(ri.getUnits());
+ resources[index].setValue(value);
}
}
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
this.setMemorySize(p.getMemory());
this.setVirtualCores(p.getVirtualCores());
}
@@ -173,79 +169,67 @@ public class ResourcePBImpl extends Resource {
if (!resource.equals(resourceInformation.getName())) {
resourceInformation.setName(resource);
}
- initResources();
- if (resources.containsKey(resource)) {
- ResourceInformation.copy(resourceInformation, resources.get(resource));
- }
+ ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ ResourceInformation.copy(resourceInformation, storedResourceInfo);
}
@Override
public void setResourceValue(String resource, Long value)
throws ResourceNotFoundException {
maybeInitBuilder();
- initResources();
if (resource == null) {
throw new IllegalArgumentException("resource type object cannot be null");
}
- if (resources == null || (!resources.containsKey(resource))) {
- throw new ResourceNotFoundException(
- "Resource " + resource + " not found");
- }
- resources.get(resource).setValue(value);
+ getResourceInformation(resource).setValue(value);
}
@Override
- public Map<String, ResourceInformation> getResources() {
+ public ResourceInformation[] getResources() {
initResources();
- return readOnlyResources;
+ return super.getResources();
}
@Override
- public ResourceInformation getResourceInformation(String resource) {
+ public ResourceInformation getResourceInformation(String resource)
+ throws ResourceNotFoundException {
initResources();
- if (this.resources.containsKey(resource)) {
- return this.resources.get(resource);
- }
- throw new ResourceNotFoundException("Could not find entry for " + resource);
+ return super.getResourceInformation(resource);
}
@Override
- public Long getResourceValue(String resource) {
+ public long getResourceValue(String resource)
+ throws ResourceNotFoundException {
initResources();
- if (this.resources.containsKey(resource)) {
- return this.resources.get(resource).getValue();
- }
- throw new ResourceNotFoundException("Could not find entry for " + resource);
+ return super.getResourceValue(resource);
}
private void initResourcesMap() {
if (resources == null) {
- resources = new HashMap<>();
- Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
if (types == null) {
throw new YarnRuntimeException(
"Got null return value from ResourceUtils.getResourceTypes()");
}
- for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
- resources.put(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
+
+ resources = new ResourceInformation[types.length];
+ readOnlyResources = new ResourceInformation[types.length];
+ for (ResourceInformation entry : types) {
+ int index = ResourceUtils.getResourceTypeIndex().get(entry.getName());
+ resources[index] = ResourceInformation.newInstance(entry);
}
- readOnlyResources = Collections.unmodifiableMap(resources);
}
}
synchronized private void mergeLocalToBuilder() {
builder.clearResourceValueMap();
- if (resources != null && !resources.isEmpty()) {
- for (Map.Entry<String, ResourceInformation> entry :
- resources.entrySet()) {
- ResourceInformationProto.Builder e =
- ResourceInformationProto.newBuilder();
- e.setKey(entry.getKey());
- e.setUnits(entry.getValue().getUnits());
- e.setType(
- ProtoUtils.converToProtoFormat(entry.getValue().getResourceType()));
- e.setValue(entry.getValue().getValue());
+ if(resources != null && resources.length != 0) {
+ for (ResourceInformation resInfo : resources) {
+ ResourceInformationProto.Builder e = ResourceInformationProto
+ .newBuilder();
+ e.setKey(resInfo.getName());
+ e.setUnits(resInfo.getUnits());
+ e.setType(ProtoUtils.converToProtoFormat(resInfo.getResourceType()));
+ e.setValue(resInfo.getValue());
builder.addResourceValueMap(e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b586d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 79bb03d..ea5c8a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -23,10 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.Set;
/**
* A {@link ResourceCalculator} which uses the concept of
@@ -56,10 +55,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
LogFactory.getLog(DominantResourceCalculator.class);
- private Set<String> resourceNames;
+ private String[] resourceNames;
public DominantResourceCalculator() {
- resourceNames = ResourceUtils.getResourceTypes().keySet();
+ resourceNames = ResourceUtils.getResourceNamesArray();
}
/**
@@ -88,7 +87,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
} else if (diff <= -1) {
rhsGreater = true;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + rName, ye);
}
@@ -163,7 +162,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
.getValue();
min = min < tmp ? min : tmp;
max = max > tmp ? max : tmp;
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -187,7 +186,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
long tmp = availableResource.getValue() / requiredResourceValue;
min = min < tmp ? min : tmp;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -206,14 +205,9 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public boolean isInvalidDivisor(Resource r) {
- for (String resource : resourceNames) {
- try {
- if (r.getResourceValue(resource).equals(0L)) {
- return true;
- }
- } catch (YarnException ye) {
- throw new IllegalArgumentException(
- "Error getting resource value for " + resource, ye);
+ for (ResourceInformation res : r.getResources()) {
+ if (res.getValue() == 0L) {
+ return true;
}
}
return false;
@@ -235,7 +229,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
float tmp =
(float) aResourceInformation.getValue() / (float) bResourceValue;
ratio = ratio > tmp ? ratio : tmp;
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -256,7 +250,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
ret.getResourceInformation(resource);
resourceInformation.setValue(
divideAndCeil(resourceInformation.getValue(), denominator));
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -307,7 +301,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
tmp.setValue(Math.min(value, maximumValue));
ret.setResourceInformation(resource, tmp);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -347,7 +341,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
ResourceInformation
.copy(rResourceInformation, ret.getResourceInformation(resource));
ret.getResourceInformation(resource).setValue(value);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -372,28 +366,29 @@ public class DominantResourceCalculator extends ResourceCalculator {
Resource ret = Resource.newInstance(r);
for (String resource : resourceNames) {
try {
- ResourceInformation rResourceInformation =
- r.getResourceInformation(resource);
- ResourceInformation stepFactorResourceInformation =
- stepFactor.getResourceInformation(resource);
+ ResourceInformation rResourceInformation = r
+ .getResourceInformation(resource);
+ ResourceInformation stepFactorResourceInformation = stepFactor
+ .getResourceInformation(resource);
ResourceInformation tmp = ret.getResourceInformation(resource);
- Long rValue = rResourceInformation.getValue();
- Long stepFactorValue = UnitsConversionUtil
- .convert(stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
- Long value;
+ long rValue = rResourceInformation.getValue();
+ long stepFactorValue = UnitsConversionUtil.convert(
+ stepFactorResourceInformation.getUnits(),
+ rResourceInformation.getUnits(),
+ stepFactorResourceInformation.getValue());
+ long value;
if (stepFactorValue != 0) {
- value = roundUp ?
- roundUp((long) Math.ceil(rValue * by), stepFactorValue) :
- roundDown((long) (rValue * by), stepFactorValue);
+ value = roundUp
+ ? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
+ : roundDown((long) (rValue * by), stepFactorValue);
} else {
- value =
- roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
+ value = roundUp
+ ? (long) Math.ceil(rValue * by)
+ : (long) (rValue * by);
}
tmp.setValue(value);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -416,7 +411,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
if(sResourceValue > bResourceInformation.getValue()) {
return false;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
return false;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org