You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/09/12 20:41:29 UTC
[14/50] [abbrv] hadoop git commit: YARN-6788. [YARN-3926] Improve
performance of resource profile branch (Contributed by Sunil Govindan via
Daniel Templeton)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
deleted file mode 100644
index 86cf872..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ /dev/null
@@ -1,488 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util.resource;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.conf.ConfigurationProvider;
-import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Helper class to read the resource-types to be supported by the system.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class ResourceUtils {
-
- public static final String UNITS = ".units";
- public static final String TYPE = ".type";
- public static final String MINIMUM_ALLOCATION = ".minimum-allocation";
- public static final String MAXIMUM_ALLOCATION = ".maximum-allocation";
-
- private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
- private static final String VCORES = ResourceInformation.VCORES.getName();
-
- private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
- static {
- DISALLOWED_NAMES.add("memory");
- DISALLOWED_NAMES.add(MEMORY);
- DISALLOWED_NAMES.add(VCORES);
- }
-
- private static volatile Object lock;
- private static Map<String, ResourceInformation> readOnlyResources;
- private static volatile Object nodeLock;
- private static Map<String, ResourceInformation> readOnlyNodeResources;
-
-
- static final Log LOG = LogFactory.getLog(ResourceUtils.class);
-
- private ResourceUtils() {
- }
-
- private static void checkMandatatoryResources(
- Map<String, ResourceInformation> resourceInformationMap)
- throws YarnRuntimeException {
- if (resourceInformationMap.containsKey(MEMORY)) {
- ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
- String memUnits = ResourceInformation.MEMORY_MB.getUnits();
- ResourceTypes memType = ResourceInformation.MEMORY_MB.getResourceType();
- if (!memInfo.getUnits().equals(memUnits) || !memInfo.getResourceType()
- .equals(memType)) {
- throw new YarnRuntimeException(
- "Attempt to re-define mandatory resource 'memory-mb'. It can only"
- + " be of type 'COUNTABLE' and have units 'Mi'.");
- }
- }
-
- if (resourceInformationMap.containsKey(VCORES)) {
- ResourceInformation vcoreInfo = resourceInformationMap.get(VCORES);
- String vcoreUnits = ResourceInformation.VCORES.getUnits();
- ResourceTypes vcoreType = ResourceInformation.VCORES.getResourceType();
- if (!vcoreInfo.getUnits().equals(vcoreUnits) || !vcoreInfo
- .getResourceType().equals(vcoreType)) {
- throw new YarnRuntimeException(
- "Attempt to re-define mandatory resource 'vcores'. It can only be"
- + " of type 'COUNTABLE' and have units ''(no units).");
- }
- }
- }
-
- private static void addManadtoryResources(
- Map<String, ResourceInformation> res) {
- ResourceInformation ri;
- if (!res.containsKey(MEMORY)) {
- LOG.info("Adding resource type - name = " + MEMORY + ", units = "
- + ResourceInformation.MEMORY_MB.getUnits() + ", type = "
- + ResourceTypes.COUNTABLE);
- ri = ResourceInformation
- .newInstance(MEMORY,
- ResourceInformation.MEMORY_MB.getUnits());
- res.put(MEMORY, ri);
- }
- if (!res.containsKey(VCORES)) {
- LOG.info("Adding resource type - name = " + VCORES + ", units = , type = "
- + ResourceTypes.COUNTABLE);
- ri =
- ResourceInformation.newInstance(VCORES);
- res.put(VCORES, ri);
- }
- }
-
- private static void setMinimumAllocationForMandatoryResources(
- Map<String, ResourceInformation> res, Configuration conf) {
- String[][] resourceTypesKeys =
- {
- { ResourceInformation.MEMORY_MB.getName(),
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
- ResourceInformation.MEMORY_MB.getName()
- },
- { ResourceInformation.VCORES.getName(),
- YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
- ResourceInformation.VCORES.getName()
- }
- };
- for (String[] arr : resourceTypesKeys) {
- String resourceTypesKey =
- YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
- long minimumResourceTypes = conf.getLong(resourceTypesKey, -1);
- long minimumConf = conf.getLong(arr[1], -1);
- long minimum;
- if (minimumResourceTypes != -1) {
- minimum = minimumResourceTypes;
- if (minimumConf != -1) {
- LOG.warn("Using minimum allocation for memory specified in "
- + "resource-types config file with key "
- + minimumResourceTypes + ", ignoring minimum specified using "
- + arr[1]);
- }
- } else {
- minimum = conf.getLong(arr[1], Long.parseLong(arr[2]));
- }
- ResourceInformation ri = res.get(arr[3]);
- ri.setMinimumAllocation(minimum);
- }
- }
-
- private static void setMaximumAllocationForMandatoryResources(
- Map<String, ResourceInformation> res, Configuration conf) {
- String[][] resourceTypesKeys =
- {
- {
- ResourceInformation.MEMORY_MB.getName(),
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
- ResourceInformation.MEMORY_MB.getName()
- },
- {
- ResourceInformation.VCORES.getName(),
- YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
- String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
- ResourceInformation.VCORES.getName()
- }
- };
- for (String[] arr : resourceTypesKeys) {
- String resourceTypesKey =
- YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
- long maximumResourceTypes = conf.getLong(resourceTypesKey, -1);
- long maximumConf = conf.getLong(arr[1], -1);
- long maximum;
- if (maximumResourceTypes != -1) {
- maximum = maximumResourceTypes;
- if (maximumConf != -1) {
- LOG.warn("Using maximum allocation for memory specified in "
- + "resource-types config file with key "
- + maximumResourceTypes + ", ignoring maximum specified using "
- + arr[1]);
- }
- } else {
- maximum = conf.getLong(arr[1], Long.parseLong(arr[2]));
- }
- ResourceInformation ri = res.get(arr[3]);
- ri.setMaximumAllocation(maximum);
- }
- }
-
- @VisibleForTesting
- static void initializeResourcesMap(Configuration conf,
- Map<String, ResourceInformation> resourceInformationMap) {
-
- String[] resourceNames = conf.getStrings(YarnConfiguration.RESOURCE_TYPES);
-
- if (resourceNames != null && resourceNames.length != 0) {
- for (String resourceName : resourceNames) {
- String resourceUnits = conf.get(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName + UNITS, "");
- String resourceTypeName = conf.get(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName + TYPE,
- ResourceTypes.COUNTABLE.toString());
- Long minimumAllocation = conf.getLong(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName
- + MINIMUM_ALLOCATION, 0L);
- Long maximumAllocation = conf.getLong(
- YarnConfiguration.RESOURCE_TYPES + "." + resourceName
- + MAXIMUM_ALLOCATION, Long.MAX_VALUE);
- if (resourceName == null || resourceName.isEmpty()
- || resourceUnits == null || resourceTypeName == null) {
- throw new YarnRuntimeException(
- "Incomplete configuration for resource type '" + resourceName
- + "'. One of name, units or type is configured incorrectly.");
- }
- if (DISALLOWED_NAMES.contains(resourceName)) {
- throw new YarnRuntimeException(
- "Resource type cannot be named '" + resourceName
- + "'. That name is disallowed.");
- }
- ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
- LOG.info("Adding resource type - name = " + resourceName + ", units = "
- + resourceUnits + ", type = " + resourceTypeName);
- if (resourceInformationMap.containsKey(resourceName)) {
- throw new YarnRuntimeException(
- "Error in config, key '" + resourceName + "' specified twice");
- }
- resourceInformationMap.put(resourceName, ResourceInformation
- .newInstance(resourceName, resourceUnits, 0L, resourceType,
- minimumAllocation, maximumAllocation));
- }
- }
- checkMandatatoryResources(resourceInformationMap);
- addManadtoryResources(resourceInformationMap);
- setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
- setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
- readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
- }
-
- /**
- * Get the resource types to be supported by the system.
- * @return A map of the resource name to a ResouceInformation object
- * which contains details such as the unit.
- */
- public static Map<String, ResourceInformation> getResourceTypes() {
- return getResourceTypes(null,
- YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
- }
-
- private static Map<String, ResourceInformation> getResourceTypes(
- Configuration conf) {
- return getResourceTypes(conf,
- YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
- }
-
- private static Map<String, ResourceInformation> getResourceTypes(
- Configuration conf, String resourceFile) {
- if (lock == null) {
- synchronized (ResourceUtils.class) {
- if (lock == null) {
- synchronized (ResourceUtils.class) {
- Map<String, ResourceInformation> resources = new HashMap<>();
- if (conf == null) {
- conf = new YarnConfiguration();
- }
- try {
- addResourcesFileToConf(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
- initializeResourcesMap(conf, resources);
- lock = new Object();
- } catch (FileNotFoundException fe) {
- LOG.info("Unable to find '" + resourceFile
- + "'. Falling back to memory and vcores as resources", fe);
- initializeResourcesMap(conf, resources);
- lock = new Object();
- }
- }
- }
- }
- }
- return readOnlyResources;
- }
-
- private static InputStream getConfInputStream(String resourceFile,
- Configuration conf) throws IOException, YarnException {
-
- ConfigurationProvider provider =
- ConfigurationProviderFactory.getConfigurationProvider(conf);
- try {
- provider.init(conf);
- } catch (Exception e) {
- throw new IOException(e);
- }
-
- InputStream ris = provider.getConfigurationInputStream(conf, resourceFile);
- if (ris == null) {
- if (conf.getResource(resourceFile) == null) {
- throw new FileNotFoundException("Unable to find " + resourceFile);
- }
- throw new IOException(
- "Unable to open resource types file '" + resourceFile
- + "'. Using provider " + provider);
- }
- return ris;
- }
-
- private static void addResourcesFileToConf(String resourceFile,
- Configuration conf) throws FileNotFoundException {
- try {
- InputStream ris = getConfInputStream(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
- conf.addResource(ris);
- } catch (FileNotFoundException fe) {
- throw fe;
- } catch (IOException ie) {
- LOG.fatal("Exception trying to read resource types configuration '"
- + resourceFile + "'.", ie);
- throw new YarnRuntimeException(ie);
- } catch (YarnException ye) {
- LOG.fatal("YARN Exception trying to read resource types configuration '"
- + resourceFile + "'.", ye);
- throw new YarnRuntimeException(ye);
- }
- }
-
- @VisibleForTesting
- static void resetResourceTypes() {
- lock = null;
- }
-
- @VisibleForTesting
- public static void resetResourceTypes(Configuration conf) {
- lock = null;
- getResourceTypes(conf);
- }
-
- public static String getUnits(String resourceValue) {
- String units;
- for (int i = 0; i < resourceValue.length(); i++) {
- if (Character.isAlphabetic(resourceValue.charAt(i))) {
- units = resourceValue.substring(i);
- if (StringUtils.isAlpha(units)) {
- return units;
- }
- }
- }
- return "";
- }
-
- /**
- * Function to get the resources for a node. This function will look at the
- * file {@link YarnConfiguration#NODE_RESOURCES_CONFIGURATION_FILE} to
- * determine the node resources.
- *
- * @param conf configuration file
- * @return a map to resource name to the ResourceInformation object. The map
- * is guaranteed to have entries for memory and vcores
- */
- public static Map<String, ResourceInformation> getNodeResourceInformation(
- Configuration conf) {
- if (nodeLock == null) {
- synchronized (ResourceUtils.class) {
- if (nodeLock == null) {
- synchronized (ResourceUtils.class) {
- Map<String, ResourceInformation> nodeResources =
- initializeNodeResourceInformation(conf);
- addManadtoryResources(nodeResources);
- checkMandatatoryResources(nodeResources);
- readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
- nodeLock = new Object();
- }
- }
- }
- }
- return readOnlyNodeResources;
- }
-
- private static Map<String, ResourceInformation>
- initializeNodeResourceInformation(Configuration conf) {
- Map<String, ResourceInformation> nodeResources = new HashMap<>();
- try {
- addResourcesFileToConf(
- YarnConfiguration.NODE_RESOURCES_CONFIGURATION_FILE, conf);
- for (Map.Entry<String, String> entry : conf) {
- String key = entry.getKey();
- String value = entry.getValue();
- if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
- addResourceInformation(key, value, nodeResources);
- }
- }
- } catch (FileNotFoundException fe) {
- LOG.info("Couldn't find node resources file");
- }
- return nodeResources;
- }
-
- private static void addResourceInformation(String prop, String value,
- Map<String, ResourceInformation> nodeResources) {
- String[] parts = prop.split("\\.");
- LOG.info("Found resource entry " + prop);
- if (parts.length == 4) {
- String resourceType = parts[3];
- if (!nodeResources.containsKey(resourceType)) {
- nodeResources
- .put(resourceType, ResourceInformation.newInstance(resourceType));
- }
- String units = getUnits(value);
- Long resourceValue =
- Long.valueOf(value.substring(0, value.length() - units.length()));
- nodeResources.get(resourceType).setValue(resourceValue);
- nodeResources.get(resourceType).setUnits(units);
- LOG.debug("Setting value for resource type " + resourceType + " to "
- + resourceValue + " with units " + units);
- }
- }
-
- @VisibleForTesting
- synchronized public static void resetNodeResources() {
- nodeLock = null;
- }
-
- public static Resource getResourceTypesMinimumAllocation() {
- Map<String, ResourceInformation> resourceTypes = getResourceTypes();
- Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : resourceTypes
- .entrySet()) {
- String name = entry.getKey();
- if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
- ret.setMemorySize(entry.getValue().getMinimumAllocation());
- continue;
- }
- if (name.equals(ResourceInformation.VCORES.getName())) {
- Long tmp = entry.getValue().getMinimumAllocation();
- if (tmp > Integer.MAX_VALUE) {
- tmp = (long) Integer.MAX_VALUE;
- }
- ret.setVirtualCores(tmp.intValue());
- continue;
- }
- ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
- }
- return ret;
- }
-
- /**
- * Get a Resource object with for the maximum allocation possible.
- * @return a Resource object with the maximum allocation for the scheduler
- */
- public static Resource getResourceTypesMaximumAllocation() {
- Map<String, ResourceInformation> resourceTypes = getResourceTypes();
- Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : resourceTypes
- .entrySet()) {
- String name = entry.getKey();
- if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
- ret.setMemorySize(entry.getValue().getMaximumAllocation());
- continue;
- }
- if (name.equals(ResourceInformation.VCORES.getName())) {
- Long tmp = entry.getValue().getMaximumAllocation();
- if (tmp > Integer.MAX_VALUE) {
- tmp = (long) Integer.MAX_VALUE;
- }
- ret.setVirtualCores(tmp.intValue());
- continue;
- }
- ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index d143e93..f62114d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -18,30 +18,31 @@
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Arrays;
@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" })
@Unstable
public class Resources {
+ private static final Log LOG =
+ LogFactory.getLog(Resources.class);
+
/**
* Helper class to create a resource with a fixed value for all resource
* types. For example, a NONE resource which returns 0 for any resource type.
*/
- static class FixedValueResource extends Resource {
+ static class FixedValueResource extends BaseResource {
- private Map<String, ResourceInformation> resources;
private long resourceValue;
private String name;
@@ -53,7 +54,7 @@ public class Resources {
FixedValueResource(String rName, long value) {
this.resourceValue = value;
this.name = rName;
- resources = initResourceMap();
+ initResourceMap();
}
private int resourceValueToInt() {
@@ -96,31 +97,6 @@ public class Resources {
}
@Override
- public Map<String, ResourceInformation> getResources() {
- return Collections.unmodifiableMap(this.resources);
- }
-
- @Override
- public ResourceInformation getResourceInformation(String resource)
- throws YarnException {
- if (resources.containsKey(resource)) {
- ResourceInformation value = this.resources.get(resource);
- ResourceInformation ret = ResourceInformation.newInstance(value);
- ret.setValue(resourceValue);
- return ret;
- }
- throw new YarnException("" + resource + " not found");
- }
-
- @Override
- public Long getResourceValue(String resource) throws YarnException {
- if (resources.containsKey(resource)) {
- return resourceValue;
- }
- throw new YarnException("" + resource + " not found");
- }
-
- @Override
public void setResourceInformation(String resource,
ResourceInformation resourceInformation)
throws ResourceNotFoundException {
@@ -133,24 +109,24 @@ public class Resources {
throw new RuntimeException(name + " cannot be modified!");
}
- private Map<String, ResourceInformation> initResourceMap() {
- Map<String, ResourceInformation> tmp = new HashMap<>();
- Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+ private void initResourceMap() {
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
if (types != null) {
- for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
- tmp.put(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
- tmp.get(entry.getKey()).setValue(resourceValue);
+ resources = new ResourceInformation[types.length];
+ readOnlyResources = new ResourceInformation[types.length];
+ for (int index = 0; index < types.length; index++) {
+ resources[index] = ResourceInformation.newInstance(types[index]);
+ resources[index].setValue(resourceValue);
+
+ // this is a fix for getVirtualCores returning an int
+ if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES
+ .getName().equals(resources[index].getName())) {
+ resources[index].setValue((long) Integer.MAX_VALUE);
+ }
}
}
- // this is a fix for getVirtualCores returning an int
- if (resourceValue > Integer.MAX_VALUE) {
- tmp.get(ResourceInformation.VCORES.getName())
- .setValue((long) Integer.MAX_VALUE);
- }
- return tmp;
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
}
-
}
public static Resource createResource(int memory) {
@@ -197,17 +173,19 @@ public class Resources {
}
public static Resource addTo(Resource lhs, Resource rhs) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -219,17 +197,19 @@ public class Resources {
}
public static Resource subtractFrom(Resource lhs, Resource rhs) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -263,10 +243,9 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
- ResourceInformation lhsValue = entry.getValue();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
+ ResourceInformation lhsValue = entry;
lhs.setResourceValue(name, (long) (lhsValue.getValue() * by));
}
return lhs;
@@ -282,17 +261,21 @@ public class Resources {
*/
public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = (long) (UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue()) * by);
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (long) (((rhsValue.getUnits()
+ .equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue()))
+ * by);
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -311,10 +294,9 @@ public class Resources {
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- for (Map.Entry<String, ResourceInformation> entry : out.getResources()
- .entrySet()) {
- String name = entry.getKey();
- ResourceInformation lhsValue = entry.getValue();
+ for (ResourceInformation entry : out.getResources()) {
+ String name = entry.getName();
+ ResourceInformation lhsValue = entry;
out.setResourceValue(name, (long) (lhsValue.getValue() * by));
}
return out;
@@ -416,19 +398,21 @@ public class Resources {
}
public static boolean fitsIn(Resource smaller, Resource bigger) {
- for (Map.Entry<String, ResourceInformation> entry : smaller.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : smaller.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = bigger.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
if(lhsValue.getValue() > convertedRhs) {
return false;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
return false;
}
}
@@ -442,19 +426,21 @@ public class Resources {
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
ResourceInformation outInfo =
lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue;
ret.setResourceInformation(name, outInfo);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -463,19 +449,21 @@ public class Resources {
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
ResourceInformation outInfo =
lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue;
ret.setResourceInformation(name, outInfo);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 38554b6..b530150 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -276,13 +276,17 @@ public class TestResourceUtils {
String resourceFile = entry.getKey();
ResourceUtils.resetNodeResources();
File dest;
- File source =
- new File(conf.getClassLoader().getResource(resourceFile).getFile());
+ File source = new File(
+ conf.getClassLoader().getResource(resourceFile).getFile());
dest = new File(source.getParent(), "node-resources.xml");
FileUtils.copyFile(source, dest);
- Map<String, ResourceInformation> actual =
- ResourceUtils.getNodeResourceInformation(conf);
- Assert.assertEquals(entry.getValue().getResources(), actual);
+ Map<String, ResourceInformation> actual = ResourceUtils
+ .getNodeResourceInformation(conf);
+ Assert.assertEquals(actual.size(),
+ entry.getValue().getResources().length);
+ for (ResourceInformation resInfo : entry.getValue().getResources()) {
+ Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+ }
dest.delete();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 1555e55..a8404fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -105,7 +105,7 @@ public class TestResources {
unboundedClone.compareTo(createResource(0, Integer.MAX_VALUE)) > 0);
}
- @Test(timeout=10000)
+ @Test(timeout = 10000)
public void testCompareToWithNoneResource() {
assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0);
assertTrue(Resources.none().compareTo(createResource(1, 0)) < 0);
@@ -114,7 +114,6 @@ public class TestResources {
assertTrue(Resources.none().compareTo(createResource(1, 0, 0)) < 0);
assertTrue(Resources.none().compareTo(createResource(0, 1, 0)) < 0);
assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
- assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
}
@Test(timeout=10000)
@@ -246,7 +245,9 @@ public class TestResources {
}
@Test
- public void testMultiplyAndAddTo() {
+ public void testMultiplyAndAddTo() throws Exception {
+ unsetExtraResourceType();
+ setupExtraResourceType();
assertEquals(createResource(6, 4),
multiplyAndAddTo(createResource(3, 1), createResource(2, 2), 1.5));
assertEquals(createResource(6, 4, 0),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index 7987ded..ab33336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -119,13 +119,13 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
private Resource parseResource(String key, Map value) throws IOException {
Resource resource = Resource.newInstance(0, 0);
Iterator iterator = value.entrySet().iterator();
- Map<String, ResourceInformation> resourceTypes =
- ResourceUtils.getResourceTypes();
+ Map<String, ResourceInformation> resourceTypes = ResourceUtils
+ .getResourceTypes();
while (iterator.hasNext()) {
Map.Entry resourceEntry = (Map.Entry) iterator.next();
String resourceName = resourceEntry.getKey().toString();
- ResourceInformation resourceValue =
- fromString(resourceName, resourceEntry.getValue().toString());
+ ResourceInformation resourceValue = fromString(resourceName,
+ resourceEntry.getValue().toString());
if (resourceName.equals(MEMORY)) {
resource.setMemorySize(resourceValue.getValue());
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 8ef6d99..0982ef9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -178,16 +178,15 @@ public class RMAppAttemptMetrics {
private void updateUsageMap(Resource allocated, long deltaUsedMillis,
Map<String, AtomicLong> targetMap) {
- for (Map.Entry<String, ResourceInformation> entry : allocated.getResources()
- .entrySet()) {
+ for (ResourceInformation entry : allocated.getResources()) {
AtomicLong resourceUsed;
- if (!targetMap.containsKey(entry.getKey())) {
+ if (!targetMap.containsKey(entry.getName())) {
resourceUsed = new AtomicLong(0);
- targetMap.put(entry.getKey(), resourceUsed);
+ targetMap.put(entry.getName(), resourceUsed);
}
- resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis)
+ resourceUsed = targetMap.get(entry.getName());
+ resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis)
/ DateUtils.MILLIS_PER_SECOND);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index d7bf1bc..db63cd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -1006,13 +1006,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
for (RMContainer rmContainer : this.liveContainers.values()) {
long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
Resource resource = rmContainer.getContainer().getResource();
- for (Map.Entry<String, ResourceInformation> entry : resource
- .getResources().entrySet()) {
+ for (ResourceInformation entry : resource.getResources()) {
long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue().getValue() * usedMillis
+ .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
+ value += entry.getValue() * usedMillis
/ DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getKey(), value);
+ resourceSecondsMap.put(entry.getName(), value);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 887b854..81491b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import java.util.Arrays;
import java.util.EnumSet;
import javax.xml.bind.annotation.XmlRootElement;
@@ -73,7 +74,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
- return minAllocResource.getResource().getResources().keySet().toString();
+ return Arrays.toString(minAllocResource.getResource().getResources());
}
public int getMaxClusterLevelAppPriority() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aeaafec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8ae630b..b24a309 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -249,6 +249,7 @@ public class TestAppManager{
ResourceScheduler scheduler = mockResourceScheduler();
((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
((RMContextImpl) rmContext).setYarnConfiguration(conf);
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org