You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/08/09 09:44:11 UTC
[hadoop] branch branch-3.2 updated: YARN-9124. Resolve
contradiction in ResourceUtils: addMandatoryResources /
checkMandatoryResources work differently (#1121)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 4c4f7d9 YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)
4c4f7d9 is described below
commit 4c4f7d9c80302d18d4abf51b26955b0ae1aabbc2
Author: Adam Antal <ad...@cloudera.com>
AuthorDate: Fri Aug 2 13:04:34 2019 +0200
YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)
(cherry picked from commit cbcada804d119b837ad99de71d7f44cb4629026e)
---
.../yarn/api/records/ResourceInformation.java | 6 +++-
.../hadoop/yarn/util/resource/ResourceUtils.java | 35 +++++++++++++++++-----
2 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index c83c3a2..3f2f4ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -53,7 +53,11 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
public static final ResourceInformation FPGAS =
ResourceInformation.newInstance(FPGA_URI);
- public static final Map<String, ResourceInformation> MANDATORY_RESOURCES =
+ /**
+ * Special resources that should be treated separately
+ * from arbitrary resource types.
+ */
+ public static final Map<String, ResourceInformation> SPECIAL_RESOURCES =
ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS, FPGA_URI, FPGAS);
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index c2d7201..fd8be24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -81,14 +81,28 @@ public class ResourceUtils {
private ResourceUtils() {
}
- private static void checkMandatoryResources(
+ /**
+ * Ensures that historical resource types (like {@link
+ * ResourceInformation#MEMORY_URI}, {@link ResourceInformation#VCORES_URI})
+ * are not getting overridden in the resourceInformationMap.
+ *
+ * Also checks whether {@link ResourceInformation#SPECIAL_RESOURCES} are not
+ * configured poorly: having their proper units and types.
+ *
+ * @param resourceInformationMap Map object having keys as resources names
+ * and {@link ResourceInformation} objects as
+ * values
+ * @throws YarnRuntimeException if either of the two above
+ * conditions do not hold
+ */
+ private static void checkSpecialResources(
Map<String, ResourceInformation> resourceInformationMap)
throws YarnRuntimeException {
/*
- * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to
- * 'MEMORY' for historical reasons
+ * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource
+ * names, in addition to 'MEMORY' for historical reasons
*/
- String keys[] = { "memory", ResourceInformation.MEMORY_URI,
+ String[] keys = { "memory", ResourceInformation.MEMORY_URI,
ResourceInformation.VCORES_URI };
for(String key : keys) {
if (resourceInformationMap.containsKey(key)) {
@@ -99,7 +113,7 @@ public class ResourceUtils {
}
for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
- ResourceInformation.MANDATORY_RESOURCES.entrySet()) {
+ ResourceInformation.SPECIAL_RESOURCES.entrySet()) {
String key = mandatoryResourceEntry.getKey();
ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
@@ -122,6 +136,13 @@ public class ResourceUtils {
}
}
+ /**
+ * Ensures that {@link ResourceUtils#MEMORY} and {@link ResourceUtils#VCORES}
+ * resources are contained in the map received as parameter.
+ *
+ * @param res Map object having keys as resources names
+ * and {@link ResourceInformation} objects as values
+ */
private static void addMandatoryResources(
Map<String, ResourceInformation> res) {
ResourceInformation ri;
@@ -259,7 +280,7 @@ public class ResourceUtils {
validateNameOfResourceNameAndThrowException(name);
}
- checkMandatoryResources(resourceInformationMap);
+ checkSpecialResources(resourceInformationMap);
addMandatoryResources(resourceInformationMap);
setAllocationForMandatoryResources(resourceInformationMap, conf);
@@ -488,7 +509,7 @@ public class ResourceUtils {
if (!initializedNodeResources) {
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
conf);
- checkMandatoryResources(nodeResources);
+ checkSpecialResources(nodeResources);
addMandatoryResources(nodeResources);
setAllocationForMandatoryResources(nodeResources, conf);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org