You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/08/09 09:44:30 UTC
[hadoop] branch branch-3.1 updated: YARN-9124. Resolve
contradiction in ResourceUtils: addMandatoryResources /
checkMandatoryResources work differently (#1121)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 600a61f YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)
600a61f is described below
commit 600a61f4101ed5865fcf4f6b07f8ae643d19d5c7
Author: Adam Antal <ad...@cloudera.com>
AuthorDate: Fri Aug 2 13:04:34 2019 +0200
YARN-9124. Resolve contradiction in ResourceUtils: addMandatoryResources / checkMandatoryResources work differently (#1121)
(cherry picked from commit cbcada804d119b837ad99de71d7f44cb4629026e)
---
.../yarn/api/records/ResourceInformation.java | 6 +++-
.../hadoop/yarn/util/resource/ResourceUtils.java | 35 +++++++++++++++++-----
2 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 904ff4b..d802cdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -53,7 +53,11 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
public static final ResourceInformation FPGAS =
ResourceInformation.newInstance(FPGA_URI);
- public static final Map<String, ResourceInformation> MANDATORY_RESOURCES =
+ /**
+ * Special resources that should be treated separately
+ * from arbitrary resource types.
+ */
+ public static final Map<String, ResourceInformation> SPECIAL_RESOURCES =
ImmutableMap.of(MEMORY_URI, MEMORY_MB, VCORES_URI, VCORES, GPU_URI, GPUS, FPGA_URI, FPGAS);
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index f211f49..142dafa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -80,14 +80,28 @@ public class ResourceUtils {
private ResourceUtils() {
}
- private static void checkMandatoryResources(
+ /**
+ * Ensures that historical resource types (like {@link
+ * ResourceInformation#MEMORY_URI}, {@link ResourceInformation#VCORES_URI})
+ * are not getting overridden in the resourceInformationMap.
+ *
+ * Also checks whether {@link ResourceInformation#SPECIAL_RESOURCES} are not
+ * configured poorly: having their proper units and types.
+ *
+ * @param resourceInformationMap Map object having keys as resources names
+ * and {@link ResourceInformation} objects as
+ * values
+ * @throws YarnRuntimeException if either of the two above
+ * conditions do not hold
+ */
+ private static void checkSpecialResources(
Map<String, ResourceInformation> resourceInformationMap)
throws YarnRuntimeException {
/*
- * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource names, in addition to
- * 'MEMORY' for historical reasons
+ * Supporting 'memory', 'memory-mb', 'vcores' also as invalid resource
+ * names, in addition to 'MEMORY' for historical reasons
*/
- String keys[] = { "memory", ResourceInformation.MEMORY_URI,
+ String[] keys = { "memory", ResourceInformation.MEMORY_URI,
ResourceInformation.VCORES_URI };
for(String key : keys) {
if (resourceInformationMap.containsKey(key)) {
@@ -98,7 +112,7 @@ public class ResourceUtils {
}
for (Map.Entry<String, ResourceInformation> mandatoryResourceEntry :
- ResourceInformation.MANDATORY_RESOURCES.entrySet()) {
+ ResourceInformation.SPECIAL_RESOURCES.entrySet()) {
String key = mandatoryResourceEntry.getKey();
ResourceInformation mandatoryRI = mandatoryResourceEntry.getValue();
@@ -121,6 +135,13 @@ public class ResourceUtils {
}
}
+ /**
+ * Ensures that {@link ResourceUtils#MEMORY} and {@link ResourceUtils#VCORES}
+ * resources are contained in the map received as parameter.
+ *
+ * @param res Map object having keys as resources names
+ * and {@link ResourceInformation} objects as values
+ */
private static void addMandatoryResources(
Map<String, ResourceInformation> res) {
ResourceInformation ri;
@@ -258,7 +279,7 @@ public class ResourceUtils {
validateNameOfResourceNameAndThrowException(name);
}
- checkMandatoryResources(resourceInformationMap);
+ checkSpecialResources(resourceInformationMap);
addMandatoryResources(resourceInformationMap);
setAllocationForMandatoryResources(resourceInformationMap, conf);
@@ -487,7 +508,7 @@ public class ResourceUtils {
if (!initializedNodeResources) {
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
conf);
- checkMandatoryResources(nodeResources);
+ checkSpecialResources(nodeResources);
addMandatoryResources(nodeResources);
setAllocationForMandatoryResources(nodeResources, conf);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org