You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/03/21 00:47:00 UTC
[hadoop] 04/06: YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 26e1c49d28cc111f8e3f7266b71b24c67f453f3a
Author: Sunil G <su...@apache.org>
AuthorDate: Wed Dec 13 22:00:07 2017 +0530
YARN-7383. Node resource is not parsed correctly for resource names containing dot. Contributed by Gergely Novák.
---
.../apache/hadoop/yarn/util/resource/ResourceUtils.java | 15 ++++++---------
.../hadoop/yarn/util/resource/TestResourceUtils.java | 5 ++++-
.../test/resources/resource-types/node-resources-2.xml | 5 +++++
.../test/resources/resource-types/resource-types-4.xml | 7 ++++++-
4 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index abf58a6..65eb5a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -461,21 +461,18 @@ public class ResourceUtils {
for (Map.Entry<String, String> entry : conf) {
String key = entry.getKey();
String value = entry.getValue();
-
- if (key.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
- addResourceInformation(key, value, nodeResources);
- }
+ addResourceTypeInformation(key, value, nodeResources);
}
return nodeResources;
}
- private static void addResourceInformation(String prop, String value,
+ private static void addResourceTypeInformation(String prop, String value,
Map<String, ResourceInformation> nodeResources) {
- String[] parts = prop.split("\\.");
- LOG.info("Found resource entry " + prop);
- if (parts.length == 4) {
- String resourceType = parts[3];
+ if (prop.startsWith(YarnConfiguration.NM_RESOURCES_PREFIX)) {
+ LOG.info("Found resource entry " + prop);
+ String resourceType = prop.substring(
+ YarnConfiguration.NM_RESOURCES_PREFIX.length());
if (!nodeResources.containsKey(resourceType)) {
nodeResources
.put(resourceType, ResourceInformation.newInstance(resourceType));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 80555ca..b511705 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -124,9 +124,10 @@ public class TestResourceUtils {
new ResourceFileInformation("resource-types-3.xml", 3);
testFile3.resourceNameUnitsMap.put("resource2", "");
ResourceFileInformation testFile4 =
- new ResourceFileInformation("resource-types-4.xml", 4);
+ new ResourceFileInformation("resource-types-4.xml", 5);
testFile4.resourceNameUnitsMap.put("resource1", "G");
testFile4.resourceNameUnitsMap.put("resource2", "m");
+ testFile4.resourceNameUnitsMap.put("yarn.io/gpu", "");
ResourceFileInformation[] tests = {testFile1, testFile2, testFile3,
testFile4};
@@ -292,6 +293,8 @@ public class TestResourceUtils {
ResourceInformation.newInstance("resource1", "Gi", 5L));
test3Resources.setResourceInformation("resource2",
ResourceInformation.newInstance("resource2", "m", 2L));
+ test3Resources.setResourceInformation("yarn.io/gpu",
+ ResourceInformation.newInstance("yarn.io/gpu", "", 1));
testRun.put("node-resources-2.xml", test3Resources);
for (Map.Entry<String, Resource> entry : testRun.entrySet()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
index 9d9b3dc..382d5dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-2.xml
@@ -36,4 +36,9 @@ limitations under the License. See accompanying LICENSE file.
<value>2m</value>
</property>
+ <property>
+ <name>yarn.nodemanager.resource-type.yarn.io/gpu</name>
+ <value>1</value>
+ </property>
+
</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
index c84316a..ea8d2bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-4.xml
@@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
<property>
<name>yarn.resource-types</name>
- <value>resource1,resource2</value>
+ <value>resource1,resource2,yarn.io/gpu</value>
</property>
<property>
@@ -31,4 +31,9 @@ limitations under the License. See accompanying LICENSE file.
<value>m</value>
</property>
+ <property>
+ <name>yarn.resource-types.yarn.io/gpu.units</name>
+ <value></value>
+ </property>
+
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org