You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2020/05/11 17:37:13 UTC
[hadoop] branch branch-2.10 updated: YARN-9444. YARN API
ResourceUtils's getRequestedResourcesFromConfig doesn't recognize
yarn.io/gpu as a valid resource. Contributed by Gergely Pollak
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new d0b0fb6 YARN-9444. YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource. Contributed by Gergely Pollak
d0b0fb6 is described below
commit d0b0fb6e040c032306823a6956a4d99d3dc7b9d7
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Mon May 11 17:07:35 2020 +0000
YARN-9444. YARN API ResourceUtils's getRequestedResourcesFromConfig doesn't recognize yarn.io/gpu as a valid resource. Contributed by Gergely Pollak
---
.../hadoop/yarn/util/resource/ResourceUtils.java | 4 +-
.../yarn/util/resource/TestResourceUtils.java | 48 ++++++++++++++++++++++
2 files changed, 51 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index ab40223..011bd97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -66,6 +66,8 @@ public class ResourceUtils {
private static final Pattern RESOURCE_REQUEST_VALUE_PATTERN =
Pattern.compile("^([0-9]+) ?([a-zA-Z]*)$");
+ public static final String YARN_IO_OPTIONAL = "(yarn\\.io/)?";
+
private static volatile boolean initializedResources = false;
private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
new ConcurrentHashMap<String, Integer>();
@@ -596,7 +598,7 @@ public class ResourceUtils {
Configuration configuration, String prefix) {
List<ResourceInformation> result = new ArrayList<>();
Map<String, String> customResourcesMap = configuration
- .getValByRegex("^" + Pattern.quote(prefix) + "[^.]+$");
+ .getValByRegex("^" + Pattern.quote(prefix) + YARN_IO_OPTIONAL + "[^.]+$");
for (Entry<String, String> resource : customResourcesMap.entrySet()) {
String resourceName = resource.getKey().substring(prefix.length());
Matcher matcher =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index b511705..c42e037 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -30,8 +30,13 @@ import org.junit.Before;
import org.junit.Test;
import java.io.File;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Test class to verify all resource utility methods.
@@ -154,6 +159,49 @@ public class TestResourceUtils {
}
@Test
+ public void testGetRequestedResourcesFromConfig() {
+ Configuration conf = new Configuration();
+
+ //these resource type configurations should be recognised
+ String propertyPrefix = "mapreduce.mapper.proper.rt.";
+ String[] expectedKeys = {
+ "yarn.io/gpu",
+ "yarn.io/fpga",
+ "yarn.io/anything_without_a_dot",
+ "regular_rt",
+ "regular_rt/with_slash"};
+
+ String[] invalidKeys = {
+ propertyPrefix + "too.many_parts",
+ propertyPrefix + "yarn.notio/gpu",
+ "incorrect.prefix.yarn.io/gpu",
+ propertyPrefix + "yarn.io/",
+ propertyPrefix};
+
+ for (String s : expectedKeys) {
+ //setting the properties which are expected to be in the resource list
+ conf.set(propertyPrefix + s, "42");
+ }
+
+ for (String s : invalidKeys) {
+ //setting the properties which are expected to be in the resource list
+ conf.set(s, "24");
+ }
+
+ List<ResourceInformation> properList =
+ ResourceUtils.getRequestedResourcesFromConfig(conf, propertyPrefix);
+ Set<String> expectedSet =
+ new HashSet<>(Arrays.asList(expectedKeys));
+
+ Assert.assertEquals(properList.size(), expectedKeys.length);
+ for(Iterator<ResourceInformation> propIter = properList.iterator();
+ propIter.hasNext();) {
+ ResourceInformation item = propIter.next();
+ Assert.assertTrue(expectedSet.contains(item.getName()));
+ }
+ }
+
+ @Test
public void testGetResourceTypesConfigErrors() throws Exception {
Configuration conf = new YarnConfiguration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org