You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by te...@apache.org on 2017/10/17 16:26:35 UTC
[23/36] hadoop git commit: YARN-6933. [YARN-3926]
ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan
R.
YARN-6933. [YARN-3926] ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan R.
(cherry picked from commit 805095496dfd8e50f71b70cf20845e954d3ba47c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3becd158
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3becd158
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3becd158
Branch: refs/heads/resource-types
Commit: 3becd15839426dc129486c5bbdb5379c36bcd4f2
Parents: 6fc855b
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 6 18:51:14 2017 +0530
Committer: Daniel Templeton <te...@apache.org>
Committed: Tue Oct 17 09:25:46 2017 -0700
----------------------------------------------------------------------
.../yarn/util/resource/ResourceUtils.java | 38 ++++++++++----------
.../resource-types/resource-types-error-2.xml | 4 +++
.../resource-types/resource-types-error-3.xml | 2 +-
3 files changed, 23 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3becd158/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 8193a20..df17db6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -41,10 +41,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -60,13 +58,6 @@ public class ResourceUtils {
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
private static final String VCORES = ResourceInformation.VCORES.getName();
- private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
- static {
- DISALLOWED_NAMES.add("memory");
- DISALLOWED_NAMES.add(MEMORY);
- DISALLOWED_NAMES.add(VCORES);
- }
-
private static volatile boolean initializedResources = false;
private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
new ConcurrentHashMap<String, Integer>();
@@ -82,9 +73,21 @@ public class ResourceUtils {
private ResourceUtils() {
}
- private static void checkMandatatoryResources(
+ private static void checkMandatoryResources(
Map<String, ResourceInformation> resourceInformationMap)
throws YarnRuntimeException {
+ /*
+ * Supporting 'memory' also as invalid resource name, in addition to
+ * 'MEMORY' for historical reasons
+ */
+ String key = "memory";
+ if (resourceInformationMap.containsKey(key)) {
+ LOG.warn("Attempt to define resource '" + key +
+ "', but it is not allowed.");
+ throw new YarnRuntimeException("Attempt to re-define mandatory resource '"
+ + key + "'.");
+ }
+
if (resourceInformationMap.containsKey(MEMORY)) {
ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
String memUnits = ResourceInformation.MEMORY_MB.getUnits();
@@ -110,7 +113,7 @@ public class ResourceUtils {
}
}
- private static void addManadtoryResources(
+ private static void addMandatoryResources(
Map<String, ResourceInformation> res) {
ResourceInformation ri;
if (!res.containsKey(MEMORY)) {
@@ -150,11 +153,6 @@ public class ResourceUtils {
"Incomplete configuration for resource type '" + resourceName
+ "'. One of name, units or type is configured incorrectly.");
}
- if (DISALLOWED_NAMES.contains(resourceName)) {
- throw new YarnRuntimeException(
- "Resource type cannot be named '" + resourceName
- + "'. That name is disallowed.");
- }
ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
LOG.info("Adding resource type - name = " + resourceName + ", units = "
+ resourceUnits + ", type = " + resourceTypeName);
@@ -166,8 +164,8 @@ public class ResourceUtils {
.newInstance(resourceName, resourceUnits, 0L, resourceType));
}
}
- checkMandatatoryResources(resourceInformationMap);
- addManadtoryResources(resourceInformationMap);
+ checkMandatoryResources(resourceInformationMap);
+ addMandatoryResources(resourceInformationMap);
resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
updateKnownResources();
updateResourceTypeIndex();
@@ -366,8 +364,8 @@ public class ResourceUtils {
if (!initializedNodeResources) {
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
conf);
- addManadtoryResources(nodeResources);
- checkMandatatoryResources(nodeResources);
+ addMandatoryResources(nodeResources);
+ checkMandatoryResources(nodeResources);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
initializedNodeResources = true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3becd158/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
index ca428eb..fa43b6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
@@ -26,4 +26,8 @@ limitations under the License. See accompanying LICENSE file.
<value>G</value>
</property>
+ <property>
+ <name>yarn.resource-types.vcores.units</name>
+ <value>Az</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3becd158/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
index 08b8a6d..539d657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
@@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
<property>
<name>yarn.resource-types</name>
- <value>vcores,resource1</value>
+ <value>resource1,resource1</value>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org