You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/09/12 20:41:40 UTC
[25/50] [abbrv] hadoop git commit: YARN-6933. [YARN-3926]
ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan
R.
YARN-6933. [YARN-3926] ResourceUtils.DISALLOWED_NAMES check is duplicated. Contributed by Manikandan R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80509549
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80509549
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80509549
Branch: refs/heads/YARN-5972
Commit: 805095496dfd8e50f71b70cf20845e954d3ba47c
Parents: 53df3ea
Author: Sunil G <su...@apache.org>
Authored: Wed Sep 6 18:51:14 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 12 09:19:12 2017 -0700
----------------------------------------------------------------------
.../yarn/util/resource/ResourceUtils.java | 38 ++++++++++----------
.../resource-types/resource-types-error-2.xml | 4 +++
.../resource-types/resource-types-error-3.xml | 2 +-
3 files changed, 23 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80509549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 94c2e97..e3e25d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -42,10 +42,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -63,13 +61,6 @@ public class ResourceUtils {
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
private static final String VCORES = ResourceInformation.VCORES.getName();
- private static final Set<String> DISALLOWED_NAMES = new HashSet<>();
- static {
- DISALLOWED_NAMES.add("memory");
- DISALLOWED_NAMES.add(MEMORY);
- DISALLOWED_NAMES.add(VCORES);
- }
-
private static volatile boolean initializedResources = false;
private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
new ConcurrentHashMap<String, Integer>();
@@ -85,9 +76,21 @@ public class ResourceUtils {
private ResourceUtils() {
}
- private static void checkMandatatoryResources(
+ private static void checkMandatoryResources(
Map<String, ResourceInformation> resourceInformationMap)
throws YarnRuntimeException {
+ /*
+ * Supporting 'memory' also as invalid resource name, in addition to
+ * 'MEMORY' for historical reasons
+ */
+ String key = "memory";
+ if (resourceInformationMap.containsKey(key)) {
+ LOG.warn("Attempt to define resource '" + key +
+ "', but it is not allowed.");
+ throw new YarnRuntimeException("Attempt to re-define mandatory resource '"
+ + key + "'.");
+ }
+
if (resourceInformationMap.containsKey(MEMORY)) {
ResourceInformation memInfo = resourceInformationMap.get(MEMORY);
String memUnits = ResourceInformation.MEMORY_MB.getUnits();
@@ -113,7 +116,7 @@ public class ResourceUtils {
}
}
- private static void addManadtoryResources(
+ private static void addMandatoryResources(
Map<String, ResourceInformation> res) {
ResourceInformation ri;
if (!res.containsKey(MEMORY)) {
@@ -229,11 +232,6 @@ public class ResourceUtils {
"Incomplete configuration for resource type '" + resourceName
+ "'. One of name, units or type is configured incorrectly.");
}
- if (DISALLOWED_NAMES.contains(resourceName)) {
- throw new YarnRuntimeException(
- "Resource type cannot be named '" + resourceName
- + "'. That name is disallowed.");
- }
ResourceTypes resourceType = ResourceTypes.valueOf(resourceTypeName);
LOG.info("Adding resource type - name = " + resourceName + ", units = "
+ resourceUnits + ", type = " + resourceTypeName);
@@ -246,8 +244,8 @@ public class ResourceUtils {
minimumAllocation, maximumAllocation));
}
}
- checkMandatatoryResources(resourceInformationMap);
- addManadtoryResources(resourceInformationMap);
+ checkMandatoryResources(resourceInformationMap);
+ addMandatoryResources(resourceInformationMap);
setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
@@ -454,8 +452,8 @@ public class ResourceUtils {
if (!initializedNodeResources) {
Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
conf);
- addManadtoryResources(nodeResources);
- checkMandatatoryResources(nodeResources);
+ addMandatoryResources(nodeResources);
+ checkMandatoryResources(nodeResources);
setMinimumAllocationForMandatoryResources(nodeResources, conf);
setMaximumAllocationForMandatoryResources(nodeResources, conf);
readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80509549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
index ca428eb..fa43b6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-2.xml
@@ -26,4 +26,8 @@ limitations under the License. See accompanying LICENSE file.
<value>G</value>
</property>
+ <property>
+ <name>yarn.resource-types.vcores.units</name>
+ <value>Az</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80509549/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
index 08b8a6d..539d657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/resource-types-error-3.xml
@@ -18,7 +18,7 @@ limitations under the License. See accompanying LICENSE file.
<property>
<name>yarn.resource-types</name>
- <value>vcores,resource1</value>
+ <value>resource1,resource1</value>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org