You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/09/19 22:38:16 UTC
[hadoop] branch branch-2 updated: YARN-7410. Cleanup
FixedValueResource to avoid dependency to ResourceUtils. (wangda)
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 727cbcb YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. (wangda)
727cbcb is described below
commit 727cbcb41dd1c6e85f6a451b4fc5057b6c87628b
Author: Wangda Tan <wa...@apache.org>
AuthorDate: Mon Nov 6 15:26:49 2017 -0800
YARN-7410. Cleanup FixedValueResource to avoid dependency to ResourceUtils. (wangda)
(cherry picked from commit df27824b80b8eca03d02837efe2da31f089e67ec)
---
.../apache/hadoop/yarn/api/records/Resource.java | 2 +-
.../hadoop/yarn/util/resource/ResourceUtils.java | 1 +
.../hadoop/yarn/util/resource/Resources.java | 59 ++++++++++++++++++++++
3 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index f7c699f..ce3ea13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -374,7 +374,7 @@ public abstract class Resource implements Comparable<Resource> {
}
}
- private void throwExceptionWhenArrayOutOfBound(int index) {
+ protected void throwExceptionWhenArrayOutOfBound(int index) {
String exceptionMsg = String.format(
"Trying to access ResourceInformation for given index=%d. "
+ "Acceptable index range is [0,%d), please check double check "
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index a359ad1..4c65cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -290,6 +290,7 @@ public class ResourceUtils {
updateKnownResources();
updateResourceTypeIndex();
initializedResources = true;
+ numKnownResourceTypes = resourceTypes.size();
}
private static void updateKnownResources() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 325bce4..11e53ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -125,6 +125,65 @@ public class Resources {
throw new RuntimeException(name + " cannot be modified!");
}
+ /*
+ * FixedValueResource cannot be updated when any resource types refresh
+ * by using approach introduced by YARN-7307 and do operations like
+ * Resources.compare(resource_x, Resources.none()) will throw exceptions.
+ *
+ * That's why we do reinitialize resource maps for following methods.
+ */
+
+ @Override
+ public ResourceInformation getResourceInformation(int index)
+ throws ResourceNotFoundException {
+ ResourceInformation ri = null;
+ try {
+ ri = super.getResourceInformation(index);
+ } catch (ResourceNotFoundException e) {
+ // Retry once to reinitialize resource information.
+ initResourceMap();
+ try {
+ return super.getResourceInformation(index);
+ } catch (ResourceNotFoundException ee) {
+ throwExceptionWhenArrayOutOfBound(index);
+ }
+ }
+ return ri;
+ }
+
+ @Override
+ public ResourceInformation getResourceInformation(String resource)
+ throws ResourceNotFoundException {
+ ResourceInformation ri;
+ try {
+ ri = super.getResourceInformation(resource);
+ } catch (ResourceNotFoundException e) {
+ // Retry once to reinitialize resource information.
+ initResourceMap();
+ try {
+ return super.getResourceInformation(resource);
+ } catch (ResourceNotFoundException ee) {
+ throw ee;
+ }
+ }
+ return ri;
+ }
+
+ @Override
+ public ResourceInformation[] getResources() {
+ if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) {
+ // Retry once to reinitialize resource information.
+ initResourceMap();
+ if (resources.length != ResourceUtils.getNumberOfKnownResourceTypes()) {
+ throw new ResourceNotFoundException("Failed to reinitialize "
+ + "FixedValueResource to get number of resource types same "
+ + "as configured");
+ }
+ }
+
+ return resources;
+ }
+
private void initResourceMap() {
ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
if (types != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org