You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2020/12/10 18:16:38 UTC
[hadoop] branch branch-2.10 updated: YARN-9785. Fix
DominantResourceCalculator when one resource is zero. Contributed by Bibin
A Chundatt, Sunil Govindan, Bilwa S T.
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 406957f YARN-9785. Fix DominantResourceCalculator when one resource is zero. Contributed by Bibin A Chundatt, Sunil Govindan, Bilwa S T.
406957f is described below
commit 406957f5e8146ae215de8c904bbb5d7df5fd65a4
Author: Zhankun Tang <zt...@apache.org>
AuthorDate: Wed Sep 4 12:05:29 2019 +0800
YARN-9785. Fix DominantResourceCalculator when one resource is zero. Contributed by Bibin A Chundatt, Sunil Govindan, Bilwa S T.
(cherry picked from commit fff4fbc9576d393a57489f3cd40770ec882f25dc)
---
.../util/resource/DominantResourceCalculator.java | 28 ++++++++++++++++++++--
.../yarn/util/resource/TestResourceCalculator.java | 23 ++++++++++++++++++
2 files changed, 49 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 2191005..23a3a86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -103,7 +103,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
return 0;
}
- if (isInvalidDivisor(clusterResource)) {
+ if (isAllInvalidDivisor(clusterResource)) {
return this.compare(lhs, rhs);
}
@@ -280,6 +280,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
firstShares[i] = calculateShare(clusterRes[i], firstRes[i]);
secondShares[i] = calculateShare(clusterRes[i], secondRes[i]);
+ if (firstShares[i] == Float.POSITIVE_INFINITY ||
+ secondShares[i] == Float.POSITIVE_INFINITY) {
+ continue;
+ }
+
if (firstShares[i] > max[0]) {
max[0] = firstShares[i];
}
@@ -298,7 +303,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
*/
private double calculateShare(ResourceInformation clusterRes,
ResourceInformation res) {
- // Convert the resources' units into the cluster resource's units
+ if (clusterRes.getValue() == 0) {
+ return Float.POSITIVE_INFINITY;
+ }
+ // Convert the resources' units into the cluster resource's units
long value = UnitsConversionUtil.convert(res.getUnits(),
clusterRes.getUnits(), res.getValue());
@@ -321,6 +329,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
// lhsShares and rhsShares must necessarily have the same length, because
// everyone uses the same master resource list.
for (int i = lhsShares.length - 1; i >= 0; i--) {
+ if (lhsShares[i] == Float.POSITIVE_INFINITY ||
+ rhsShares[i] == Float.POSITIVE_INFINITY) {
+ continue;
+ }
diff = lhsShares[i] - rhsShares[i];
if (diff != 0.0) {
@@ -380,6 +392,18 @@ public class DominantResourceCalculator extends ResourceCalculator {
return false;
}
+ public boolean isAllInvalidDivisor(Resource r) {
+ boolean flag = true;
+ for (ResourceInformation res : r.getResources()) {
+ if (flag == true && res.getValue() == 0L) {
+ flag = true;
+ continue;
+ }
+ flag = false;
+ }
+ return flag;
+ }
+
@Override
public float ratio(Resource a, Resource b) {
float ratio = 0.0f;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
index 5b4155c..13b0dd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -188,6 +188,7 @@ public class TestResourceCalculator {
testCompareDefault(cluster);
} else if (resourceCalculator instanceof DominantResourceCalculator) {
testCompareDominant(cluster);
+ testCompareDominantZeroValueResource();
}
}
@@ -203,6 +204,28 @@ public class TestResourceCalculator {
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 0, 0), 1);
}
+ /**
+ * Verify compare when one or all the resource are zero.
+ */
+ private void testCompareDominantZeroValueResource(){
+ Resource cluster = newResource(4L, 4, 0);
+ assertComparison(cluster, newResource(2, 1, 1), newResource(1, 1, 2), 1);
+ assertComparison(cluster, newResource(2, 2, 1), newResource(1, 2, 2), 1);
+ assertComparison(cluster, newResource(2, 2, 1), newResource(2, 2, 2), 0);
+ assertComparison(cluster, newResource(0, 2, 1), newResource(0, 2, 2), 0);
+ assertComparison(cluster, newResource(0, 1, 2), newResource(1, 1, 2), -1);
+ assertComparison(cluster, newResource(1, 1, 2), newResource(2, 1, 2), -1);
+
+ // cluster resource zero
+ cluster = newResource(0, 0, 0);
+ assertComparison(cluster, newResource(2, 1, 1), newResource(1, 1, 1), 1);
+ assertComparison(cluster, newResource(2, 2, 2), newResource(1, 1, 1), 1);
+ assertComparison(cluster, newResource(2, 1, 1), newResource(1, 2, 1), 0);
+ assertComparison(cluster, newResource(1, 1, 1), newResource(1, 1, 1), 0);
+ assertComparison(cluster, newResource(1, 1, 1), newResource(1, 1, 2), -1);
+ assertComparison(cluster, newResource(1, 1, 1), newResource(1, 2, 1), -1);
+ }
+
private void testCompareDominant(Resource cluster) {
assertComparison(cluster, newResource(2, 1, 1), newResource(2, 1, 1), 0);
assertComparison(cluster, newResource(2, 1, 1), newResource(1, 2, 1), 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org