You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/08/02 16:59:51 UTC
hadoop git commit: YARN-7159. Normalize unit of resource objects in
RM to avoid unit conversion in critical path. Contributed by Manikandan R.
Repository: hadoop
Updated Branches:
refs/heads/trunk 7526815e3 -> 12a095a49
YARN-7159. Normalize unit of resource objects in RM to avoid unit conversion in critical path. Contributed by Manikandan R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12a095a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12a095a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12a095a4
Branch: refs/heads/trunk
Commit: 12a095a496dd59066d73a7a6c24129b5b6a9d650
Parents: 7526815
Author: Sunil G <su...@apache.org>
Authored: Thu Aug 2 22:29:21 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Aug 2 22:29:21 2018 +0530
----------------------------------------------------------------------
.../yarn/api/records/ResourceInformation.java | 2 +-
.../yarn/util/resource/ResourceUtils.java | 7 ++
.../yarn/conf/TestResourceInformation.java | 2 +-
.../api/records/impl/pb/ResourcePBImpl.java | 14 ++-
.../resource/DominantResourceCalculator.java | 65 +++-----------
.../hadoop/yarn/util/resource/Resources.java | 43 ++--------
.../hadoop/yarn/api/TestResourcePBImpl.java | 90 ++++++++++++++++++++
.../yarn/util/resource/TestResourceUtils.java | 40 +++++++++
.../resource-types/node-resources-3.xml | 33 +++++++
.../resourcemanager/TestClientRMService.java | 88 +++++++++++++++++++
.../fair/TestFairSchedulerConfiguration.java | 2 +-
11 files changed, 291 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 904ff4b..c83c3a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -202,7 +202,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
ResourceInformation ret = new ResourceInformation();
ret.setName(name);
ret.setResourceType(type);
- ret.setUnits(units);
+ ret.setUnitsWithoutValidation(units);
ret.setValue(value);
ret.setMinimumAllocation(minimumAllocation);
ret.setMaximumAllocation(maximumAllocation);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 3dbd609..c2d7201 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -527,6 +528,12 @@ public class ResourceUtils {
String units = getUnits(value);
Long resourceValue =
Long.valueOf(value.substring(0, value.length() - units.length()));
+ String destUnit = getDefaultUnit(resourceType);
+ if(!units.equals(destUnit)) {
+ resourceValue = UnitsConversionUtil.convert(
+ units, destUnit, resourceValue);
+ units = destUnit;
+ }
nodeResources.get(resourceType).setValue(resourceValue);
nodeResources.get(resourceType).setUnits(units);
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java
index 66bf320..c342dbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestResourceInformation.java
@@ -43,7 +43,7 @@ public class TestResourceInformation {
Assert.assertEquals("Resource units incorrect", units, ri.getUnits());
units = "z";
try {
- ResourceInformation.newInstance(name, units);
+ ResourceInformation.newInstance(name, units).setUnits(units);
Assert.fail(units + "is not a valid unit");
} catch (IllegalArgumentException ie) {
// do nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 15d2470..144f48f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -173,9 +173,17 @@ public class ResourcePBImpl extends Resource {
ri.setResourceType(entry.hasType()
? ProtoUtils.convertFromProtoFormat(entry.getType())
: ResourceTypes.COUNTABLE);
- ri.setUnits(
- entry.hasUnits() ? entry.getUnits() : resourceInformation.getUnits());
- ri.setValue(entry.hasValue() ? entry.getValue() : 0L);
+ String units = entry.hasUnits() ? entry.getUnits() :
+ ResourceUtils.getDefaultUnit(entry.getKey());
+ long value = entry.hasValue() ? entry.getValue() : 0L;
+ String destUnit = ResourceUtils.getDefaultUnit(entry.getKey());
+ if(!units.equals(destUnit)) {
+ ri.setValue(UnitsConversionUtil.convert(units, destUnit, value));
+ ri.setUnits(destUnit);
+ } else {
+ ri.setUnits(units);
+ ri.setValue(value);
+ }
return ri;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 2e85ebc..9aeb51c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -298,11 +297,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
*/
private double calculateShare(ResourceInformation clusterRes,
ResourceInformation res) {
- // Convert the resources' units into the cluster resource's units
- long value = UnitsConversionUtil.convert(res.getUnits(),
- clusterRes.getUnits(), res.getValue());
-
- return (double) value / clusterRes.getValue();
+ return (double) res.getValue() / clusterRes.getValue();
}
/**
@@ -340,11 +335,8 @@ public class DominantResourceCalculator extends ResourceCalculator {
ResourceInformation availableResource = available
.getResourceInformation(i);
ResourceInformation requiredResource = required.getResourceInformation(i);
- long requiredResourceValue = UnitsConversionUtil.convert(
- requiredResource.getUnits(), availableResource.getUnits(),
- requiredResource.getValue());
- if (requiredResourceValue != 0) {
- long tmp = availableResource.getValue() / requiredResourceValue;
+ if (requiredResource.getValue() != 0) {
+ long tmp = availableResource.getValue() / requiredResource.getValue();
min = min < tmp ? min : tmp;
}
}
@@ -387,11 +379,8 @@ public class DominantResourceCalculator extends ResourceCalculator {
for (int i = 0; i < maxLength; i++) {
ResourceInformation aResourceInformation = a.getResourceInformation(i);
ResourceInformation bResourceInformation = b.getResourceInformation(i);
- long bResourceValue = UnitsConversionUtil.convert(
- bResourceInformation.getUnits(), aResourceInformation.getUnits(),
- bResourceInformation.getValue());
float tmp = (float) aResourceInformation.getValue()
- / (float) bResourceValue;
+ / (float) bResourceInformation.getValue();
ratio = ratio > tmp ? ratio : tmp;
}
return ratio;
@@ -437,23 +426,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
ResourceInformation tmp = ret.getResourceInformation(i);
long rValue = rResourceInformation.getValue();
- long minimumValue = UnitsConversionUtil.convert(
- minimumResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- minimumResourceInformation.getValue());
- long maximumValue = UnitsConversionUtil.convert(
- maximumResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- maximumResourceInformation.getValue());
- long stepFactorValue = UnitsConversionUtil.convert(
- stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
- long value = Math.max(rValue, minimumValue);
- if (stepFactorValue != 0) {
- value = roundUp(value, stepFactorValue);
+ long value = Math.max(rValue, minimumResourceInformation.getValue());
+ if (stepFactorResourceInformation.getValue() != 0) {
+ value = roundUp(value, stepFactorResourceInformation.getValue());
}
- tmp.setValue(Math.min(value, maximumValue));
+ tmp.setValue(Math.min(value, maximumResourceInformation.getValue()));
ret.setResourceInformation(i, tmp);
}
return ret;
@@ -478,10 +455,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
.getResourceInformation(i);
long rValue = rResourceInformation.getValue();
- long stepFactorValue = UnitsConversionUtil.convert(
- stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
+ long stepFactorValue = stepFactorResourceInformation.getValue();
long value = rValue;
if (stepFactorValue != 0) {
value = roundUp
@@ -506,10 +480,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
.getResourceInformation(i);
long rValue = rResourceInformation.getValue();
- long stepFactorValue = UnitsConversionUtil.convert(
- stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
+ long stepFactorValue = stepFactorResourceInformation.getValue();
ret.setResourceValue(i, ResourceCalculator
.roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue));
}
@@ -539,10 +510,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
ResourceInformation tmp = ret.getResourceInformation(i);
long rValue = rResourceInformation.getValue();
- long stepFactorValue = UnitsConversionUtil.convert(
- stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
+ long stepFactorValue = stepFactorResourceInformation.getValue();
long value;
if (stepFactorValue != 0) {
value = roundUp
@@ -566,10 +534,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
.getResourceInformation(i);
ResourceInformation bResourceInformation = bigger
.getResourceInformation(i);
- long sResourceValue = UnitsConversionUtil.convert(
- sResourceInformation.getUnits(), bResourceInformation.getUnits(),
- sResourceInformation.getValue());
- if (sResourceValue > bResourceInformation.getValue()) {
+ if (sResourceInformation.getValue() > bResourceInformation.getValue()) {
return false;
}
}
@@ -587,11 +552,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
ResourceInformation tmp = ret.getResourceInformation(i);
long rValue = rResourceInformation.getValue();
- long stepFactorValue = UnitsConversionUtil.convert(
- stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
-
+ long stepFactorValue = stepFactorResourceInformation.getValue();
long value = rValue;
if (stepFactorValue != 0) {
value = roundDown(rValue, stepFactorValue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 8636577..48c2c36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
/**
* Resources is a computation class which provides a set of apis to do
@@ -257,12 +256,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
- long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
- ? rhsValue.getValue()
- : UnitsConversionUtil.convert(rhsValue.getUnits(),
- lhsValue.getUnits(), rhsValue.getValue());
- lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
+ lhs.setResourceValue(i, lhsValue.getValue() + rhsValue.getValue());
} catch (ResourceNotFoundException ye) {
LOG.warn("Resource is missing:" + ye.getMessage());
continue;
@@ -281,12 +275,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
- long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
- ? rhsValue.getValue()
- : UnitsConversionUtil.convert(rhsValue.getUnits(),
- lhsValue.getUnits(), rhsValue.getValue());
- lhs.setResourceValue(i, lhsValue.getValue() - convertedRhs);
+ lhs.setResourceValue(i, lhsValue.getValue() - rhsValue.getValue());
} catch (ResourceNotFoundException ye) {
LOG.warn("Resource is missing:" + ye.getMessage());
continue;
@@ -365,12 +354,7 @@ public class Resources {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
ResourceInformation lhsValue = lhs.getResourceInformation(i);
- long convertedRhs = (long) (((rhsValue.getUnits()
- .equals(lhsValue.getUnits()))
- ? rhsValue.getValue()
- : UnitsConversionUtil.convert(rhsValue.getUnits(),
- lhsValue.getUnits(), rhsValue.getValue()))
- * by);
+ long convertedRhs = (long) (rhsValue.getValue() * by);
lhs.setResourceValue(i, lhsValue.getValue() + convertedRhs);
} catch (ResourceNotFoundException ye) {
LOG.warn("Resource is missing:" + ye.getMessage());
@@ -511,12 +495,7 @@ public class Resources {
try {
ResourceInformation rhsValue = bigger.getResourceInformation(i);
ResourceInformation lhsValue = smaller.getResourceInformation(i);
-
- long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
- ? rhsValue.getValue()
- : UnitsConversionUtil.convert(rhsValue.getUnits(),
- lhsValue.getUnits(), rhsValue.getValue());
- if (lhsValue.getValue() > convertedRhs) {
+ if (lhsValue.getValue() > rhsValue.getValue()) {
return false;
}
} catch (ResourceNotFoundException ye) {
@@ -539,12 +518,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
- long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
- ? rhsValue.getValue()
- : UnitsConversionUtil.convert(rhsValue.getUnits(),
- lhsValue.getUnits(), rhsValue.getValue());
- ResourceInformation outInfo = lhsValue.getValue() < convertedRhs
+ ResourceInformation outInfo = lhsValue.getValue() < rhsValue.getValue()
? lhsValue
: rhsValue;
ret.setResourceInformation(i, outInfo);
@@ -563,12 +537,7 @@ public class Resources {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
ResourceInformation lhsValue = lhs.getResourceInformation(i);
-
- long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
- ? rhsValue.getValue()
- : UnitsConversionUtil.convert(rhsValue.getUnits(),
- lhsValue.getUnits(), rhsValue.getValue());
- ResourceInformation outInfo = lhsValue.getValue() > convertedRhs
+ ResourceInformation outInfo = lhsValue.getValue() > rhsValue.getValue()
? lhsValue
: rhsValue;
ret.setResourceInformation(i, outInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
index 4887b50..5ab528b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourcePBImpl.java
@@ -18,11 +18,19 @@
package org.apache.hadoop.yarn.api;
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.util.resource.TestResourceUtils;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -31,6 +39,27 @@ import static org.junit.Assert.assertEquals;
* Test class to handle various proto related tests for resources.
*/
public class TestResourcePBImpl {
+
+ @Before
+ public void setup() throws Exception {
+ ResourceUtils.resetResourceTypes();
+
+ String resourceTypesFile = "resource-types-4.xml";
+ Configuration conf = new YarnConfiguration();
+ TestResourceUtils.setupResourceTypes(conf, resourceTypesFile);
+ }
+
+ @After
+ public void teardown() {
+ Configuration conf = new YarnConfiguration();
+ File source = new File(
+ conf.getClassLoader().getResource("resource-types-4.xml").getFile());
+ File dest = new File(source.getParent(), "resource-types.xml");
+ if (dest.exists()) {
+ dest.delete();
+ }
+ }
+
@Test
public void testEmptyResourcePBInit() throws Exception {
Resource res = new ResourcePBImpl();
@@ -85,4 +114,65 @@ public class TestResourcePBImpl {
assertEquals("Cast to Integer.MAX_VALUE if the long is greater than "
+ "Integer.MAX_VALUE", Integer.MAX_VALUE, res.getVirtualCores());
}
+
+ @Test
+ public void testResourcePBWithExtraResources() throws Exception {
+
+ //Resource 'resource1' has been passed as 4T
+ //4T should be converted to 4000G
+ YarnProtos.ResourceInformationProto riProto =
+ YarnProtos.ResourceInformationProto.newBuilder().setType(
+ YarnProtos.ResourceTypeInfoProto.newBuilder().
+ setName("resource1").setType(
+ YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
+ setValue(4).setUnits("T").setKey("resource1").build();
+
+ YarnProtos.ResourceProto proto =
+ YarnProtos.ResourceProto.newBuilder().setMemory(1024).
+ setVirtualCores(3).addResourceValueMap(riProto).build();
+ Resource res = new ResourcePBImpl(proto);
+
+ Assert.assertEquals(4000,
+ res.getResourceInformation("resource1").getValue());
+ Assert.assertEquals("G",
+ res.getResourceInformation("resource1").getUnits());
+
+ //Resource 'resource2' has been passed as 4M
+ //4M should be converted to 4000000000m
+ YarnProtos.ResourceInformationProto riProto1 =
+ YarnProtos.ResourceInformationProto.newBuilder().setType(
+ YarnProtos.ResourceTypeInfoProto.newBuilder().
+ setName("resource2").setType(
+ YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
+ setValue(4).setUnits("M").setKey("resource2").build();
+
+ YarnProtos.ResourceProto proto1 =
+ YarnProtos.ResourceProto.newBuilder().setMemory(1024).
+ setVirtualCores(3).addResourceValueMap(riProto1).build();
+ Resource res1 = new ResourcePBImpl(proto1);
+
+ Assert.assertEquals(4000000000L,
+ res1.getResourceInformation("resource2").getValue());
+ Assert.assertEquals("m",
+ res1.getResourceInformation("resource2").getUnits());
+
+ //Resource 'resource1' has been passed as 3M
+ //3M should be converted to 0G
+ YarnProtos.ResourceInformationProto riProto2 =
+ YarnProtos.ResourceInformationProto.newBuilder().setType(
+ YarnProtos.ResourceTypeInfoProto.newBuilder().
+ setName("resource1").setType(
+ YarnProtos.ResourceTypesProto.COUNTABLE).getType()).
+ setValue(3).setUnits("M").setKey("resource1").build();
+
+ YarnProtos.ResourceProto proto2 =
+ YarnProtos.ResourceProto.newBuilder().setMemory(1024).
+ setVirtualCores(3).addResourceValueMap(riProto2).build();
+ Resource res2 = new ResourcePBImpl(proto2);
+
+ Assert.assertEquals(0,
+ res2.getResourceInformation("resource1").getValue());
+ Assert.assertEquals("G",
+ res2.getResourceInformation("resource1").getUnits());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 2671de8..9b48017 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -377,6 +377,46 @@ public class TestResourceUtils {
}
}
+ @Test
+ public void testGetResourceInformationWithDiffUnits() throws Exception {
+
+ Configuration conf = new YarnConfiguration();
+ Map<String, Resource> testRun = new HashMap<>();
+ setupResourceTypes(conf, "resource-types-4.xml");
+ Resource test3Resources = Resource.newInstance(0, 0);
+
+ //Resource 'resource1' has been passed as 5T
+ //5T should be converted to 5000G
+ test3Resources.setResourceInformation("resource1",
+ ResourceInformation.newInstance("resource1", "T", 5L));
+
+ //Resource 'resource2' has been passed as 2M
+ //2M should be converted to 2000000000m
+ test3Resources.setResourceInformation("resource2",
+ ResourceInformation.newInstance("resource2", "M", 2L));
+ test3Resources.setResourceInformation("yarn.io/gpu",
+ ResourceInformation.newInstance("yarn.io/gpu", "", 1));
+ testRun.put("node-resources-3.xml", test3Resources);
+
+ for (Map.Entry<String, Resource> entry : testRun.entrySet()) {
+ String resourceFile = entry.getKey();
+ ResourceUtils.resetNodeResources();
+ File dest;
+ File source = new File(
+ conf.getClassLoader().getResource(resourceFile).getFile());
+ dest = new File(source.getParent(), "node-resources.xml");
+ FileUtils.copyFile(source, dest);
+ Map<String, ResourceInformation> actual = ResourceUtils
+ .getNodeResourceInformation(conf);
+ Assert.assertEquals(actual.size(),
+ entry.getValue().getResources().length);
+ for (ResourceInformation resInfo : entry.getValue().getResources()) {
+ Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+ }
+ dest.delete();
+ }
+ }
+
public static String setupResourceTypes(Configuration conf, String filename)
throws Exception {
File source = new File(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml
new file mode 100644
index 0000000..23f6a6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/resources/resource-types/node-resources-3.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+ <property>
+ <name>yarn.nodemanager.resource-type.resource1</name>
+ <value>5T</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource-type.resource2</name>
+ <value>2M</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource-type.yarn.io/gpu</name>
+ <value>1</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index d66a866..6644e44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -33,6 +33,7 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
@@ -49,6 +50,7 @@ import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CyclicBarrier;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -157,7 +159,9 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.UTCClock;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Assume;
@@ -2243,4 +2247,88 @@ public class TestClientRMService {
rmService.getApplications(request).getApplicationList().size());
rmService.setDisplayPerUserApps(false);
}
+
+ @Test
+ public void testRegisterNMWithDiffUnits() throws Exception {
+ ResourceUtils.resetResourceTypes();
+ Configuration yarnConf = new YarnConfiguration();
+ String resourceTypesFile = "resource-types-4.xml";
+ InputStream source =
+ yarnConf.getClassLoader().getResourceAsStream(resourceTypesFile);
+ File dest = new File(yarnConf.getClassLoader().
+ getResource(".").getPath(), "resource-types.xml");
+ FileUtils.copyInputStreamToFile(source, dest);
+ ResourceUtils.getResourceTypes();
+
+ yarnConf.setClass(
+ CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+
+ MockRM rm = new MockRM(yarnConf) {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(this.rmContext, scheduler,
+ this.rmAppManager, this.applicationACLsManager, this.queueACLsManager,
+ this.getRMContext().getRMDelegationTokenSecretManager());
+ };
+ };
+ rm.start();
+
+ Resource resource = BuilderUtils.newResource(1024, 1);
+ resource.setResourceInformation("memory-mb",
+ ResourceInformation.newInstance("memory-mb", "G", 1024));
+ resource.setResourceInformation("resource1",
+ ResourceInformation.newInstance("resource1", "T", 1));
+ resource.setResourceInformation("resource2",
+ ResourceInformation.newInstance("resource2", "M", 1));
+
+ MockNM node = rm.registerNode("host1:1234", resource);
+ node.nodeHeartbeat(true);
+
+ // Create a client.
+ Configuration conf = new Configuration();
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
+ LOG.info("Connecting to ResourceManager at " + rmAddress);
+ ApplicationClientProtocol client =
+ (ApplicationClientProtocol) rpc
+ .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
+
+ // Make call
+ GetClusterNodesRequest request =
+ GetClusterNodesRequest.newInstance(EnumSet.of(NodeState.RUNNING));
+ List<NodeReport> nodeReports =
+ client.getClusterNodes(request).getNodeReports();
+ Assert.assertEquals(1, nodeReports.size());
+ Assert.assertNotSame("Node is expected to be healthy!", NodeState.UNHEALTHY,
+ nodeReports.get(0).getNodeState());
+ Assert.assertEquals(1, nodeReports.size());
+
+ //Resource 'resource1' has been passed as 1T while registering NM.
+ //1T should be converted to 1000G
+ Assert.assertEquals("G", nodeReports.get(0).getCapability().
+ getResourceInformation("resource1").getUnits());
+ Assert.assertEquals(1000, nodeReports.get(0).getCapability().
+ getResourceInformation("resource1").getValue());
+
+ //Resource 'resource2' has been passed as 1M while registering NM
+ //1M should be converted to 1000000000M
+ Assert.assertEquals("m", nodeReports.get(0).getCapability().
+ getResourceInformation("resource2").getUnits());
+ Assert.assertEquals(1000000000, nodeReports.get(0).getCapability().
+ getResourceInformation("resource2").getValue());
+
+ //Resource 'memory-mb' has been passed as 1024G while registering NM
+ //1024G should be converted to 976562Mi
+ Assert.assertEquals("Mi", nodeReports.get(0).getCapability().
+ getResourceInformation("memory-mb").getUnits());
+ Assert.assertEquals(976562, nodeReports.get(0).getCapability().
+ getResourceInformation("memory-mb").getValue());
+
+ rpc.stopProxy(client, conf);
+ rm.close();
+
+ if (dest.exists()) {
+ dest.delete();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12a095a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 70f83ab..69d3ab9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -414,7 +414,7 @@ public class TestFairSchedulerConfiguration {
calculator.normalize(customResource(10000L, ""), min, max, increment)
.getResourceInformation(A_CUSTOM_RESOURCE));
assertEquals(customResourceInformation(20000L, ""),
- calculator.normalize(customResource(10001L, ""), min, max, increment)
+ calculator.normalize(customResource(19999L, ""), min, max, increment)
.getResourceInformation(A_CUSTOM_RESOURCE));
assertEquals(customResourceInformation(10L, "k"),
calculator.normalize(customResource(9L, "k"), min, max, increment)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org