You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/09/12 20:41:23 UTC
[08/50] [abbrv] hadoop git commit: YARN-4829. Add support for binary
units in Resource class.(vvasudev via asuresh)
YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ba69899
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ba69899
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ba69899
Branch: refs/heads/YARN-5972
Commit: 7ba698997b6845b664b2f8c82e384b457c30d176
Parents: ceb12c5
Author: Arun Suresh <as...@apache.org>
Authored: Thu Mar 17 23:50:22 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 12 09:19:10 2017 -0700
----------------------------------------------------------------------
.../yarn/api/records/ResourceInformation.java | 2 +-
.../hadoop/yarn/util/UnitsConversionUtil.java | 45 ++++++++++++++------
.../yarn/util/TestUnitsConversionUtil.java | 17 +++++++-
.../api/records/impl/pb/ResourcePBImpl.java | 2 +-
.../yarn/util/resource/TestResourceUtils.java | 2 +-
5 files changed, 52 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba69899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 80e3192..a17e81b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
private static final String VCORES_URI = "vcores";
public static final ResourceInformation MEMORY_MB =
- ResourceInformation.newInstance(MEMORY_URI, "M");
+ ResourceInformation.newInstance(MEMORY_URI, "Mi");
public static final ResourceInformation VCORES =
ResourceInformation.newInstance(VCORES_URI);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba69899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
index 7785263..47bb3df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
@@ -46,7 +46,8 @@ public class UnitsConversionUtil {
}
private static final String[] UNITS =
- {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"};
+ { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti",
+ "Pi" };
private static final List<String> SORTED_UNITS = Arrays.asList(UNITS);
public static final Set<String> KNOWN_UNITS = createKnownUnitsSet();
private static final Converter PICO =
@@ -65,6 +66,15 @@ public class UnitsConversionUtil {
private static final Converter PETA =
new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L);
+ private static final Converter KILO_BINARY = new Converter(1024L, 1L);
+ private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L);
+ private static final Converter GIGA_BINARY =
+ new Converter(1024L * 1024L * 1024L, 1L);
+ private static final Converter TERA_BINARY =
+ new Converter(1024L * 1024L * 1024L * 1024L, 1L);
+ private static final Converter PETA_BINARY =
+ new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L);
+
private static Set<String> createKnownUnitsSet() {
Set<String> ret = new HashSet<>();
ret.addAll(Arrays.asList(UNITS));
@@ -93,6 +103,16 @@ public class UnitsConversionUtil {
return TERA;
case "P":
return PETA;
+ case "Ki":
+ return KILO_BINARY;
+ case "Mi":
+ return MEGA_BINARY;
+ case "Gi":
+ return GIGA_BINARY;
+ case "Ti":
+ return TERA_BINARY;
+ case "Pi":
+ return PETA_BINARY;
default:
throw new IllegalArgumentException(
"Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS);
@@ -112,28 +132,29 @@ public class UnitsConversionUtil {
if (toUnit == null || fromUnit == null || fromValue == null) {
throw new IllegalArgumentException("One or more arguments are null");
}
- Long tmp;
String overflowMsg =
"Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
+ "' will result in an overflow of Long";
+ if (fromUnit.equals(toUnit)) {
+ return fromValue;
+ }
Converter fc = getConverter(fromUnit);
Converter tc = getConverter(toUnit);
Long numerator = fc.numerator * tc.denominator;
Long denominator = fc.denominator * tc.numerator;
+ Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
if (numerator < denominator) {
- if (!toUnit.equals(fromUnit)) {
- tmp = Long.MAX_VALUE / numerator;
- if (tmp < fromValue) {
- throw new IllegalArgumentException(overflowMsg);
- }
+ if (numeratorMultiplierLimit < fromValue) {
+ throw new IllegalArgumentException(overflowMsg);
}
return (fromValue * numerator) / denominator;
}
- tmp = numerator / denominator;
- if (!toUnit.equals(fromUnit)) {
- if ((Long.MAX_VALUE / tmp) < fromValue) {
- throw new IllegalArgumentException(overflowMsg);
- }
+ if (numeratorMultiplierLimit > fromValue) {
+ return (numerator * fromValue) / denominator;
+ }
+ Long tmp = numerator / denominator;
+ if ((Long.MAX_VALUE / tmp) < fromValue) {
+ throw new IllegalArgumentException(overflowMsg);
}
return fromValue * tmp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba69899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
index 421768f..0f999e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.util;
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.junit.Assert;
import org.junit.Test;
@@ -45,6 +44,7 @@ public class TestUnitsConversionUtil {
fromUnit = "";
Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
UnitsConversionUtil.convert(fromUnit, "k", test));
+
Assert
.assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
UnitsConversionUtil.convert(fromUnit, "M", test));
@@ -63,6 +63,21 @@ public class TestUnitsConversionUtil {
Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
+
+ Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
+ UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
+
+ Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
+ UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
+
+ Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
+ UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
+
+ Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
+ UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
+
+ Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
+ UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba69899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index fc91ec3..174f3cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -95,7 +95,7 @@ public class ResourcePBImpl extends Resource {
initResourcesMap();
ResourceInformation ri =
this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
- return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue());
+ return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba69899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 606e89c..4bb68c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -195,7 +195,7 @@ public class TestResourceUtils {
// we must always have memory and vcores with their fixed units
Assert.assertTrue(ret.containsKey("memory-mb"));
ResourceInformation memInfo = ret.get("memory-mb");
- Assert.assertEquals("M", memInfo.getUnits());
+ Assert.assertEquals("Mi", memInfo.getUnits());
Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
Assert.assertTrue(ret.containsKey("vcores"));
ResourceInformation vcoresInfo = ret.get("vcores");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org