You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2011/12/29 11:27:14 UTC
svn commit: r1225489 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/conf/ src/main/java/org/apache/hadoop/util/
src/test/java/org/apache/hadoop/conf/ src/test/java/org/apache/hadoop/util/
Author: harsh
Date: Thu Dec 29 10:27:14 2011
New Revision: 1225489
URL: http://svn.apache.org/viewvc?rev=1225489&view=rev
Log:
HADOOP-7910. Add Configuration.getLongBytes to handle human readable byte size values. (Sho Shimauchi via harsh)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1225489&r1=1225488&r2=1225489&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Dec 29 10:27:14 2011
@@ -12,6 +12,8 @@ Trunk (unreleased changes)
HADOOP-7875. Add helper class to unwrap protobuf ServiceException.
(suresh)
+ HADOOP-7910. Add Configuration.getLongBytes to handle human readable byte size values. (Sho Shimauchi via harsh)
+
IMPROVEMENTS
HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm)
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1225489&r1=1225488&r2=1225489&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java Thu Dec 29 10:27:14 2011
@@ -737,6 +737,27 @@ public class Configuration implements It
return Long.parseLong(valueString);
}
+ /**
+ * Get the value of the <code>name</code> property as a <code>long</code> or
+ * human readable format. If no such property exists, the provided default
+ * value is returned, or if the specified value is not a valid
+ * <code>long</code> or human readable format, then an error is thrown. You
+ * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
+ * t(tera), p(peta), e(exa)
+ *
+ * @param name property name.
+ * @param defaultValue default value.
+ * @throws NumberFormatException when the value is invalid
+ * @return property value as a <code>long</code>,
+ * or <code>defaultValue</code>.
+ */
+ public long getLongBytes(String name, long defaultValue) {
+ String valueString = getTrimmed(name);
+ if (valueString == null)
+ return defaultValue;
+ return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
+ }
+
private String getHexDigits(String value) {
boolean negative = false;
String str = value;
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1225489&r1=1225488&r2=1225489&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java Thu Dec 29 10:27:14 2011
@@ -661,7 +661,14 @@ public class StringUtils {
if (Character.isDigit(lastchar))
return Long.parseLong(s);
else {
- long prefix = TraditionalBinaryPrefix.valueOf(lastchar).value;
+ long prefix;
+ try {
+ prefix = TraditionalBinaryPrefix.valueOf(lastchar).value;
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Invalid size prefix '" + lastchar
+ + "' in '" + s
+ + "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)");
+ }
long num = Long.parseLong(s.substring(0, lastpos));
if (num > (Long.MAX_VALUE/prefix) || num < (Long.MIN_VALUE/prefix)) {
throw new IllegalArgumentException(s + " does not fit in a Long");
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java?rev=1225489&r1=1225488&r2=1225489&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java Thu Dec 29 10:27:14 2011
@@ -405,12 +405,16 @@ public class TestConfiguration extends T
conf.addResource(fileResource);
assertEquals(20, conf.getInt("test.int1", 0));
assertEquals(20, conf.getLong("test.int1", 0));
+ assertEquals(20, conf.getLongBytes("test.int1", 0));
assertEquals(20, conf.getInt("test.int2", 0));
assertEquals(20, conf.getLong("test.int2", 0));
+ assertEquals(20, conf.getLongBytes("test.int2", 0));
assertEquals(-20, conf.getInt("test.int3", 0));
assertEquals(-20, conf.getLong("test.int3", 0));
+ assertEquals(-20, conf.getLongBytes("test.int3", 0));
assertEquals(-20, conf.getInt("test.int4", 0));
assertEquals(-20, conf.getLong("test.int4", 0));
+ assertEquals(-20, conf.getLongBytes("test.int4", 0));
try {
conf.getInt("test.int5", 0);
fail("Property had invalid int value, but was read successfully.");
@@ -419,6 +423,26 @@ public class TestConfiguration extends T
}
}
+ public void testHumanReadableValues() throws IOException {
+ out = new BufferedWriter(new FileWriter(CONFIG));
+ startConfig();
+ appendProperty("test.humanReadableValue1", "1m");
+ appendProperty("test.humanReadableValue2", "1M");
+ appendProperty("test.humanReadableValue5", "1MBCDE");
+
+ endConfig();
+ Path fileResource = new Path(CONFIG);
+ conf.addResource(fileResource);
+ assertEquals(1048576, conf.getLongBytes("test.humanReadableValue1", 0));
+ assertEquals(1048576, conf.getLongBytes("test.humanReadableValue2", 0));
+ try {
+ conf.getLongBytes("test.humanReadableValue5", 0);
+ fail("Property had invalid human readable value, but was read successfully.");
+ } catch (NumberFormatException e) {
+ // pass
+ }
+ }
+
public void testBooleanValues() throws IOException {
out=new BufferedWriter(new FileWriter(CONFIG));
startConfig();
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java?rev=1225489&r1=1225488&r2=1225489&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java Thu Dec 29 10:27:14 2011
@@ -143,8 +143,62 @@ public class TestStringUtils extends Uni
}
assertEquals(0L, StringUtils.TraditionalBinaryPrefix.string2long("0"));
- assertEquals(-1259520L, StringUtils.TraditionalBinaryPrefix.string2long("-1230k"));
- assertEquals(956703965184L, StringUtils.TraditionalBinaryPrefix.string2long("891g"));
+ assertEquals(1024L, StringUtils.TraditionalBinaryPrefix.string2long("1k"));
+ assertEquals(-1024L, StringUtils.TraditionalBinaryPrefix.string2long("-1k"));
+ assertEquals(1259520L,
+ StringUtils.TraditionalBinaryPrefix.string2long("1230K"));
+ assertEquals(-1259520L,
+ StringUtils.TraditionalBinaryPrefix.string2long("-1230K"));
+ assertEquals(104857600L,
+ StringUtils.TraditionalBinaryPrefix.string2long("100m"));
+ assertEquals(-104857600L,
+ StringUtils.TraditionalBinaryPrefix.string2long("-100M"));
+ assertEquals(956703965184L,
+ StringUtils.TraditionalBinaryPrefix.string2long("891g"));
+ assertEquals(-956703965184L,
+ StringUtils.TraditionalBinaryPrefix.string2long("-891G"));
+ assertEquals(501377302265856L,
+ StringUtils.TraditionalBinaryPrefix.string2long("456t"));
+ assertEquals(-501377302265856L,
+ StringUtils.TraditionalBinaryPrefix.string2long("-456T"));
+ assertEquals(11258999068426240L,
+ StringUtils.TraditionalBinaryPrefix.string2long("10p"));
+ assertEquals(-11258999068426240L,
+ StringUtils.TraditionalBinaryPrefix.string2long("-10P"));
+ assertEquals(1152921504606846976L,
+ StringUtils.TraditionalBinaryPrefix.string2long("1e"));
+ assertEquals(-1152921504606846976L,
+ StringUtils.TraditionalBinaryPrefix.string2long("-1E"));
+
+ String tooLargeNumStr = "10e";
+ try {
+ StringUtils.TraditionalBinaryPrefix.string2long(tooLargeNumStr);
+ fail("Test passed for a number " + tooLargeNumStr + " too large");
+ } catch (IllegalArgumentException e) {
+ assertEquals(tooLargeNumStr + " does not fit in a Long", e.getMessage());
+ }
+
+ String tooSmallNumStr = "-10e";
+ try {
+ StringUtils.TraditionalBinaryPrefix.string2long(tooSmallNumStr);
+ fail("Test passed for a number " + tooSmallNumStr + " too small");
+ } catch (IllegalArgumentException e) {
+ assertEquals(tooSmallNumStr + " does not fit in a Long", e.getMessage());
+ }
+
+ String invalidFormatNumStr = "10kb";
+ char invalidPrefix = 'b';
+ try {
+ StringUtils.TraditionalBinaryPrefix.string2long(invalidFormatNumStr);
+ fail("Test passed for a number " + invalidFormatNumStr
+ + " has invalid format");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Invalid size prefix '" + invalidPrefix + "' in '"
+ + invalidFormatNumStr
+ + "'. Allowed prefixes are k, m, g, t, p, e(case insensitive)",
+ e.getMessage());
+ }
+
}
@Test