You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2017/05/06 17:47:52 UTC
[08/12] flink git commit: [FLINK-6470] [core] Add a utility to parse
memory sizes
[FLINK-6470] [core] Add a utility to parse memory sizes
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/50b8dda3
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/50b8dda3
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/50b8dda3
Branch: refs/heads/master
Commit: 50b8dda37d5194297e4a6e41460fbc13e67a393b
Parents: aed3b80
Author: Stephan Ewen <se...@apache.org>
Authored: Fri May 5 13:41:29 2017 +0200
Committer: Stephan Ewen <se...@apache.org>
Committed: Sat May 6 19:41:53 2017 +0200
----------------------------------------------------------------------
.../apache/flink/configuration/MemorySize.java | 251 +++++++++++++++++++
.../flink/configuration/MemorySizeTest.java | 204 +++++++++++++++
2 files changed, 455 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/50b8dda3/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
new file mode 100644
index 0000000..c34c721
--- /dev/null
+++ b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.util.Locale;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * MemorySize is a representation of a number of bytes, viewable in different units.
+ *
+ * <h2>Parsing</h2>
+ *
+ * The size can be parsed from a text expression. If the expression is a pure number,
+ * the value will be interpreted as bytes.
+ *
+ * <p>To make larger values more compact, the common size suffixes are supported:
+ *
+ * <ul>
+ * <li>q or 1b or 1bytes (bytes)
+ * <li>1k or 1kb or 1kibibytes (interpreted as kibibytes = 1024 bytes)
+ * <li>1m or 1mb or 1mebibytes (interpreted as mebibytes = 1024 kibibytes)
+ * <li>1g or 1gb or 1gibibytes (interpreted as gibibytes = 1024 mebibytes)
+ * <li>1t or 1tb or 1tebibytes (interpreted as tebibytes = 1024 gibibytes)
+ * </ul>
+ */
+@PublicEvolving
+public class MemorySize implements java.io.Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String[] BYTES_UNITS = { "b", "bytes" };
+
+ private static final String[] KILO_BYTES_UNITS = { "k", "kb", "kibibytes" };
+
+ private static final String[] MEGA_BYTES_UNITS = { "m", "mb", "mebibytes" };
+
+ private static final String[] GIGA_BYTES_UNITS = { "g", "gb", "gibibytes" };
+
+ private static final String[] TERA_BYTES_UNITS = { "t", "tb", "tebibytes" };
+
+ private static final String ALL_UNITS = concatenateUnits(
+ BYTES_UNITS, KILO_BYTES_UNITS, MEGA_BYTES_UNITS, GIGA_BYTES_UNITS, TERA_BYTES_UNITS);
+
+ // ------------------------------------------------------------------------
+
+ /** The memory size, in bytes */
+ private final long bytes;
+
+ /**
+ * Constructs a new MemorySize.
+ *
+ * @param bytes The size, in bytes. Must be zero or larger.
+ */
+ public MemorySize(long bytes) {
+ checkArgument(bytes >= 0, "bytes must be >= 0");
+ this.bytes = bytes;
+ }
+
+ // ------------------------------------------------------------------------
+
+ /**
+ * Gets the memory size in bytes.
+ */
+ public long getBytes() {
+ return bytes;
+ }
+
+ /**
+ * Gets the memory size in Kibibytes (= 1024 bytes).
+ */
+ public long getKibiBytes() {
+ return bytes >> 10;
+ }
+
+ /**
+ * Gets the memory size in Mebibytes (= 1024 Kibibytes).
+ */
+ public long getMebiBytes() {
+ return bytes >> 20;
+ }
+
+ /**
+ * Gets the memory size in Gibibytes (= 1024 Mebibytes).
+ */
+ public long getGibiBytes() {
+ return bytes >> 30;
+ }
+
+ /**
+ * Gets the memory size in Tebibytes (= 1024 Gibibytes).
+ */
+ public long getTebiBytes() {
+ return bytes >> 40;
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ return (int) (bytes ^ (bytes >>> 32));
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj == this || obj.getClass() == this.getClass() && ((MemorySize) obj).bytes == this.bytes;
+ }
+
+ @Override
+ public String toString() {
+ return bytes + " bytes";
+ }
+
+ // ------------------------------------------------------------------------
+ // Parsing
+ // ------------------------------------------------------------------------
+
+ /**
+ * Parses the given string as as MemorySize.
+ * The supported expressions are listed under {@link MemorySize}.
+ *
+ * @param text The string to parse
+ * @return The parsed MemorySize
+ *
+ * @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
+ */
+ public static MemorySize parse(String text) throws IllegalArgumentException {
+ return new MemorySize(parseBytes(text));
+ }
+
+ /**
+ * Parses the given string as bytes.
+ * The supported expressions are listed under {@link MemorySize}.
+ *
+ * @param text The string to parse
+ * @return The parsed size, in bytes.
+ *
+ * @throws IllegalArgumentException Thrown, if the expression cannot be parsed.
+ */
+ public static long parseBytes(String text) throws IllegalArgumentException {
+ checkNotNull(text, "text");
+
+ final String trimmed = text.trim();
+ checkArgument(!trimmed.isEmpty(), "argument is an empty- or whitespace-only string");
+
+ final int len = trimmed.length();
+ int pos = 0;
+
+ char current;
+ while (pos < len && (current = trimmed.charAt(pos)) >= '0' && current <= '9') {
+ pos++;
+ }
+
+ final String number = trimmed.substring(0, pos);
+ final String unit = trimmed.substring(pos).trim().toLowerCase(Locale.US);
+
+ if (number.isEmpty()) {
+ throw new NumberFormatException("text does not start with a number");
+ }
+
+ final long value;
+ try {
+ value = Long.parseLong(number); // this throws a NumberFormatException on overflow
+ }
+ catch (NumberFormatException e) {
+ throw new IllegalArgumentException("The value '" + number +
+ "' cannot be re represented as 64bit number (numeric overflow).");
+ }
+
+ final long multiplier;
+ if (unit.isEmpty()) {
+ multiplier = 1L;
+ }
+ else {
+ if (matchesAny(unit, BYTES_UNITS)) {
+ multiplier = 1L;
+ }
+ else if (matchesAny(unit, KILO_BYTES_UNITS)) {
+ multiplier = 1024L;
+ }
+ else if (matchesAny(unit, MEGA_BYTES_UNITS)) {
+ multiplier = 1024L * 1024L;
+ }
+ else if (matchesAny(unit, GIGA_BYTES_UNITS)) {
+ multiplier = 1024L * 1024L * 1024L;
+ }
+ else if (matchesAny(unit, TERA_BYTES_UNITS)) {
+ multiplier = 1024L * 1024L * 1024L * 1024L;
+ }
+ else {
+ throw new IllegalArgumentException("Memory size unit '" + unit +
+ "' does not match any of the recognized units: " + ALL_UNITS);
+ }
+ }
+
+ try {
+ return Math.multiplyExact(value, multiplier);
+ }
+ catch (ArithmeticException e) {
+ throw new IllegalArgumentException("The value '" + text +
+ "' cannot be re represented as 64bit number of bytes (numeric overflow).");
+ }
+ }
+
+ private static boolean matchesAny(String str, String[] variants) {
+ for (String s : variants) {
+ if (s.equals(str)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static String concatenateUnits(final String[]... allUnits) {
+ final StringBuilder builder = new StringBuilder(128);
+
+ for (String[] units : allUnits) {
+ builder.append('(');
+
+ for (String unit : units) {
+ builder.append(unit);
+ builder.append(" | ");
+ }
+
+ builder.setLength(builder.length() - 3);
+ builder.append(") / ");
+ }
+
+ builder.setLength(builder.length() - 3);
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/50b8dda3/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
new file mode 100644
index 0000000..dbdd96b
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizeTest.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.core.testutils.CommonTestUtils;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.*;
+/**
+ * Tests for the {@link MemorySize} class.
+ */
+public class MemorySizeTest {
+
+ @Test
+ public void testUnitConversion() {
+ final MemorySize zero = new MemorySize(0);
+ assertEquals(0, zero.getBytes());
+ assertEquals(0, zero.getKibiBytes());
+ assertEquals(0, zero.getMebiBytes());
+ assertEquals(0, zero.getGibiBytes());
+ assertEquals(0, zero.getTebiBytes());
+
+ final MemorySize bytes = new MemorySize(955);
+ assertEquals(955, bytes.getBytes());
+ assertEquals(0, bytes.getKibiBytes());
+ assertEquals(0, bytes.getMebiBytes());
+ assertEquals(0, bytes.getGibiBytes());
+ assertEquals(0, bytes.getTebiBytes());
+
+ final MemorySize kilos = new MemorySize(18500);
+ assertEquals(18500, kilos.getBytes());
+ assertEquals(18, kilos.getKibiBytes());
+ assertEquals(0, kilos.getMebiBytes());
+ assertEquals(0, kilos.getGibiBytes());
+ assertEquals(0, kilos.getTebiBytes());
+
+ final MemorySize megas = new MemorySize(15 * 1024 * 1024);
+ assertEquals(15_728_640, megas.getBytes());
+ assertEquals(15_360, megas.getKibiBytes());
+ assertEquals(15, megas.getMebiBytes());
+ assertEquals(0, megas.getGibiBytes());
+ assertEquals(0, megas.getTebiBytes());
+
+ final MemorySize teras = new MemorySize(2L * 1024 * 1024 * 1024 * 1024 + 10);
+ assertEquals(2199023255562L, teras.getBytes());
+ assertEquals(2147483648L, teras.getKibiBytes());
+ assertEquals(2097152, teras.getMebiBytes());
+ assertEquals(2048, teras.getGibiBytes());
+ assertEquals(2, teras.getTebiBytes());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testInvalid() {
+ new MemorySize(-1);
+ }
+
+ @Test
+ public void testStandardUtils() throws IOException {
+ final MemorySize size = new MemorySize(1234567890L);
+ final MemorySize cloned = CommonTestUtils.createCopySerializable(size);
+
+ assertEquals(size, cloned);
+ assertEquals(size.hashCode(), cloned.hashCode());
+ assertEquals(size.toString(), cloned.toString());
+ }
+
+ @Test
+ public void testParseBytes() {
+ assertEquals(1234, MemorySize.parseBytes("1234"));
+ assertEquals(1234, MemorySize.parseBytes("1234b"));
+ assertEquals(1234, MemorySize.parseBytes("1234 b"));
+ assertEquals(1234, MemorySize.parseBytes("1234bytes"));
+ assertEquals(1234, MemorySize.parseBytes("1234 bytes"));
+ }
+
+ @Test
+ public void testParseKibiBytes() {
+ assertEquals(667766, MemorySize.parse("667766k").getKibiBytes());
+ assertEquals(667766, MemorySize.parse("667766 k").getKibiBytes());
+ assertEquals(667766, MemorySize.parse("667766kb").getKibiBytes());
+ assertEquals(667766, MemorySize.parse("667766 kb").getKibiBytes());
+ assertEquals(667766, MemorySize.parse("667766kibibytes").getKibiBytes());
+ assertEquals(667766, MemorySize.parse("667766 kibibytes").getKibiBytes());
+ }
+
+ @Test
+ public void testParseMebiBytes() {
+ assertEquals(7657623, MemorySize.parse("7657623m").getMebiBytes());
+ assertEquals(7657623, MemorySize.parse("7657623 m").getMebiBytes());
+ assertEquals(7657623, MemorySize.parse("7657623mb").getMebiBytes());
+ assertEquals(7657623, MemorySize.parse("7657623 mb").getMebiBytes());
+ assertEquals(7657623, MemorySize.parse("7657623mebibytes").getMebiBytes());
+ assertEquals(7657623, MemorySize.parse("7657623 mebibytes").getMebiBytes());
+ }
+
+ @Test
+ public void testParseGibiBytes() {
+ assertEquals(987654, MemorySize.parse("987654g").getGibiBytes());
+ assertEquals(987654, MemorySize.parse("987654 g").getGibiBytes());
+ assertEquals(987654, MemorySize.parse("987654gb").getGibiBytes());
+ assertEquals(987654, MemorySize.parse("987654 gb").getGibiBytes());
+ assertEquals(987654, MemorySize.parse("987654gibibytes").getGibiBytes());
+ assertEquals(987654, MemorySize.parse("987654 gibibytes").getGibiBytes());
+ }
+
+ @Test
+ public void testParseTebiBytes() {
+ assertEquals(1234567, MemorySize.parse("1234567t").getTebiBytes());
+ assertEquals(1234567, MemorySize.parse("1234567 t").getTebiBytes());
+ assertEquals(1234567, MemorySize.parse("1234567tb").getTebiBytes());
+ assertEquals(1234567, MemorySize.parse("1234567 tb").getTebiBytes());
+ assertEquals(1234567, MemorySize.parse("1234567tebibytes").getTebiBytes());
+ assertEquals(1234567, MemorySize.parse("1234567 tebibytes").getTebiBytes());
+ }
+
+ @Test
+ public void testUpperCase() {
+ assertEquals(1L, MemorySize.parse("1 B").getBytes());
+ assertEquals(1L, MemorySize.parse("1 K").getKibiBytes());
+ assertEquals(1L, MemorySize.parse("1 M").getMebiBytes());
+ assertEquals(1L, MemorySize.parse("1 G").getGibiBytes());
+ assertEquals(1L, MemorySize.parse("1 T").getTebiBytes());
+ }
+
+ @Test
+ public void testTrimBeforeParse() {
+ assertEquals(155L, MemorySize.parseBytes(" 155 "));
+ assertEquals(155L, MemorySize.parseBytes(" 155 bytes "));
+ }
+
+ @Test
+ public void testParseInvalid() {
+ // null
+ try {
+ MemorySize.parseBytes(null);
+ fail("exception expected");
+ } catch (NullPointerException ignored) {}
+
+ // empty
+ try {
+ MemorySize.parseBytes("");
+ fail("exception expected");
+ } catch (IllegalArgumentException ignored) {}
+
+ // brank
+ try {
+ MemorySize.parseBytes(" ");
+ fail("exception expected");
+ } catch (IllegalArgumentException ignored) {}
+
+ // no number
+ try {
+ MemorySize.parseBytes("foobar or fubar or foo bazz");
+ fail("exception expected");
+ } catch (IllegalArgumentException ignored) {}
+
+ // wrong unit
+ try {
+ MemorySize.parseBytes("16 gjah");
+ fail("exception expected");
+ } catch (IllegalArgumentException ignored) {}
+
+ // multiple numbers
+ try {
+ MemorySize.parseBytes("16 16 17 18 bytes");
+ fail("exception expected");
+ } catch (IllegalArgumentException ignored) {}
+
+ // negavive number
+ try {
+ MemorySize.parseBytes("-100 bytes");
+ fail("exception expected");
+ } catch (IllegalArgumentException ignored) {}
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testParseNumberOverflow() {
+ MemorySize.parseBytes("100000000000000000000000000000000 bytes");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testParseNumberTimeUnitOverflow() {
+ MemorySize.parseBytes("100000000000000 tb");
+ }
+}