You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2020/01/16 10:30:47 UTC

[flink] 02/03: [FLINK-14853][core] Implemented MemorySize formatting

This is an automated email from the ASF dual-hosted git repository.

dwysakowicz pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git

commit f31311a50b3d5ef54d3406a94de8040fefdf2853
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Jan 9 10:59:19 2020 +0100

    [FLINK-14853][core] Implemented MemorySize formatting
---
 ...lasticsearchUpsertTableSinkFactoryTestBase.java |   2 +-
 .../flink/table/descriptors/ElasticsearchTest.java |   2 +-
 .../flink/addons/hbase/HBaseDescriptorTest.java    |   2 +-
 .../org/apache/flink/configuration/MemorySize.java | 108 ++++++++++++++-------
 .../MemorySizePrettyPrintingTest.java              |  72 ++++++++++++++
 .../ConfigOptionsDocGeneratorTest.java             |   2 +-
 .../pyflink/table/tests/test_descriptor.py         |   2 +-
 7 files changed, 150 insertions(+), 40 deletions(-)

diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
index ea4644b..d302b88 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
@@ -143,7 +143,7 @@ public abstract class ElasticsearchUpsertTableSinkFactoryTestBase extends TestLo
 		sinkOptions.put(SinkOption.BULK_FLUSH_BACKOFF_RETRIES, "3");
 		sinkOptions.put(SinkOption.BULK_FLUSH_INTERVAL, "100");
 		sinkOptions.put(SinkOption.BULK_FLUSH_MAX_ACTIONS, "1000");
-		sinkOptions.put(SinkOption.BULK_FLUSH_MAX_SIZE, "1048576 bytes");
+		sinkOptions.put(SinkOption.BULK_FLUSH_MAX_SIZE, "1 mb");
 		sinkOptions.put(SinkOption.REST_MAX_RETRY_TIMEOUT, "100");
 		sinkOptions.put(SinkOption.REST_PATH_PREFIX, "/myapp");
 		return sinkOptions;
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java
index 85326a6..b994f34 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java
@@ -134,7 +134,7 @@ public class ElasticsearchTest extends DescriptorTestBase {
 		maximumDesc.put("connector.bulk-flush.backoff.max-retries", "3");
 		maximumDesc.put("connector.bulk-flush.interval", "100");
 		maximumDesc.put("connector.bulk-flush.max-actions", "1000");
-		maximumDesc.put("connector.bulk-flush.max-size", "12582912 bytes");
+		maximumDesc.put("connector.bulk-flush.max-size", "12 mb");
 		maximumDesc.put("connector.failure-handler", "retry-rejected");
 		maximumDesc.put("connector.connection-max-retry-timeout", "100");
 		maximumDesc.put("connector.connection-path-prefix", "/myapp");
diff --git a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java
index b9a2f5d..e3c6cf6 100644
--- a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java
+++ b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java
@@ -78,7 +78,7 @@ public class HBaseDescriptorTest extends DescriptorTestBase {
 		prop1.put("connector.property-version", "1");
 		prop1.put("connector.write.buffer-flush.interval", "2s");
 		prop1.put("connector.write.buffer-flush.max-rows", "100");
-		prop1.put("connector.write.buffer-flush.max-size", "1048576 bytes");
+		prop1.put("connector.write.buffer-flush.max-size", "1 mb");
 
 		return Arrays.asList(prop0, prop1);
 	}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
index f477b6c..38d0941 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -21,7 +21,11 @@ package org.apache.flink.configuration;
 import org.apache.flink.annotation.PublicEvolving;
 
 import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
+import java.util.stream.IntStream;
 
 import static org.apache.flink.configuration.MemorySize.MemoryUnit.BYTES;
 import static org.apache.flink.configuration.MemorySize.MemoryUnit.GIGA_BYTES;
@@ -55,6 +59,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
 	/** The memory size, in bytes. */
 	private final long bytes;
 
+	/** The memorized value returned by toString(). */
+	private transient String stringified;
+
 	/**
 	 * Constructs a new MemorySize.
 	 *
@@ -117,7 +124,38 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
 
 	@Override
 	public String toString() {
-		return bytes + " bytes";
+		if (stringified == null) {
+			stringified = formatToString();
+		}
+
+		return stringified;
+	}
+
+	private String formatToString() {
+		List<MemoryUnit> orderedUnits = Arrays.asList(
+			BYTES,
+			KILO_BYTES,
+			MEGA_BYTES,
+			GIGA_BYTES,
+			TERA_BYTES);
+
+		MemoryUnit highestIntegerUnit = IntStream.range(0, orderedUnits.size())
+			.sequential()
+			.filter(idx -> bytes % orderedUnits.get(idx).getMultiplier() != 0)
+			.boxed()
+			.findFirst()
+			.map(idx -> {
+				if (idx == 0) {
+					return orderedUnits.get(0);
+				} else {
+					return orderedUnits.get(idx - 1);
+				}
+			}).orElse(BYTES);
+
+		return String.format(
+			"%d %s",
+			bytes / highestIntegerUnit.getMultiplier(),
+			highestIntegerUnit.getUnits()[1]);
 	}
 
 	@Override
@@ -224,32 +262,7 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
 					"' cannot be re represented as 64bit number (numeric overflow).");
 		}
 
-		final long multiplier;
-		if (unit.isEmpty()) {
-			multiplier = 1L;
-		}
-		else {
-			if (matchesAny(unit, BYTES)) {
-				multiplier = 1L;
-			}
-			else if (matchesAny(unit, KILO_BYTES)) {
-				multiplier = 1024L;
-			}
-			else if (matchesAny(unit, MEGA_BYTES)) {
-				multiplier = 1024L * 1024L;
-			}
-			else if (matchesAny(unit, GIGA_BYTES)) {
-				multiplier = 1024L * 1024L * 1024L;
-			}
-			else if (matchesAny(unit, TERA_BYTES)) {
-				multiplier = 1024L * 1024L * 1024L * 1024L;
-			}
-			else {
-				throw new IllegalArgumentException("Memory size unit '" + unit +
-						"' does not match any of the recognized units: " + MemoryUnit.getAllUnits());
-			}
-		}
-
+		final long multiplier = parseUnit(unit).map(MemoryUnit::getMultiplier).orElse(1L);
 		final long result = value * multiplier;
 
 		// check for overflow
@@ -261,6 +274,25 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
 		return result;
 	}
 
+	private static Optional<MemoryUnit> parseUnit(String unit) {
+		if (matchesAny(unit, BYTES)) {
+			return Optional.of(BYTES);
+		} else if (matchesAny(unit, KILO_BYTES)) {
+			return Optional.of(KILO_BYTES);
+		} else if (matchesAny(unit, MEGA_BYTES)) {
+			return Optional.of(MEGA_BYTES);
+		} else if (matchesAny(unit, GIGA_BYTES)) {
+			return Optional.of(GIGA_BYTES);
+		} else if (matchesAny(unit, TERA_BYTES)) {
+			return Optional.of(TERA_BYTES);
+		} else if (!unit.isEmpty()) {
+			throw new IllegalArgumentException("Memory size unit '" + unit +
+				"' does not match any of the recognized units: " + MemoryUnit.getAllUnits());
+		}
+
+		return Optional.empty();
+	}
+
 	private static boolean matchesAny(String str, MemoryUnit unit) {
 		for (String s : unit.getUnits()) {
 			if (s.equals(str)) {
@@ -286,22 +318,29 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
 	 */
 	public enum MemoryUnit {
 
-		BYTES(new String[] { "b", "bytes" }),
-		KILO_BYTES(new String[] { "k", "kb", "kibibytes" }),
-		MEGA_BYTES(new String[] { "m", "mb", "mebibytes" }),
-		GIGA_BYTES(new String[] { "g", "gb", "gibibytes" }),
-		TERA_BYTES(new String[] { "t", "tb", "tebibytes" });
+		BYTES(new String[] { "b", "bytes" }, 1L),
+		KILO_BYTES(new String[] { "k", "kb", "kibibytes" }, 1024L),
+		MEGA_BYTES(new String[] { "m", "mb", "mebibytes" }, 1024L * 1024L),
+		GIGA_BYTES(new String[] { "g", "gb", "gibibytes" }, 1024L * 1024L * 1024L),
+		TERA_BYTES(new String[] { "t", "tb", "tebibytes" }, 1024L * 1024L * 1024L * 1024L);
+
+		private final String[] units;
 
-		private String[] units;
+		private final long multiplier;
 
-		MemoryUnit(String[] units) {
+		MemoryUnit(String[] units, long multiplier) {
 			this.units = units;
+			this.multiplier = multiplier;
 		}
 
 		public String[] getUnits() {
 			return units;
 		}
 
+		public long getMultiplier() {
+			return multiplier;
+		}
+
 		public static String getAllUnits() {
 			return concatenateUnits(BYTES.getUnits(), KILO_BYTES.getUnits(), MEGA_BYTES.getUnits(), GIGA_BYTES.getUnits(), TERA_BYTES.getUnits());
 		}
@@ -343,6 +382,5 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
 			builder.setLength(builder.length() - 3);
 			return builder.toString();
 		}
-
 	}
 }
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
new file mode 100644
index 0000000..30d8dc6
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.configuration.MemorySize.MemoryUnit;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for {@link MemorySize#toString()}.
+ */
+@RunWith(Parameterized.class)
+public class MemorySizePrettyPrintingTest extends TestLogger {
+	@Parameterized.Parameters
+	public static Object[][] parameters() {
+		return new Object[][]{
+			new Object[]{
+				new MemorySize(MemoryUnit.KILO_BYTES.getMultiplier() + 1),
+				"1025 bytes"
+			},
+			new Object[]{
+				new MemorySize(100),
+				"100 bytes"
+			},
+			new Object[]{
+				new MemorySize(1024),
+				"1 kb"
+			},
+			new Object[]{
+				new MemorySize(MemoryUnit.GIGA_BYTES.getMultiplier() + 1),
+				String.format("%d %s", MemoryUnit.GIGA_BYTES.getMultiplier() + 1, "bytes")
+			},
+			new Object[]{
+				new MemorySize(0),
+				"0 bytes"
+			}
+		};
+	}
+
+	@Parameterized.Parameter
+	public MemorySize memorySize;
+
+	@Parameterized.Parameter(1)
+	public String expectedString;
+
+	@Test
+	public void testFormatting() {
+		assertThat(memorySize.toString(), is(expectedString));
+	}
+}
diff --git a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
index 46eb0ee..6761f2b 100644
--- a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
+++ b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
@@ -159,7 +159,7 @@ public class ConfigOptionsDocGeneratorTest {
 				"        </tr>\n" +
 				"        <tr>\n" +
 				"            <td><h5>option.memory</h5></td>\n" +
-				"            <td style=\"word-wrap: break-word;\">1024 bytes</td>\n" +
+				"            <td style=\"word-wrap: break-word;\">1 kb</td>\n" +
 				"            <td>MemorySize</td>\n" +
 				"            <td>Description</td>\n" +
 				"        </tr>\n" +
diff --git a/flink-python/pyflink/table/tests/test_descriptor.py b/flink-python/pyflink/table/tests/test_descriptor.py
index 08bbdb8..1414abe 100644
--- a/flink-python/pyflink/table/tests/test_descriptor.py
+++ b/flink-python/pyflink/table/tests/test_descriptor.py
@@ -277,7 +277,7 @@ class ElasticsearchDescriptorTest(PyFlinkTestCase):
         elasticsearch = Elasticsearch().bulk_flush_max_size("42 mb")
 
         properties = elasticsearch.to_properties()
-        expected = {'connector.bulk-flush.max-size': '44040192 bytes',
+        expected = {'connector.bulk-flush.max-size': '42 mb',
                     'connector.type': 'elasticsearch',
                     'connector.property-version': '1'}
         self.assertEqual(expected, properties)