You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by dw...@apache.org on 2020/01/16 10:30:47 UTC
[flink] 02/03: [FLINK-14853][core] Implemented MemorySize formatting
This is an automated email from the ASF dual-hosted git repository.
dwysakowicz pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
commit f31311a50b3d5ef54d3406a94de8040fefdf2853
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Thu Jan 9 10:59:19 2020 +0100
[FLINK-14853][core] Implemented MemorySize formatting
---
...lasticsearchUpsertTableSinkFactoryTestBase.java | 2 +-
.../flink/table/descriptors/ElasticsearchTest.java | 2 +-
.../flink/addons/hbase/HBaseDescriptorTest.java | 2 +-
.../org/apache/flink/configuration/MemorySize.java | 108 ++++++++++++++-------
.../MemorySizePrettyPrintingTest.java | 72 ++++++++++++++
.../ConfigOptionsDocGeneratorTest.java | 2 +-
.../pyflink/table/tests/test_descriptor.py | 2 +-
7 files changed, 150 insertions(+), 40 deletions(-)
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
index ea4644b..d302b88 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/ElasticsearchUpsertTableSinkFactoryTestBase.java
@@ -143,7 +143,7 @@ public abstract class ElasticsearchUpsertTableSinkFactoryTestBase extends TestLo
sinkOptions.put(SinkOption.BULK_FLUSH_BACKOFF_RETRIES, "3");
sinkOptions.put(SinkOption.BULK_FLUSH_INTERVAL, "100");
sinkOptions.put(SinkOption.BULK_FLUSH_MAX_ACTIONS, "1000");
- sinkOptions.put(SinkOption.BULK_FLUSH_MAX_SIZE, "1048576 bytes");
+ sinkOptions.put(SinkOption.BULK_FLUSH_MAX_SIZE, "1 mb");
sinkOptions.put(SinkOption.REST_MAX_RETRY_TIMEOUT, "100");
sinkOptions.put(SinkOption.REST_PATH_PREFIX, "/myapp");
return sinkOptions;
diff --git a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java
index 85326a6..b994f34 100644
--- a/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java
+++ b/flink-connectors/flink-connector-elasticsearch-base/src/test/java/org/apache/flink/table/descriptors/ElasticsearchTest.java
@@ -134,7 +134,7 @@ public class ElasticsearchTest extends DescriptorTestBase {
maximumDesc.put("connector.bulk-flush.backoff.max-retries", "3");
maximumDesc.put("connector.bulk-flush.interval", "100");
maximumDesc.put("connector.bulk-flush.max-actions", "1000");
- maximumDesc.put("connector.bulk-flush.max-size", "12582912 bytes");
+ maximumDesc.put("connector.bulk-flush.max-size", "12 mb");
maximumDesc.put("connector.failure-handler", "retry-rejected");
maximumDesc.put("connector.connection-max-retry-timeout", "100");
maximumDesc.put("connector.connection-path-prefix", "/myapp");
diff --git a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java
index b9a2f5d..e3c6cf6 100644
--- a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java
+++ b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java
@@ -78,7 +78,7 @@ public class HBaseDescriptorTest extends DescriptorTestBase {
prop1.put("connector.property-version", "1");
prop1.put("connector.write.buffer-flush.interval", "2s");
prop1.put("connector.write.buffer-flush.max-rows", "100");
- prop1.put("connector.write.buffer-flush.max-size", "1048576 bytes");
+ prop1.put("connector.write.buffer-flush.max-size", "1 mb");
return Arrays.asList(prop0, prop1);
}
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
index f477b6c..38d0941 100644
--- a/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
+++ b/flink-core/src/main/java/org/apache/flink/configuration/MemorySize.java
@@ -21,7 +21,11 @@ package org.apache.flink.configuration;
import org.apache.flink.annotation.PublicEvolving;
import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.List;
import java.util.Locale;
+import java.util.Optional;
+import java.util.stream.IntStream;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.BYTES;
import static org.apache.flink.configuration.MemorySize.MemoryUnit.GIGA_BYTES;
@@ -55,6 +59,9 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
/** The memory size, in bytes. */
private final long bytes;
+ /** The memorized value returned by toString(). */
+ private transient String stringified;
+
/**
* Constructs a new MemorySize.
*
@@ -117,7 +124,38 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
@Override
public String toString() {
- return bytes + " bytes";
+ if (stringified == null) {
+ stringified = formatToString();
+ }
+
+ return stringified;
+ }
+
+ private String formatToString() {
+ List<MemoryUnit> orderedUnits = Arrays.asList(
+ BYTES,
+ KILO_BYTES,
+ MEGA_BYTES,
+ GIGA_BYTES,
+ TERA_BYTES);
+
+ MemoryUnit highestIntegerUnit = IntStream.range(0, orderedUnits.size())
+ .sequential()
+ .filter(idx -> bytes % orderedUnits.get(idx).getMultiplier() != 0)
+ .boxed()
+ .findFirst()
+ .map(idx -> {
+ if (idx == 0) {
+ return orderedUnits.get(0);
+ } else {
+ return orderedUnits.get(idx - 1);
+ }
+ }).orElse(BYTES);
+
+ return String.format(
+ "%d %s",
+ bytes / highestIntegerUnit.getMultiplier(),
+ highestIntegerUnit.getUnits()[1]);
}
@Override
@@ -224,32 +262,7 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
"' cannot be re represented as 64bit number (numeric overflow).");
}
- final long multiplier;
- if (unit.isEmpty()) {
- multiplier = 1L;
- }
- else {
- if (matchesAny(unit, BYTES)) {
- multiplier = 1L;
- }
- else if (matchesAny(unit, KILO_BYTES)) {
- multiplier = 1024L;
- }
- else if (matchesAny(unit, MEGA_BYTES)) {
- multiplier = 1024L * 1024L;
- }
- else if (matchesAny(unit, GIGA_BYTES)) {
- multiplier = 1024L * 1024L * 1024L;
- }
- else if (matchesAny(unit, TERA_BYTES)) {
- multiplier = 1024L * 1024L * 1024L * 1024L;
- }
- else {
- throw new IllegalArgumentException("Memory size unit '" + unit +
- "' does not match any of the recognized units: " + MemoryUnit.getAllUnits());
- }
- }
-
+ final long multiplier = parseUnit(unit).map(MemoryUnit::getMultiplier).orElse(1L);
final long result = value * multiplier;
// check for overflow
@@ -261,6 +274,25 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
return result;
}
+ private static Optional<MemoryUnit> parseUnit(String unit) {
+ if (matchesAny(unit, BYTES)) {
+ return Optional.of(BYTES);
+ } else if (matchesAny(unit, KILO_BYTES)) {
+ return Optional.of(KILO_BYTES);
+ } else if (matchesAny(unit, MEGA_BYTES)) {
+ return Optional.of(MEGA_BYTES);
+ } else if (matchesAny(unit, GIGA_BYTES)) {
+ return Optional.of(GIGA_BYTES);
+ } else if (matchesAny(unit, TERA_BYTES)) {
+ return Optional.of(TERA_BYTES);
+ } else if (!unit.isEmpty()) {
+ throw new IllegalArgumentException("Memory size unit '" + unit +
+ "' does not match any of the recognized units: " + MemoryUnit.getAllUnits());
+ }
+
+ return Optional.empty();
+ }
+
private static boolean matchesAny(String str, MemoryUnit unit) {
for (String s : unit.getUnits()) {
if (s.equals(str)) {
@@ -286,22 +318,29 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
*/
public enum MemoryUnit {
- BYTES(new String[] { "b", "bytes" }),
- KILO_BYTES(new String[] { "k", "kb", "kibibytes" }),
- MEGA_BYTES(new String[] { "m", "mb", "mebibytes" }),
- GIGA_BYTES(new String[] { "g", "gb", "gibibytes" }),
- TERA_BYTES(new String[] { "t", "tb", "tebibytes" });
+ BYTES(new String[] { "b", "bytes" }, 1L),
+ KILO_BYTES(new String[] { "k", "kb", "kibibytes" }, 1024L),
+ MEGA_BYTES(new String[] { "m", "mb", "mebibytes" }, 1024L * 1024L),
+ GIGA_BYTES(new String[] { "g", "gb", "gibibytes" }, 1024L * 1024L * 1024L),
+ TERA_BYTES(new String[] { "t", "tb", "tebibytes" }, 1024L * 1024L * 1024L * 1024L);
+
+ private final String[] units;
- private String[] units;
+ private final long multiplier;
- MemoryUnit(String[] units) {
+ MemoryUnit(String[] units, long multiplier) {
this.units = units;
+ this.multiplier = multiplier;
}
public String[] getUnits() {
return units;
}
+ public long getMultiplier() {
+ return multiplier;
+ }
+
public static String getAllUnits() {
return concatenateUnits(BYTES.getUnits(), KILO_BYTES.getUnits(), MEGA_BYTES.getUnits(), GIGA_BYTES.getUnits(), TERA_BYTES.getUnits());
}
@@ -343,6 +382,5 @@ public class MemorySize implements java.io.Serializable, Comparable<MemorySize>
builder.setLength(builder.length() - 3);
return builder.toString();
}
-
}
}
diff --git a/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
new file mode 100644
index 0000000..30d8dc6
--- /dev/null
+++ b/flink-core/src/test/java/org/apache/flink/configuration/MemorySizePrettyPrintingTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.configuration;
+
+import org.apache.flink.configuration.MemorySize.MemoryUnit;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Tests for {@link MemorySize#toString()}.
+ */
+@RunWith(Parameterized.class)
+public class MemorySizePrettyPrintingTest extends TestLogger {
+ @Parameterized.Parameters
+ public static Object[][] parameters() {
+ return new Object[][]{
+ new Object[]{
+ new MemorySize(MemoryUnit.KILO_BYTES.getMultiplier() + 1),
+ "1025 bytes"
+ },
+ new Object[]{
+ new MemorySize(100),
+ "100 bytes"
+ },
+ new Object[]{
+ new MemorySize(1024),
+ "1 kb"
+ },
+ new Object[]{
+ new MemorySize(MemoryUnit.GIGA_BYTES.getMultiplier() + 1),
+ String.format("%d %s", MemoryUnit.GIGA_BYTES.getMultiplier() + 1, "bytes")
+ },
+ new Object[]{
+ new MemorySize(0),
+ "0 bytes"
+ }
+ };
+ }
+
+ @Parameterized.Parameter
+ public MemorySize memorySize;
+
+ @Parameterized.Parameter(1)
+ public String expectedString;
+
+ @Test
+ public void testFormatting() {
+ assertThat(memorySize.toString(), is(expectedString));
+ }
+}
diff --git a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
index 46eb0ee..6761f2b 100644
--- a/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
+++ b/flink-docs/src/test/java/org/apache/flink/docs/configuration/ConfigOptionsDocGeneratorTest.java
@@ -159,7 +159,7 @@ public class ConfigOptionsDocGeneratorTest {
" </tr>\n" +
" <tr>\n" +
" <td><h5>option.memory</h5></td>\n" +
- " <td style=\"word-wrap: break-word;\">1024 bytes</td>\n" +
+ " <td style=\"word-wrap: break-word;\">1 kb</td>\n" +
" <td>MemorySize</td>\n" +
" <td>Description</td>\n" +
" </tr>\n" +
diff --git a/flink-python/pyflink/table/tests/test_descriptor.py b/flink-python/pyflink/table/tests/test_descriptor.py
index 08bbdb8..1414abe 100644
--- a/flink-python/pyflink/table/tests/test_descriptor.py
+++ b/flink-python/pyflink/table/tests/test_descriptor.py
@@ -277,7 +277,7 @@ class ElasticsearchDescriptorTest(PyFlinkTestCase):
elasticsearch = Elasticsearch().bulk_flush_max_size("42 mb")
properties = elasticsearch.to_properties()
- expected = {'connector.bulk-flush.max-size': '44040192 bytes',
+ expected = {'connector.bulk-flush.max-size': '42 mb',
'connector.type': 'elasticsearch',
'connector.property-version': '1'}
self.assertEqual(expected, properties)