You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ir...@apache.org on 2020/04/02 18:37:23 UTC
[ignite] branch master updated: IGNITE-12836 Writing methods to
avoid limit of 65535 bytes for java.io.DataOutput#writeUTF - Fixes #7571.
This is an automated email from the ASF dual-hosted git repository.
irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 41da31e IGNITE-12836 Writing methods to avoid limit of 65535 bytes for java.io.DataOutput#writeUTF - Fixes #7571.
41da31e is described below
commit 41da31e99c72d7c424ba94a31c0c675dc4b4c79d
Author: Kirill Tkalenko <tk...@yandex.ru>
AuthorDate: Thu Apr 2 21:36:54 2020 +0300
IGNITE-12836 Writing methods to avoid limit of 65535 bytes for java.io.DataOutput#writeUTF - Fixes #7571.
Signed-off-by: Ivan Rakov <iv...@gmail.com>
---
.../apache/ignite/internal/util/IgniteUtils.java | 198 +++++++++++++++++++++
.../ignite/internal/util/IgniteUtilsSelfTest.java | 134 +++++++++++++-
.../resources/org.apache.ignite.util/bigUtf.txt | Bin 0 -> 430138 bytes
.../resources/org.apache.ignite.util/limitUtf.txt | 10 ++
4 files changed, 341 insertions(+), 1 deletion(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 2c27ace..597751d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -40,6 +40,7 @@ import java.io.PrintStream;
import java.io.Reader;
import java.io.Serializable;
import java.io.StringWriter;
+import java.io.UTFDataFormatException;
import java.io.Writer;
import java.lang.annotation.Annotation;
import java.lang.management.CompilationMXBean;
@@ -270,6 +271,7 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import sun.misc.Unsafe;
+import static java.util.Objects.isNull;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISABLE_HOSTNAME_VERIFIER;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_HOME;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_LOCAL_HOST;
@@ -304,6 +306,12 @@ public abstract class IgniteUtils {
/** */
public static final long GB = 1024L * 1024 * 1024;
+ /**
+ * String limit in bytes for {@link DataOutput#writeUTF} and
+ * {@link DataInput#readUTF()}, that use "Modified UTF-8".
+ */
+ public static final int UTF_BYTE_LIMIT = 65_535;
+
/** Minimum checkpointing page buffer size (may be adjusted by Ignite). */
public static final Long DFLT_MIN_CHECKPOINTING_PAGE_BUFFER_SIZE = GB / 4;
@@ -5711,6 +5719,29 @@ public abstract class IgniteUtils {
/**
* Writes string to output stream accounting for {@code null} values.
+ * <p>
+ * Limitation for max string lenght of {@link #UTF_BYTE_LIMIT} bytes is caused by {@link ObjectOutputStream#writeUTF}
+ * used under the hood to perform an actual write.
+ * </p>
+ * <p>
+ * If longer string is passes a {@link UTFDataFormatException} exception will be thrown.
+ * </p>
+ * <p>
+ * To write longer strings use one of two options:
+ * <ul>
+ * <li>
+ * {@link #writeLongString(DataOutput, String)} writes string as is converting it into binary array of UTF-8
+ * encoded characters.
+ * To read the value back {@link #readLongString(DataInput)} should be used.
+ * </li>
+ * <li>
+ * {@link #writeCutString(DataOutput, String)} cuts passed string to {@link #UTF_BYTE_LIMIT} bytes
+ * and then writes them without converting to byte array.
+ * No exceptions will be thrown for string of any length; written string can be read back with regular
+ * {@link #readString(DataInput)} method.
+ * </li>
+ * </ul>
+ * </p>
*
* @param out Output stream to write to.
* @param s String to write, possibly {@code null}.
@@ -5727,6 +5758,13 @@ public abstract class IgniteUtils {
/**
* Reads string from input stream accounting for {@code null} values.
*
+ * Method enables to read strings shorter than {@link #UTF_BYTE_LIMIT} bytes in UTF-8 otherwise an exception will be thrown.
+ *
+ * Strings written by {@link #writeString(DataOutput, String)} or {@link #writeCutString(DataOutput, String)}
+ * can be read by this method.
+ *
+ * @see #writeString(DataOutput, String) for more information about writing strings.
+ *
* @param in Stream to read from.
* @return Read string, possibly {@code null}.
* @throws IOException If read failed.
@@ -11699,4 +11737,164 @@ public abstract class IgniteUtils {
public static String unquote(String s) {
return s == null ? null : s.replaceAll("^\"|\"$", "");
}
+
+ /**
+ * Writes string to output stream accounting for {@code null} values. <br/>
+ *
+ * This method can write string of any length, no {@link #UTF_BYTE_LIMIT} limits are applied.
+ *
+ * @param out Output stream to write to.
+ * @param s String to write, possibly {@code null}.
+ * @throws IOException If write failed.
+ */
+ public static void writeLongString(DataOutput out, @Nullable String s) throws IOException {
+ // Write null flag.
+ out.writeBoolean(isNull(s));
+
+ if (isNull(s))
+ return;
+
+ int sLen = s.length();
+
+ // Write string length.
+ out.writeInt(sLen);
+
+ // Write byte array.
+ for (int i = 0; i < sLen; i++) {
+ char c = s.charAt(i);
+ int utfBytes = utfBytes(c);
+
+ if (utfBytes == 1)
+ out.writeByte((byte)c);
+ else if (utfBytes == 3) {
+ out.writeByte((byte)(0xE0 | (c >> 12) & 0x0F));
+ out.writeByte((byte)(0x80 | (c >> 6) & 0x3F));
+ out.writeByte((byte)(0x80 | (c & 0x3F)));
+ }
+ else {
+ out.writeByte((byte)(0xC0 | ((c >> 6) & 0x1F)));
+ out.writeByte((byte)(0x80 | (c & 0x3F)));
+ }
+ }
+ }
+
+ /**
+ * Reads string from input stream accounting for {@code null} values. <br/>
+ *
+ * This method can read string of any length, no {@link #UTF_BYTE_LIMIT} limits are applied.
+ *
+ * @param in Stream to read from.
+ * @return Read string, possibly {@code null}.
+ * @throws IOException If read failed.
+ */
+ @Nullable public static String readLongString(DataInput in) throws IOException {
+ // Check null value.
+ if (in.readBoolean())
+ return null;
+
+ // Read string length.
+ int sLen = in.readInt();
+
+ StringBuilder strBuilder = new StringBuilder(sLen);
+
+ // Read byte array.
+ for (int i = 0, b0, b1, b2; i < sLen; i++) {
+ b0 = in.readByte() & 0xff;
+
+ switch (b0 >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7: // 1 byte format: 0xxxxxxx
+ strBuilder.append((char)b0);
+ break;
+
+ case 12:
+ case 13: // 2 byte format: 110xxxxx 10xxxxxx
+ b1 = in.readByte();
+
+ if ((b1 & 0xC0) != 0x80)
+ throw new UTFDataFormatException();
+
+ strBuilder.append((char)(((b0 & 0x1F) << 6) | (b1 & 0x3F)));
+ break;
+
+ case 14: // 3 byte format: 1110xxxx 10xxxxxx 10xxxxxx
+ b1 = in.readByte();
+ b2 = in.readByte();
+
+ if ((b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80)
+ throw new UTFDataFormatException();
+
+ strBuilder.append((char)(((b0 & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F)));
+ break;
+
+ default: // 10xx xxxx, 1111 xxxx
+ throw new UTFDataFormatException();
+ }
+ }
+
+ return strBuilder.toString();
+ }
+
+ /**
+ * Writes string to output stream accounting for {@code null} values. <br/>
+ *
+ * <p>
+ * Uses {@link ObjectOutputStream#writeUTF(String)} to write the string under the hood
+ * but cuts strings longer than {@link #UTF_BYTE_LIMIT} to the limit to avoid {@link UTFDataFormatException}.
+ * </p>
+ *
+ * <p>
+ * Strings written by the method can be read by {@link #readString(DataInput)}.
+ * </p>
+ *
+ * @see #writeString(DataOutput, String) for more information.
+ *
+ * @param out Output stream to write to.
+ * @param s String to write, possibly {@code null}.
+ * @throws IOException If write failed.
+ */
+ public static void writeCutString(DataOutput out, @Nullable String s) throws IOException {
+ // Write null flag.
+ out.writeBoolean(isNull(s));
+
+ if (isNull(s))
+ return;
+
+ //Conversion of string to limit.
+ for (int i = 0, bs = 0; i < s.length(); i++) {
+ if ((bs += utfBytes(s.charAt(i))) > UTF_BYTE_LIMIT) {
+ s = s.substring(0, i);
+ break;
+ }
+ }
+
+ out.writeUTF(s);
+ }
+
+ /**
+ * Get number of bytes for {@link DataOutput#writeUTF},
+ * depending on character: <br/>
+ *
+ * One byte - If a character <code>c</code> is in the range
+ * <code>\u0001</code> through <code>\u007f</code>.<br/>
+ *
+ * Two bytes - If a character <code>c</code> is <code>\u0000</code> or
+ * is in the range <code>\u0080</code> through <code>\u07ff</code>.
+ * <br/>
+ *
+ * Three bytes - If a character <code>c</code> is in the range
+ * <code>\u0800</code> through <code>uffff</code>.
+ *
+ * @param c Character.
+ * @return Number of bytes.
+ */
+ public static int utfBytes(char c) {
+ return (c >= 0x0001 && c <= 0x007F) ? 1 : (c > 0x07FF) ? 3 : 2;
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index 772d234..2cf9462 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.util;
+import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInput;
@@ -24,6 +25,7 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.File;
+import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.lang.annotation.Documented;
@@ -52,7 +54,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
-
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -82,6 +83,9 @@ import org.junit.Assert;
import org.junit.Test;
import static java.util.Arrays.asList;
+import static java.util.Objects.nonNull;
+import static java.util.stream.Collectors.joining;
+import static org.apache.ignite.testframework.GridTestUtils.readResource;
import static org.junit.Assert.assertArrayEquals;
/**
@@ -1212,6 +1216,115 @@ public class IgniteUtilsSelfTest extends GridCommonAbstractTest {
}
/**
+ * Testing methods {@link IgniteUtils#writeLongString} and
+ * {@link IgniteUtils#readLongString} using resource files, where each line is
+ * needed to test different cases:
+ * 1){@code null}. <br/>
+ *
+ * 2)Empty line. <br/>
+ *
+ * 3)Simple strings. <br/>
+ *
+ * 4)Various combinations of strings with one, two, and three-byte
+ * characters with size greater than {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testReadWriteBigUTF() throws Exception {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ readLines("org.apache.ignite.util/bigUtf.txt", readLine -> {
+ baos.reset();
+
+ DataOutput dOut = new DataOutputStream(baos);
+ U.writeLongString(dOut, readLine);
+
+ DataInputStream dIn = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ String readBigUTF = U.readLongString(dIn);
+
+ assertEquals(readLine, readBigUTF);
+ });
+ }
+ }
+
+ /**
+ * Testing method {@link IgniteUtils#writeCutString} using resource files,
+ * where each line is needed to test different cases: <br/>
+ * 1){@code null}. <br/>
+ *
+ * 2)Empty line. <br/>
+ *
+ * 3)Simple strings. <br/>
+ *
+ * 4)String containing single-byte characters of size
+ * {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * 5)String containing single-byte characters of size more than
+ * {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * 6)String containing two-byte characters of size
+ * {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * 7)String containing two-byte characters of size more than
+ * {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * 8)String containing three-byte characters of size
+ * {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * 9)String containing three-byte characters of size more than
+ * {@link IgniteUtils#UTF_BYTE_LIMIT}. <br/>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testWriteLimitUTF() throws Exception {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ readLines("org.apache.ignite.util/limitUtf.txt", readLine -> {
+ baos.reset();
+
+ DataOutput dOut = new DataOutputStream(baos);
+ U.writeCutString(dOut, readLine);
+
+ DataInputStream dIn = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ String readUTF = U.readString(dIn);
+
+ if (nonNull(readLine)) {
+ AtomicInteger utfBytes = new AtomicInteger();
+
+ readLine = readLine.chars()
+ .filter(c -> utfBytes.addAndGet(U.utfBytes((char)c)) <= U.UTF_BYTE_LIMIT)
+ .mapToObj(c -> String.valueOf((char)c)).collect(joining());
+ }
+
+ assertEquals(readLine, readUTF);
+ });
+ }
+ }
+
+ /**
+ * Reading lines from a resource file and passing them to consumer.
+ * If read string is {@code "null"}, it is converted to {@code null}.
+ *
+ * @param rsrcName Resource name.
+ * @param consumer Consumer.
+ * @throws Exception If failed.
+ */
+ private void readLines(String rsrcName, ThrowableConsumer<String> consumer) throws Exception {
+ byte[] content = readResource(getClass().getClassLoader(), rsrcName);
+
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(content)))) {
+ String readLine;
+
+ while (nonNull(readLine = reader.readLine())) {
+ if ("null".equals(readLine))
+ readLine = null;
+
+ consumer.accept(readLine);
+ }
+ }
+ }
+
+ /**
* Test enum.
*/
private enum TestEnum {
@@ -1241,4 +1354,23 @@ public class IgniteUtilsSelfTest extends GridCommonAbstractTest {
@Ann2 private interface I4 {}
private interface I5 extends I4 {}
+
+ /**
+ * Represents an operation that accepts a single input argument and returns
+ * no result. Unlike most other functional interfaces,
+ * {@code ThrowableConsumer} is expected to operate via side-effects.
+ *
+ * Also it is able to throw {@link Exception} unlike {@link Consumer}.
+ *
+ * @param <T> The type of the input to the operation.
+ */
+ @FunctionalInterface
+ private static interface ThrowableConsumer<T> {
+ /**
+ * Performs this operation on the given argument.
+ *
+ * @param t the input argument.
+ */
+ void accept(@Nullable T t) throws Exception;
+ }
}
diff --git a/modules/core/src/test/resources/org.apache.ignite.util/bigUtf.txt b/modules/core/src/test/resources/org.apache.ignite.util/bigUtf.txt
new file mode 100644
index 0000000..4578635
Binary files /dev/null and b/modules/core/src/test/resources/org.apache.ignite.util/bigUtf.txt differ
diff --git a/modules/core/src/test/resources/org.apache.ignite.util/limitUtf.txt b/modules/core/src/test/resources/org.apache.ignite.util/limitUtf.txt
new file mode 100644
index 0000000..e0595fc
--- /dev/null
+++ b/modules/core/src/test/resources/org.apache.ignite.util/limitUtf.txt
@@ -0,0 +1,10 @@
+null
+
+123
+abc
+ [...]
+ [...]
+ [...]
+ [...]
+�
�� [...]
+�
�� [...]