You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2026/07/17 07:52:00 UTC
(shardingsphere) branch master updated: Use UTF-8 for MySQL JSON binlog value decoding (#39140)
This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 6b36581fe56 Use UTF-8 for MySQL JSON binlog value decoding (#39140)
6b36581fe56 is described below
commit 6b36581fe568607f88685a72c79c828d03e39c0e
Author: Eunbin Son <58...@users.noreply.github.com>
AuthorDate: Fri Jul 17 16:51:54 2026 +0900
Use UTF-8 for MySQL JSON binlog value decoding (#39140)
Co-authored-by: Liang Zhang <zh...@apache.org>
---
RELEASE-NOTES.md | 5 +++--
.../row/column/value/string/MySQLJsonValueDecoder.java | 5 +++--
.../column/value/string/MySQLJsonValueDecoderTest.java | 15 ++++++++++++---
3 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 41d3f411180..6cf2ae41dfb 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -18,6 +18,7 @@
1. SQL Binder: Add DialectFunctionOption to handle wrong skip column bind in ColumnSegmentBinder - [#38350](https://github.com/apache/shardingsphere/pull/38350)
1. SQL Binder: Fix wrong bind info when order by refer column from with temporary table - [#38353](https://github.com/apache/shardingsphere/pull/38353)
1. Metadata: Fix MySQL metadata loading fallback when JDBC catalog is null for named tables - [#38855](https://github.com/apache/shardingsphere/pull/38855)
+1. Metadata: Fix Oracle metadata version comparison skipping identity and collation columns on 18c and later - [#39104](https://github.com/apache/shardingsphere/pull/39104)
1. JDBC: Fix stale generated values leaking into prepared statement executeBatch calls without pending batches - [#38160](https://github.com/apache/shardingsphere/pull/38160)
1. JDBC: Fix MySQL-compatible typed string conversion for `ResultSet#getObject(index, Class<T>)` - [#38444](https://github.com/apache/shardingsphere/pull/38444)
1. Proxy: Resolve MySQL prepared statement parameter columns for where clause - [#38382](https://github.com/apache/shardingsphere/pull/38382)
@@ -32,16 +33,16 @@
1. Pipeline: Fix MySQL JSON literal decoding in migration - [#38622](https://github.com/apache/shardingsphere/pull/38622)
1. Pipeline: Fix MySQL zero-value temporal binlog decoding with fractional precision in migration - [#38629](https://github.com/apache/shardingsphere/pull/38629)
1. Pipeline: Fix escape MySQL JSON binlog control characters - [#38800](https://github.com/apache/shardingsphere/pull/38800)
+1. Pipeline: Use UTF-8 to decode MySQL JSON binlog string and key values instead of the JVM default charset - [#39140](https://github.com/apache/shardingsphere/pull/39140)
1. Sharding: Support ORDER BY MySQL VARBINARY column by wrapping byte[] values in a Comparable adapter - [#38699](https://github.com/apache/shardingsphere/pull/38699)
1. Proxy: Fix incorrect generated key handling for explicit auto-increment values - [#38810](https://github.com/apache/shardingsphere/pull/38810)
+1. Proxy: Fix microseconds decoded as nanoseconds in MySQL binary TIME value - [#39138](https://github.com/apache/shardingsphere/pull/39138)
1. Sharding: Fix generated actual index names exceeding database identifier length limits while preserving legacy generated index name compatibility - [#38449](https://github.com/apache/shardingsphere/pull/38449)
1. Sharding: Fix AUTO_INTERVAL sharding failure under JVM default locales that use comma decimal separators - [#38806](https://github.com/apache/shardingsphere/pull/38806)
1. DistSQL: Fix case-sensitive storage unit matching in `SHOW RULES USED STORAGE UNIT` - [#38848](https://github.com/apache/shardingsphere/pull/38848)
1. Sharding: Compute the Snowflake key generator epoch in UTC instead of the JVM default timezone - [#38932](https://github.com/apache/shardingsphere/pull/38932)
1. Proxy: Fix MySQL BLOB data corruption when string-like prepared statement parameters target BLOB columns - [#39072](https://github.com/apache/shardingsphere/pull/39072)
1. Agent: Fix wrong target class name in StaticMethodAdviceExecutor error logs - [#39077](https://github.com/apache/shardingsphere/pull/39077)
-1. Metadata: Fix Oracle metadata version comparison skipping identity and collation columns on 18c and later - [#39104](https://github.com/apache/shardingsphere/pull/39104)
-1. Proxy: Fix microseconds decoded as nanoseconds in MySQL binary TIME value - [#39138](https://github.com/apache/shardingsphere/pull/39138)
### Enhancements
diff --git a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java
index d1056f03274..2b123ba6faa 100644
--- a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java
+++ b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java
@@ -24,6 +24,7 @@ import org.apache.shardingsphere.infra.exception.generic.UnsupportedSQLOperation
import java.io.Serializable;
import java.math.BigInteger;
+import java.nio.charset.StandardCharsets;
/**
* JSON type value decoder for MySQL.
@@ -142,7 +143,7 @@ public final class MySQLJsonValueDecoder {
int length = byteBuf.readUnsignedShortLE();
byte[] data = new byte[length];
byteBuf.getBytes(offset, data, 0, length);
- return new String(data);
+ return new String(data, StandardCharsets.UTF_8);
}
private static void decodeValueEntry(final boolean isSmall, final ByteBuf byteBuf, final StringBuilder stringBuilder) {
@@ -213,7 +214,7 @@ public final class MySQLJsonValueDecoder {
int length = decodeDataLength(byteBuf);
byte[] buffer = new byte[length];
byteBuf.readBytes(buffer, 0, length);
- return new String(buffer);
+ return new String(buffer, StandardCharsets.UTF_8);
}
private static int decodeDataLength(final ByteBuf byteBuf) {
diff --git a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java
index d89b845027c..4157823c50d 100644
--- a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java
+++ b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
@@ -71,6 +72,13 @@ class MySQLJsonValueDecoderTest {
assertThat(MySQLJsonValueDecoder.decode(payload), is("{\"key1\":null,\"key2\":true,\"key3\":false}"));
}
+ @Test
+ void assertDecodeSmallJsonObjectWithMultibyteUtf8KeyAndValue() {
+ List<Object[]> jsonEntries = Collections.singletonList(new Object[]{JsonValueTypes.STRING, "名前", "café"});
+ ByteBuf payload = mockJsonObjectByteBuf(jsonEntries, true);
+ assertThat(MySQLJsonValueDecoder.decode(payload), is("{\"名前\":\"café\"}"));
+ }
+
@Test
void assertDecodeSmallJsonArray() {
List<Object[]> jsonEntries = new LinkedList<>();
@@ -179,7 +187,7 @@ class MySQLJsonValueDecoderTest {
ByteBuf result = Unpooled.buffer();
for (Object[] each : jsonEntries) {
writeInt(jsonByteBuf, startOffset + result.readableBytes(), isSmall);
- byte[] keyBytes = ((String) each[1]).getBytes();
+ byte[] keyBytes = ((String) each[1]).getBytes(StandardCharsets.UTF_8);
jsonByteBuf.writeShortLE(keyBytes.length);
result.writeBytes(keyBytes);
}
@@ -248,9 +256,10 @@ class MySQLJsonValueDecoderTest {
}
private void writeString(final ByteBuf jsonByteBuf, final String value) {
- byte[] result = codecDataLength(value.length());
+ byte[] valueBytes = value.getBytes(StandardCharsets.UTF_8);
+ byte[] result = codecDataLength(valueBytes.length);
jsonByteBuf.writeBytes(result, 0, result.length);
- jsonByteBuf.writeBytes(value.getBytes());
+ jsonByteBuf.writeBytes(valueBytes);
}
private byte[] codecDataLength(final int length) {