You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2026/04/16 02:11:21 UTC
(shardingsphere) branch master updated: Fix MySQL JSON literal decoding in pipeline migration (#38622)
This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 7e2f8abe1c0 Fix MySQL JSON literal decoding in pipeline migration (#38622)
7e2f8abe1c0 is described below
commit 7e2f8abe1c021c8711a7ea85980a1e20b1491db9
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Thu Apr 16 10:11:14 2026 +0800
Fix MySQL JSON literal decoding in pipeline migration (#38622)
* Support top-level JSON literals in MySQL binlog decoder
* Cover JSON literal null in pipeline E2E data
* Refine JSON literal null test coverage
* Make JSON literal null explicit in pipeline E2E
* Limit JSON literal null to one pipeline record
* Update RELEASE-NOTES.md
---
RELEASE-NOTES.md | 1 +
.../column/value/string/MySQLJsonValueDecoder.java | 3 ++
.../value/string/MySQLJsonValueDecoderTest.java | 37 +++++++++++++++++++---
.../pipeline/cases/task/E2EIncrementalTask.java | 9 +++++-
.../sqlbuilder/MySQLIntPkLargeOrderSQLBuilder.java | 2 +-
.../framework/helper/PipelineCaseHelper.java | 9 ++++++
6 files changed, 54 insertions(+), 7 deletions(-)
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 1f45472c7f0..154f7184c1c 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -19,6 +19,7 @@
1. Proxy: Resolve openGauss batch bind parameter types before reading values - [#38390](https://github.com/apache/shardingsphere/pull/38390)
1. Proxy: Fix primary key metadata loss for aliased columns in mysql prepare probe path - [#38517](https://github.com/apache/shardingsphere/pull/38517)
1. Sharding: Fix incorrect routing when irrelevant sharding conditions are present - [#38527](https://github.com/apache/shardingsphere/pull/38527)
+1. Pipeline: Fix MySQL JSON literal decoding in migration - [#38622](https://github.com/apache/shardingsphere/pull/38622)
### Enhancements
diff --git a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java
index 74ae5ce340a..4e8201e16df 100644
--- a/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java
+++ b/database/protocol/dialect/mysql/src/main/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoder.java
@@ -89,6 +89,9 @@ public final class MySQLJsonValueDecoder {
case JsonValueTypes.STRING:
outputString(decodeString(byteBuf.slice()), stringBuilder);
break;
+ case JsonValueTypes.LITERAL:
+ outputLiteral(byteBuf.readUnsignedByte(), stringBuilder);
+ break;
default:
throw new UnsupportedSQLOperationException(String.valueOf(type));
}
diff --git a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java
index c28fd848722..c4d48d1b02a 100644
--- a/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java
+++ b/database/protocol/dialect/mysql/src/test/java/org/apache/shardingsphere/database/protocol/mysql/packet/binlog/row/column/value/string/MySQLJsonValueDecoderTest.java
@@ -275,9 +275,15 @@ class MySQLJsonValueDecoderTest {
private static Stream<Arguments> decodeTopLevelScalarArguments() {
return Stream.of(
+ Arguments.of("decode top-level literal null", mockTopLevelLiteralByteBuf(JsonValueTypes.LITERAL_NULL), "null"),
+ Arguments.of("decode top-level literal true", mockTopLevelLiteralByteBuf(JsonValueTypes.LITERAL_TRUE), "true"),
+ Arguments.of("decode top-level literal false", mockTopLevelLiteralByteBuf(JsonValueTypes.LITERAL_FALSE), "false"),
+ Arguments.of("decode top-level string", mockTopLevelStringByteBuf("hello"), "\"hello\""),
+ Arguments.of("decode top-level string with escaped chars", mockTopLevelStringByteBuf("a\"\\b"), "\"a\\\"\\\\b\""),
+ Arguments.of("decode top-level int32", mockTopLevelInt32ByteBuf(123), "123"),
+ Arguments.of("decode top-level double", mockTopLevelDoubleByteBuf(45.6D), "45.6"),
Arguments.of("decode int16 top-level", mockTopLevelInt16ByteBuf(), "-32768"),
- Arguments.of("decode uint16 top-level", mockTopLevelUInt16ByteBuf(), "65535"),
- Arguments.of("decode string with escaped chars", mockTopLevelStringByteBuf(), "\"a\\\"\\\\b\""));
+ Arguments.of("decode uint16 top-level", mockTopLevelUInt16ByteBuf(), "65535"));
}
private static Stream<Arguments> decodeUnsupportedArguments() {
@@ -334,6 +340,20 @@ class MySQLJsonValueDecoderTest {
return result;
}
+ private static ByteBuf mockTopLevelLiteralByteBuf(final byte value) {
+ ByteBuf result = Unpooled.buffer();
+ result.writeByte(JsonValueTypes.LITERAL);
+ result.writeByte(value);
+ return result;
+ }
+
+ private static ByteBuf mockTopLevelInt32ByteBuf(final int value) {
+ ByteBuf result = Unpooled.buffer();
+ result.writeByte(JsonValueTypes.INT32);
+ result.writeIntLE(value);
+ return result;
+ }
+
private static ByteBuf mockTopLevelUInt16ByteBuf() {
ByteBuf result = Unpooled.buffer();
result.writeByte(JsonValueTypes.UINT16);
@@ -341,11 +361,18 @@ class MySQLJsonValueDecoderTest {
return result;
}
- private static ByteBuf mockTopLevelStringByteBuf() {
+ private static ByteBuf mockTopLevelDoubleByteBuf(final double value) {
+ ByteBuf result = Unpooled.buffer();
+ result.writeByte(JsonValueTypes.DOUBLE);
+ result.writeDoubleLE(value);
+ return result;
+ }
+
+ private static ByteBuf mockTopLevelStringByteBuf(final String value) {
ByteBuf result = Unpooled.buffer();
result.writeByte(JsonValueTypes.STRING);
- result.writeByte("a\"\\b".length());
- result.writeBytes("a\"\\b".getBytes());
+ result.writeByte(value.length());
+ result.writeBytes(value.getBytes());
return result;
}
diff --git a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/cases/task/E2EIncrementalTask.java b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/cases/task/E2EIncrementalTask.java
index c837f0717e5..14f8b587e49 100644
--- a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/cases/task/E2EIncrementalTask.java
+++ b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/cases/task/E2EIncrementalTask.java
@@ -47,6 +47,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.ThreadLocalRandom;
@RequiredArgsConstructor
@@ -75,6 +76,8 @@ public final class E2EIncrementalTask implements Runnable {
private final int loopCount;
+ private final AtomicBoolean generatedJsonLiteralNullForUpdate = new AtomicBoolean(false);
+
@Override
public void run() {
try {
@@ -152,7 +155,7 @@ public final class E2EIncrementalTask implements Runnable {
Object[] parameters = {"中文测试", randomInt, randomInt, randomInt, randomUnsignedInt, randomUnsignedInt, randomUnsignedInt,
randomUnsignedInt, 1.0F, 1.0, new BigDecimal("999"), now, now, now.toLocalDate(), now.toLocalTime(), Year.now().getValue() + 1,
new byte[]{-1, 0, 1}, new byte[]{1, 2, -1, -3},
- "D".getBytes(), "A".getBytes(), "T".getBytes(), "E", "text", "mediumText", "3", "3", PipelineCaseHelper.generateJsonString(32, true), orderId};
+ "D".getBytes(), "A".getBytes(), "T".getBytes(), "E", "text", "mediumText", "3", "3", generateMySQLJsonValue(), orderId};
log.info("update sql: {}, params: {}", sql, parameters);
DataSourceExecuteUtils.execute(dataSource, sql, parameters);
return;
@@ -180,6 +183,10 @@ public final class E2EIncrementalTask implements Runnable {
}
}
+ private String generateMySQLJsonValue() {
+ return generatedJsonLiteralNullForUpdate.compareAndSet(false, true) ? PipelineCaseHelper.generateJsonLiteralNull() : PipelineCaseHelper.generateJsonString(32, true);
+ }
+
private List<String> ignoreShardingColumns(final List<String> columnNames) {
return new ArrayList<>(columnNames.subList(2, columnNames.size()));
}
diff --git a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/dao/order/large/sqlbuilder/MySQLIntPkLargeOrderSQLBuilder.java b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/dao/order/large/sqlbuilder/MySQLIntPkLargeOrderSQLBuilder.java
index 3273bccad36..8a5235c8a05 100644
--- a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/dao/order/large/sqlbuilder/MySQLIntPkLargeOrderSQLBuilder.java
+++ b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/dao/order/large/sqlbuilder/MySQLIntPkLargeOrderSQLBuilder.java
@@ -103,7 +103,7 @@ public final class MySQLIntPkLargeOrderSQLBuilder implements IntPkLargeOrderSQLB
"1", "t", "e", "s", "t",
PipelineCaseHelper.generateString(2), "☠️x☺️x✋x☹️", PipelineCaseHelper.generateString(1),
"1", "2",
- PipelineCaseHelper.generateJsonString(32, false)
+ 0 == i ? PipelineCaseHelper.generateJsonLiteralNull() : PipelineCaseHelper.generateJsonString(32, false)
};
result.add(params);
}
diff --git a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/framework/helper/PipelineCaseHelper.java b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/framework/helper/PipelineCaseHelper.java
index f71e1486536..2ff2fa76e28 100644
--- a/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/framework/helper/PipelineCaseHelper.java
+++ b/test/e2e/operation/pipeline/src/test/java/org/apache/shardingsphere/test/e2e/operation/pipeline/framework/helper/PipelineCaseHelper.java
@@ -75,6 +75,15 @@ public final class PipelineCaseHelper {
return String.format("{\"test\":\"%s\"}", value);
}
+ /**
+ * Generate json literal null.
+ *
+ * @return json literal null
+ */
+ public static String generateJsonLiteralNull() {
+ return "null";
+ }
+
/**
* Generate float value.
*