You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by tu...@apache.org on 2022/10/28 08:19:07 UTC
[shardingsphere] branch master updated: Replace gson to jackson for openGauss mppdb decoding (#21822)
This is an automated email from the ASF dual-hosted git repository.
tuichenchuxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new a511f8e5f15 Replace gson to jackson for openGauss mppdb decoding (#21822)
a511f8e5f15 is described below
commit a511f8e5f15661620402dce1237815d4239324d7
Author: Hongsheng Zhong <zh...@apache.org>
AuthorDate: Fri Oct 28 16:18:59 2022 +0800
Replace gson to jackson for openGauss mppdb decoding (#21822)
* Replace gson to jackson for openGauss mppdb decoding
* Add more jackson dependencies in management
* Remove unused paths
---
.github/workflows/it-scaling.yml | 2 --
.../proxy-native/src/main/release-docs/LICENSE | 8 +++---
distribution/proxy/src/main/release-docs/LICENSE | 8 +++---
kernel/data-pipeline/dialect/opengauss/pom.xml | 10 ++++++--
.../opengauss/ingest/wal/decode/MppTableData.java | 18 ++++++-------
.../ingest/wal/decode/MppdbDecodingPlugin.java | 19 +++++++++++---
.../ingest/wal/decode/MppdbDecodingPluginTest.java | 30 ++++++++++++++--------
.../ingest/wal/decode/DecodingPlugin.java | 8 +++---
pom.xml | 21 +++++++++++++++
9 files changed, 85 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/it-scaling.yml b/.github/workflows/it-scaling.yml
index 7a669ae78d4..586f1db2a56 100644
--- a/.github/workflows/it-scaling.yml
+++ b/.github/workflows/it-scaling.yml
@@ -29,7 +29,6 @@ on:
- '**/*-distsql*/**/src/main/**'
- 'kernel/data-pipeline/**/src/main/**'
- 'kernel/data-pipeline/**/pom.xml'
- - 'features/**/src/main/**/scaling/**'
- '**/src/main/**/data/pipeline/**'
- '**/src/main/resources/META-INF/services/*.data.pipeline.*'
- '!test/**'
@@ -49,7 +48,6 @@ on:
- '**/*-distsql*/**/src/main/**'
- 'kernel/data-pipeline/**/src/main/**'
- 'kernel/data-pipeline/**/pom.xml'
- - 'features/**/src/main/**/scaling/**'
- '**/src/main/**/data/pipeline/**'
- '**/src/main/resources/META-INF/services/*.data.pipeline.*'
- '!test/**'
diff --git a/distribution/proxy-native/src/main/release-docs/LICENSE b/distribution/proxy-native/src/main/release-docs/LICENSE
index ffb7abf6074..bf4d8a0f8cf 100644
--- a/distribution/proxy-native/src/main/release-docs/LICENSE
+++ b/distribution/proxy-native/src/main/release-docs/LICENSE
@@ -255,10 +255,10 @@ The text of each license is the standard Apache 2.0 license.
httpclient5 5.1.3: https://hc.apache.org/httpcomponents-client-5.1.x, Apache 2.0
httpcore5-h2 5.1.3: https://hc.apache.org/httpcomponents-core-5.1.x, Apache 2.0
httpcore5 5.1.3: https://hc.apache.org/httpcomponents-core-5.1.x, Apache 2.0
- jackson-annotations 2.10.0: http://github.com/FasterXML/jackson, Apache 2.0
- jackson-core 2.13.2: http://github.com/FasterXML/jackson, Apache 2.0
- jackson-databind 2.10.0: http://github.com/FasterXML/jackson, Apache 2.0
- jackson-dataformat-yaml 2.13.2: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-annotations 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-core 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-databind 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-dataformat-yaml 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
jcl-over-slf4j 1.7.36: https://github.com/qos-ch/slf4j, Apache 2.0
jetcd-common 0.7.3: https://github.com/etcd-io/jetcd, Apache 2.0
jetcd-core 0.7.3: https://github.com/etcd-io/jetcd, Apache 2.0
diff --git a/distribution/proxy/src/main/release-docs/LICENSE b/distribution/proxy/src/main/release-docs/LICENSE
index ffb7abf6074..bf4d8a0f8cf 100644
--- a/distribution/proxy/src/main/release-docs/LICENSE
+++ b/distribution/proxy/src/main/release-docs/LICENSE
@@ -255,10 +255,10 @@ The text of each license is the standard Apache 2.0 license.
httpclient5 5.1.3: https://hc.apache.org/httpcomponents-client-5.1.x, Apache 2.0
httpcore5-h2 5.1.3: https://hc.apache.org/httpcomponents-core-5.1.x, Apache 2.0
httpcore5 5.1.3: https://hc.apache.org/httpcomponents-core-5.1.x, Apache 2.0
- jackson-annotations 2.10.0: http://github.com/FasterXML/jackson, Apache 2.0
- jackson-core 2.13.2: http://github.com/FasterXML/jackson, Apache 2.0
- jackson-databind 2.10.0: http://github.com/FasterXML/jackson, Apache 2.0
- jackson-dataformat-yaml 2.13.2: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-annotations 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-core 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-databind 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
+ jackson-dataformat-yaml 2.13.4: http://github.com/FasterXML/jackson, Apache 2.0
jcl-over-slf4j 1.7.36: https://github.com/qos-ch/slf4j, Apache 2.0
jetcd-common 0.7.3: https://github.com/etcd-io/jetcd, Apache 2.0
jetcd-core 0.7.3: https://github.com/etcd-io/jetcd, Apache 2.0
diff --git a/kernel/data-pipeline/dialect/opengauss/pom.xml b/kernel/data-pipeline/dialect/opengauss/pom.xml
index af978547766..144e6a67fb5 100644
--- a/kernel/data-pipeline/dialect/opengauss/pom.xml
+++ b/kernel/data-pipeline/dialect/opengauss/pom.xml
@@ -32,11 +32,17 @@
<groupId>org.apache.shardingsphere</groupId>
<artifactId>shardingsphere-data-pipeline-postgresql</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
diff --git a/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppTableData.java b/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppTableData.java
index 7bc3039f5da..9743cd785c4 100644
--- a/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppTableData.java
+++ b/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppTableData.java
@@ -17,7 +17,7 @@
package org.apache.shardingsphere.data.pipeline.opengauss.ingest.wal.decode;
-import com.google.gson.annotations.SerializedName;
+import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import lombok.Setter;
@@ -28,27 +28,27 @@ import lombok.Setter;
@Getter
public final class MppTableData {
- @SerializedName("table_name")
+ @JsonProperty("table_name")
private String tableName;
- @SerializedName("op_type")
+ @JsonProperty("op_type")
private String opType;
- @SerializedName("columns_name")
+ @JsonProperty("columns_name")
private String[] columnsName;
- @SerializedName("columns_type")
+ @JsonProperty("columns_type")
private String[] columnsType;
- @SerializedName("columns_val")
+ @JsonProperty("columns_val")
private String[] columnsVal;
- @SerializedName("old_keys_name")
+ @JsonProperty("old_keys_name")
private String[] oldKeysName;
- @SerializedName("old_keys_type")
+ @JsonProperty("old_keys_type")
private String[] oldKeysType;
- @SerializedName("old_keys_val")
+ @JsonProperty("old_keys_val")
private String[] oldKeysVal;
}
diff --git a/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPlugin.java b/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPlugin.java
index cdd5c091dd6..4577c287bc9 100644
--- a/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPlugin.java
+++ b/kernel/data-pipeline/dialect/opengauss/src/main/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPlugin.java
@@ -17,8 +17,10 @@
package org.apache.shardingsphere.data.pipeline.opengauss.ingest.wal.decode;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
-import com.google.gson.Gson;
import lombok.AllArgsConstructor;
import org.apache.shardingsphere.data.pipeline.core.ingest.IngestDataChangeType;
import org.apache.shardingsphere.data.pipeline.core.ingest.exception.IngestException;
@@ -48,6 +50,13 @@ import java.util.List;
@AllArgsConstructor
public final class MppdbDecodingPlugin implements DecodingPlugin {
+ private static final ObjectMapper OBJECT_MAPPER;
+
+ static {
+ OBJECT_MAPPER = new ObjectMapper();
+ OBJECT_MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ }
+
private final BaseTimestampUtils timestampUtils;
@Override
@@ -77,8 +86,12 @@ public final class MppdbDecodingPlugin implements DecodingPlugin {
}
private AbstractRowEvent readTableEvent(final String mppData) {
- Gson mppDataGson = new Gson();
- MppTableData mppTableData = mppDataGson.fromJson(mppData, MppTableData.class);
+ MppTableData mppTableData;
+ try {
+ mppTableData = OBJECT_MAPPER.readValue(mppData, MppTableData.class);
+ } catch (final JsonProcessingException ex) {
+ throw new RuntimeException(ex);
+ }
AbstractRowEvent result;
String rowEventType = mppTableData.getOpType();
switch (rowEventType) {
diff --git a/kernel/data-pipeline/dialect/opengauss/src/test/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPluginTest.java b/kernel/data-pipeline/dialect/opengauss/src/test/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPluginTest.java
index e42195df3eb..c84035fbd02 100644
--- a/kernel/data-pipeline/dialect/opengauss/src/test/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPluginTest.java
+++ b/kernel/data-pipeline/dialect/opengauss/src/test/java/org/apache/shardingsphere/data/pipeline/opengauss/ingest/wal/decode/MppdbDecodingPluginTest.java
@@ -17,7 +17,8 @@
package org.apache.shardingsphere.data.pipeline.opengauss.ingest.wal.decode;
-import com.google.gson.Gson;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.SneakyThrows;
import org.apache.shardingsphere.data.pipeline.core.ingest.exception.IngestException;
import org.apache.shardingsphere.data.pipeline.postgresql.ingest.wal.decode.DecodingException;
import org.apache.shardingsphere.data.pipeline.postgresql.ingest.wal.event.DeleteRowEvent;
@@ -43,6 +44,8 @@ import static org.mockito.Mockito.when;
public final class MppdbDecodingPluginTest {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
private final LogSequenceNumber pgSequenceNumber = LogSequenceNumber.valueOf("0/14EFDB8");
private final OpenGaussLogSequenceNumber logSequenceNumber = new OpenGaussLogSequenceNumber(pgSequenceNumber);
@@ -56,13 +59,18 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsType(insertTypes);
tableData.setColumnsName(IntStream.range(0, insertTypes.length).mapToObj(idx -> "data" + idx).toArray(String[]::new));
tableData.setColumnsVal(IntStream.range(0, insertTypes.length).mapToObj(idx -> "'1 2 3'").toArray(String[]::new));
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
WriteRowEvent actual = (WriteRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
IntStream.range(0, insertTypes.length).forEach(each -> assertThat(actual.getAfterRow().get(each), is("1 2 3")));
}
+ @SneakyThrows
+ private String toJSON(final MppTableData tableData) {
+ return OBJECT_MAPPER.writeValueAsString(tableData);
+ }
+
@Test
public void assertDecodeUpdateRowEvent() {
MppTableData tableData = new MppTableData();
@@ -71,7 +79,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(new String[]{"data"});
tableData.setColumnsType(new String[]{"character varying"});
tableData.setColumnsVal(new String[]{"'1 2 3'"});
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
UpdateRowEvent actual = (UpdateRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
@@ -88,7 +96,7 @@ public final class MppdbDecodingPluginTest {
tableData.setOldKeysType(deleteTypes);
tableData.setOldKeysName(IntStream.range(0, deleteTypes.length).mapToObj(idx -> "data" + idx).toArray(String[]::new));
tableData.setOldKeysVal(deleteValues);
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
DeleteRowEvent actual = (DeleteRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
@@ -103,7 +111,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(new String[]{"data"});
tableData.setColumnsType(new String[]{"money"});
tableData.setColumnsVal(new String[]{"'$1.08'"});
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
WriteRowEvent actual = (WriteRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
@@ -119,7 +127,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(new String[]{"data"});
tableData.setColumnsType(new String[]{"boolean"});
tableData.setColumnsVal(new String[]{Boolean.TRUE.toString()});
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
WriteRowEvent actual = (WriteRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
@@ -140,7 +148,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(IntStream.range(0, insertTypes.length).mapToObj(idx -> "data" + idx).toArray(String[]::new));
tableData.setColumnsType(insertTypes);
tableData.setColumnsVal(insertValues);
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
TimestampUtils timestampUtils = mock(TimestampUtils.class);
when(timestampUtils.toTime(null, "21:21:21")).thenReturn(Time.valueOf("21:21:21"));
when(timestampUtils.toTime(null, "21:21:21 pst")).thenReturn(Time.valueOf("13:21:21"));
@@ -161,7 +169,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(new String[]{"data"});
tableData.setColumnsType(new String[]{"bytea"});
tableData.setColumnsVal(new String[]{"'\\xff00ab'"});
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
WriteRowEvent actual = (WriteRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
@@ -178,7 +186,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(new String[]{"data"});
tableData.setColumnsType(new String[]{"raw"});
tableData.setColumnsVal(new String[]{"'7D'"});
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
WriteRowEvent actual = (WriteRowEvent) new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
assertThat(actual.getLogSequenceNumber(), is(logSequenceNumber));
assertThat(actual.getTableName(), is("test"));
@@ -201,7 +209,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsName(new String[]{"data"});
tableData.setColumnsType(new String[]{"character varying"});
tableData.setColumnsVal(new String[]{"1 2 3"});
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
new MppdbDecodingPlugin(null).decode(data, logSequenceNumber);
}
@@ -215,7 +223,7 @@ public final class MppdbDecodingPluginTest {
tableData.setColumnsVal(new String[]{"'1 2 3'"});
TimestampUtils timestampUtils = mock(TimestampUtils.class);
when(timestampUtils.toTime(null, "1 2 3")).thenThrow(new SQLException(""));
- ByteBuffer data = ByteBuffer.wrap(new Gson().toJson(tableData).getBytes());
+ ByteBuffer data = ByteBuffer.wrap(toJSON(tableData).getBytes());
new MppdbDecodingPlugin(new OpenGaussTimestampUtils(timestampUtils)).decode(data, logSequenceNumber);
}
}
diff --git a/kernel/data-pipeline/dialect/postgresql/src/main/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/wal/decode/DecodingPlugin.java b/kernel/data-pipeline/dialect/postgresql/src/main/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/wal/decode/DecodingPlugin.java
index a82ee5a9c88..5e3823e2c7e 100644
--- a/kernel/data-pipeline/dialect/postgresql/src/main/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/wal/decode/DecodingPlugin.java
+++ b/kernel/data-pipeline/dialect/postgresql/src/main/java/org/apache/shardingsphere/data/pipeline/postgresql/ingest/wal/decode/DecodingPlugin.java
@@ -22,16 +22,16 @@ import org.apache.shardingsphere.data.pipeline.postgresql.ingest.wal.event.Abstr
import java.nio.ByteBuffer;
/**
- * logical replication decoding plugin interface.
+ * Logical replication decoding plugin.
*/
public interface DecodingPlugin {
/**
- * Decode wal event from logical replication data.
+ * Decode WAL event from logical replication data.
*
* @param data of logical replication
- * @param logSequenceNumber wal lsn
- * @return wal event
+ * @param logSequenceNumber log sequence number
+ * @return WAL event
*/
AbstractWalEvent decode(ByteBuffer data, BaseLogSequenceNumber logSequenceNumber);
}
diff --git a/pom.xml b/pom.xml
index a6a768c9703..c257238db18 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,6 +67,7 @@
<antlr4.version>4.9.2</antlr4.version>
<snakeyaml.version>1.33</snakeyaml.version>
<gson.version>2.9.1</gson.version>
+ <jackson.version>2.13.4</jackson.version>
<groovy.version>4.0.6</groovy.version>
<jaxb.version>2.3.0</jaxb.version>
@@ -203,6 +204,26 @@
<artifactId>gson</artifactId>
<version>${gson.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ <version>${jackson.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.groovy</groupId>
<artifactId>groovy</artifactId>