You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/05/26 04:06:22 UTC
[incubator-paimon] branch master updated: [flink] Add KafkaCanalEventParser (#1226)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ae2b99634 [flink] Add KafkaCanalEventParser (#1226)
ae2b99634 is described below
commit ae2b996341be6e7666a5b1b0bb263f07cdd8d197
Author: HZY <48...@users.noreply.github.com>
AuthorDate: Fri May 26 12:06:17 2023 +0800
[flink] Add KafkaCanalEventParser (#1226)
---
.../java/org/apache/paimon/utils/TypeUtils.java | 10 +-
paimon-flink/paimon-flink-common/pom.xml | 6 +
.../action/cdc/{mysql => }/ComputedColumn.java | 7 +-
.../action/cdc/{mysql => }/TableNameConverter.java | 6 +-
.../action/cdc/kafka/canal/CanalFieldParser.java | 126 +++++++
.../cdc/kafka/canal/CanalJsonEventParser.java | 381 +++++++++++++++++++++
.../flink/action/cdc/mysql/MySqlActionUtils.java | 1 +
.../cdc/mysql/MySqlDebeziumJsonEventParser.java | 25 +-
.../action/cdc/mysql/MySqlSyncDatabaseAction.java | 1 +
.../action/cdc/mysql/MySqlSyncTableAction.java | 1 +
.../flink/action/cdc/mysql/MySqlTypeUtils.java | 143 +++++++-
.../action/cdc/kafka/KafkaEventParserTest.java | 307 +++++++++++++++++
12 files changed, 979 insertions(+), 35 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
index b853d4881..cfcfe7e26 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
@@ -106,8 +106,14 @@ public class TypeUtils {
if (d == ((float) d)) {
return (float) d;
} else {
- throw new NumberFormatException(
- s + " cannot be cast to float due to precision loss");
+ // Compatible canal-cdc
+ Float f = Float.valueOf(s);
+ if (f.toString().length() != s.length()) {
+ throw new NumberFormatException(
+ s + " cannot be cast to float due to precision loss");
+ } else {
+ return f;
+ }
}
case DOUBLE:
return Double.valueOf(s);
diff --git a/paimon-flink/paimon-flink-common/pom.xml b/paimon-flink/paimon-flink-common/pom.xml
index c12b9587a..2c3e066c0 100644
--- a/paimon-flink/paimon-flink-common/pom.xml
+++ b/paimon-flink/paimon-flink-common/pom.xml
@@ -38,6 +38,7 @@ under the License.
<flink.cdc.version>2.3.0</flink.cdc.version>
<frocksdbjni.version>6.20.3-ververica-2.0</frocksdbjni.version>
<geometry.version>2.2.0</geometry.version>
+ <druid.version>1.2.15</druid.version>
</properties>
<dependencies>
@@ -267,6 +268,11 @@ under the License.
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>druid</artifactId>
+ <version>${druid.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/ComputedColumn.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/ComputedColumn.java
similarity index 90%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/ComputedColumn.java
rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/ComputedColumn.java
index e489366f3..df8142120 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/ComputedColumn.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/ComputedColumn.java
@@ -16,8 +16,9 @@
* limitations under the License.
*/
-package org.apache.paimon.flink.action.cdc.mysql;
+package org.apache.paimon.flink.action.cdc;
+import org.apache.paimon.flink.action.cdc.mysql.Expression;
import org.apache.paimon.types.DataType;
import javax.annotation.Nullable;
@@ -48,13 +49,13 @@ public class ComputedColumn implements Serializable {
return expression.outputType();
}
- String fieldReference() {
+ public String fieldReference() {
return expression.fieldReference();
}
/** Compute column's value from given argument. Return null if input is null. */
@Nullable
- String eval(@Nullable String input) {
+ public String eval(@Nullable String input) {
if (input == null) {
return null;
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/TableNameConverter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java
similarity index 88%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/TableNameConverter.java
rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java
index 94996b48a..692b10252 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/TableNameConverter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.paimon.flink.action.cdc.mysql;
+package org.apache.paimon.flink.action.cdc;
import java.io.Serializable;
@@ -29,11 +29,11 @@ public class TableNameConverter implements Serializable {
private final String prefix;
private final String suffix;
- TableNameConverter(boolean caseSensitive) {
+ public TableNameConverter(boolean caseSensitive) {
this(caseSensitive, "", "");
}
- TableNameConverter(boolean caseSensitive, String prefix, String suffix) {
+ public TableNameConverter(boolean caseSensitive, String prefix, String suffix) {
this.caseSensitive = caseSensitive;
this.prefix = prefix;
this.suffix = suffix;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalFieldParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalFieldParser.java
new file mode 100644
index 000000000..b65c7396b
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalFieldParser.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.kafka.canal;
+
+/** Converts some special types such as enum、set、geometry. */
+public class CanalFieldParser {
+
+ protected static byte[] convertGeoType2WkbArray(byte[] mysqlGeomBytes) {
+ if (mysqlGeomBytes == null) {
+ return null;
+ }
+ int sridLength = 4;
+ boolean hasSrid = false;
+ for (int i = 0; i < sridLength; ++i) {
+ if (mysqlGeomBytes[i] != 0) {
+ hasSrid = true;
+ break;
+ }
+ }
+ byte[] wkb;
+ if (hasSrid) {
+ wkb = new byte[mysqlGeomBytes.length];
+ // byteOrder + geometry
+ System.arraycopy(mysqlGeomBytes, 4, wkb, 0, 5);
+ // srid
+ System.arraycopy(mysqlGeomBytes, 0, wkb, 5, 4);
+ // geometry
+ System.arraycopy(mysqlGeomBytes, 9, wkb, 9, wkb.length - 9);
+
+ // set srid flag
+ if (wkb[0] == 0) {
+ // big endian
+ wkb[1] = (byte) (wkb[1] + 32);
+ } else {
+ wkb[4] = (byte) (wkb[4] + 32);
+ }
+ } else {
+ wkb = new byte[mysqlGeomBytes.length - 4];
+ System.arraycopy(mysqlGeomBytes, 4, wkb, 0, wkb.length);
+ }
+ return wkb;
+ }
+
+ protected static String convertSet(String value, String mysqlType) {
+ if (value == null) {
+ return null;
+ }
+ // mysql set type value can be filled with more than one, value is a bit string conversion
+ // from the long
+ int indexes = Integer.parseInt(value);
+ return getSetValuesByIndex(mysqlType, indexes);
+ }
+
+ protected static String convertEnum(String value, String mysqlType) {
+ if (value == null) {
+ return null;
+ }
+ int elementIndex = Integer.parseInt(value);
+ // enum('a','b','c')
+ return getEnumValueByIndex(mysqlType, elementIndex);
+ }
+
+ protected static String getEnumValueByIndex(String mysqlType, int elementIndex) {
+ String[] options = extractEnumValueByIndex(mysqlType);
+
+ return options[elementIndex - 1];
+ }
+
+ protected static String getSetValuesByIndex(String mysqlType, int indexes) {
+ String[] options = extractSetValuesByIndex(mysqlType);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("[");
+ int index = 0;
+ boolean first = true;
+ int optionLen = options.length;
+
+ while (indexes != 0L) {
+ if (indexes % 2L != 0) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(',');
+ }
+ if (index < optionLen) {
+ sb.append(options[index]);
+ } else {
+ throw new RuntimeException(
+ String.format(
+ "extractSetValues from mysqlType[%s],index:%d failed",
+ mysqlType, indexes));
+ }
+ }
+ ++index;
+ indexes = indexes >>> 1;
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ private static String[] extractSetValuesByIndex(String mysqlType) {
+ // set('x','y')
+ return mysqlType.substring(5, mysqlType.length() - 2).split("'\\s*,\\s*'");
+ }
+
+ private static String[] extractEnumValueByIndex(String mysqlType) {
+ // enum('x','y')
+ return mysqlType.substring(6, mysqlType.length() - 2).split("'\\s*,\\s*'");
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalJsonEventParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalJsonEventParser.java
new file mode 100644
index 000000000..6c2441a3a
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalJsonEventParser.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.kafka.canal;
+
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
+import org.apache.paimon.flink.action.cdc.mysql.MySqlTypeUtils;
+import org.apache.paimon.flink.sink.cdc.CdcRecord;
+import org.apache.paimon.flink.sink.cdc.EventParser;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.RowKind;
+import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.utils.StringUtils;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.NullNode;
+
+import com.alibaba.druid.DbType;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLName;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddColumn;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropColumnItem;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
+import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlAlterTableChangeColumn;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlAlterTableModifyColumn;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** {@link EventParser} for Canal-json. */
+public class CanalJsonEventParser implements EventParser<String> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CanalJsonEventParser.class);
+
+ private static final String FIELD_DATA = "data";
+ private static final String FIELD_OLD = "old";
+ private static final String TYPE = "type";
+ private static final String OP_INSERT = "INSERT";
+ private static final String OP_UPDATE = "UPDATE";
+ private static final String OP_DELETE = "DELETE";
+ private static final String OP_CREATE = "CREATE";
+
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
+ private JsonNode root;
+ private Map<String, String> mySqlFieldTypes;
+ private Map<String, DataType> paimonFieldTypes;
+
+ private final boolean caseSensitive;
+ private final TableNameConverter tableNameConverter;
+ private final List<ComputedColumn> computedColumns;
+
+ public CanalJsonEventParser(boolean caseSensitive, List<ComputedColumn> computedColumns) {
+ this(caseSensitive, new TableNameConverter(caseSensitive), computedColumns);
+ }
+
+ public CanalJsonEventParser(boolean caseSensitive, TableNameConverter tableNameConverter) {
+ this(caseSensitive, tableNameConverter, Collections.emptyList());
+ }
+
+ public CanalJsonEventParser(
+ boolean caseSensitive,
+ TableNameConverter tableNameConverter,
+ List<ComputedColumn> computedColumns) {
+ this.caseSensitive = caseSensitive;
+ this.tableNameConverter = tableNameConverter;
+ this.computedColumns = computedColumns;
+ }
+
+ @Override
+ public void setRawEvent(String rawEvent) {
+ try {
+ root = objectMapper.readValue(rawEvent, JsonNode.class);
+ Preconditions.checkNotNull(
+ root.get("mysqlType"),
+ "CanalJsonEventParser only supports canal-json format,"
+ + "please make sure that your topic's format is accurate.");
+ JsonNode mysqlType = root.get("mysqlType");
+
+ if (!isUpdatedDataFields()) {
+ updateFieldTypes(mysqlType);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public String tableName() {
+ String tableName = root.get("table").asText();
+ return tableNameConverter.convert(tableName);
+ }
+
+ private void updateFieldTypes(JsonNode schema) {
+ mySqlFieldTypes = new LinkedHashMap<>();
+ paimonFieldTypes = new LinkedHashMap<>();
+ Iterator<String> iterator = schema.fieldNames();
+ while (iterator.hasNext()) {
+ String fieldName = iterator.next();
+ String fieldType = schema.get(fieldName).asText();
+ mySqlFieldTypes.put(fieldName, fieldType);
+ paimonFieldTypes.put(fieldName, MySqlTypeUtils.toDataType(fieldType));
+ }
+ }
+
+ @Override
+ public boolean isUpdatedDataFields() {
+ if (root.get("isDdl") == null) {
+ return false;
+ } else {
+ return "true".equals(root.get("isDdl").asText());
+ }
+ }
+
+ @Override
+ public Optional<List<DataField>> getUpdatedDataFields() {
+ String sql = root.get("sql").asText();
+
+ if (StringUtils.isEmpty(sql)) {
+ return Optional.empty();
+ }
+
+ List<DataField> result = new ArrayList<>();
+ int id = 0;
+ for (Map.Entry<String, DataType> fieldType : paimonFieldTypes.entrySet()) {
+ result.add(
+ new DataField(
+ id++,
+ caseSensitive ? fieldType.getKey() : fieldType.getKey().toLowerCase(),
+ fieldType.getValue()));
+ }
+ SQLStatement sqlStatement = SQLUtils.parseSingleStatement(sql, DbType.mysql);
+ if (sqlStatement instanceof SQLAlterTableStatement) {
+ SQLAlterTableStatement sqlAlterTableStatement = (SQLAlterTableStatement) sqlStatement;
+ for (int i = 0; i < sqlAlterTableStatement.getItems().size(); i++) {
+ SQLAlterTableItem sqlAlterTableItem = sqlAlterTableStatement.getItems().get(i);
+ if (sqlAlterTableItem instanceof SQLAlterTableAddColumn) {
+ SQLAlterTableAddColumn sqlAlterTableAddColumn =
+ (SQLAlterTableAddColumn) sqlAlterTableItem;
+ List<SQLColumnDefinition> columns = sqlAlterTableAddColumn.getColumns();
+ for (SQLColumnDefinition column : columns) {
+ String columnName = column.getColumnName().replace("`", "");
+ String columnType = column.getDataType().toString();
+ DataType dataType = MySqlTypeUtils.toDataType(columnType);
+ boolean notNull = column.toString().toUpperCase().contains("NOT NULL");
+ dataType = notNull ? dataType.notNull() : dataType.nullable();
+ result =
+ result.stream()
+ .filter(dataField -> !dataField.name().equals(columnName))
+ .collect(Collectors.toList());
+ result.add(
+ new DataField(
+ id++,
+ caseSensitive ? columnName : columnName.toLowerCase(),
+ dataType));
+ mySqlFieldTypes.put(columnName, columnType);
+ paimonFieldTypes.put(columnName, MySqlTypeUtils.toDataType(columnType));
+ }
+ } else if (sqlAlterTableItem instanceof SQLAlterTableDropColumnItem) {
+ SQLAlterTableDropColumnItem sqlAlterTableDropColumnItem =
+ (SQLAlterTableDropColumnItem) sqlAlterTableItem;
+ List<String> columnNames =
+ sqlAlterTableDropColumnItem.getColumns().stream()
+ .map(SQLName::getSimpleName)
+ .collect(Collectors.toList());
+ columnNames.forEach(
+ columnName -> {
+ mySqlFieldTypes.remove(columnName);
+ paimonFieldTypes.remove(columnName);
+ });
+
+ result =
+ result.stream()
+ .filter(dataField -> !columnNames.contains(dataField.name()))
+ .collect(Collectors.toList());
+ } else if (sqlAlterTableItem instanceof MySqlAlterTableModifyColumn) {
+ MySqlAlterTableModifyColumn mySqlAlterTableModifyColumn =
+ (MySqlAlterTableModifyColumn) sqlAlterTableItem;
+ SQLColumnDefinition newColumnDefinition =
+ mySqlAlterTableModifyColumn.getNewColumnDefinition();
+ String columnName = newColumnDefinition.getColumnName().replace("`", "");
+ String columnType = newColumnDefinition.getDataType().toString();
+ DataType dataType = MySqlTypeUtils.toDataType(columnType);
+ boolean notNull =
+ newColumnDefinition.toString().toUpperCase().contains("NOT NULL");
+ dataType = notNull ? dataType.notNull() : dataType.nullable();
+ DataType finalDataType = dataType;
+ result =
+ result.stream()
+ .map(
+ dataField -> {
+ if (dataField.name().equals(columnName)) {
+ dataField =
+ new DataField(
+ dataField.id(),
+ caseSensitive
+ ? columnName
+ : columnName
+ .toLowerCase(),
+ finalDataType);
+ }
+ return dataField;
+ })
+ .collect(Collectors.toList());
+ mySqlFieldTypes.put(columnName, columnType);
+ paimonFieldTypes.put(columnName, MySqlTypeUtils.toDataType(columnType));
+ } else if (sqlAlterTableItem instanceof MySqlAlterTableChangeColumn) {
+ MySqlAlterTableChangeColumn mySqlAlterTableChangeColumn =
+ (MySqlAlterTableChangeColumn) sqlAlterTableItem;
+ SQLColumnDefinition newColumnDefinition =
+ mySqlAlterTableChangeColumn.getNewColumnDefinition();
+ String oldColumnName =
+ mySqlAlterTableChangeColumn
+ .getColumnName()
+ .getSimpleName()
+ .replace("`", "");
+ String columnName = newColumnDefinition.getColumnName().replace("`", "");
+ String columnType = newColumnDefinition.getDataType().toString();
+ DataType dataType = MySqlTypeUtils.toDataType(columnType);
+ boolean notNull =
+ newColumnDefinition.toString().toUpperCase().contains("NOT NULL");
+ dataType = notNull ? dataType.notNull() : dataType.nullable();
+ result =
+ result.stream()
+ .filter(dataField -> !dataField.name().equals(oldColumnName))
+ .collect(Collectors.toList());
+ result.add(
+ new DataField(
+ id,
+ caseSensitive ? columnName : columnName.toLowerCase(),
+ dataType));
+ mySqlFieldTypes.remove(oldColumnName);
+ paimonFieldTypes.remove(oldColumnName);
+ mySqlFieldTypes.put(columnName, columnType);
+ paimonFieldTypes.put(columnName, MySqlTypeUtils.toDataType(columnType));
+ }
+ }
+ }
+
+ return Optional.of(result);
+ }
+
+ @Override
+ public List<CdcRecord> getRecords() {
+ List<CdcRecord> records = new ArrayList<>();
+ String type = root.get(TYPE).asText();
+ if (OP_UPDATE.equals(type)) {
+ ArrayNode data = (ArrayNode) root.get(FIELD_DATA);
+ ArrayNode old =
+ root.get(FIELD_OLD) instanceof NullNode
+ ? null
+ : (ArrayNode) root.get(FIELD_OLD);
+ for (int i = 0; i < data.size(); i++) {
+ Map<String, String> after = extractRow(data.get(i));
+ if (old != null) {
+ Map<String, String> before = extractRow(old.get(i));
+ // fields in "old" (before) means the fields are changed
+ // fields not in "old" (before) means the fields are not changed
+ // so we just copy the not changed fields into before
+ for (Map.Entry<String, String> entry : after.entrySet()) {
+ if (!before.containsKey(entry.getKey())) {
+ before.put(entry.getKey(), entry.getValue());
+ }
+ }
+ before = caseSensitive ? before : keyCaseInsensitive(before);
+ records.add(new CdcRecord(RowKind.DELETE, before));
+ }
+ after = caseSensitive ? after : keyCaseInsensitive(after);
+ records.add(new CdcRecord(RowKind.INSERT, after));
+ }
+ } else if (OP_INSERT.equals(type)) {
+ ArrayNode data = (ArrayNode) root.get(FIELD_DATA);
+ for (int i = 0; i < data.size(); i++) {
+ Map<String, String> after = extractRow(data.get(i));
+ after = caseSensitive ? after : keyCaseInsensitive(after);
+ records.add(new CdcRecord(RowKind.INSERT, after));
+ }
+ } else if (OP_DELETE.equals(type)) {
+ ArrayNode data = (ArrayNode) root.get(FIELD_DATA);
+ for (int i = 0; i < data.size(); i++) {
+ Map<String, String> after = extractRow(data.get(i));
+ after = caseSensitive ? after : keyCaseInsensitive(after);
+ records.add(new CdcRecord(RowKind.DELETE, after));
+ }
+ }
+
+ return records;
+ }
+
+ private Map<String, String> extractRow(JsonNode recordRow) {
+ Map<String, Object> jsonMap =
+ objectMapper.convertValue(recordRow, new TypeReference<Map<String, Object>>() {});
+ if (jsonMap == null) {
+ return new HashMap<>();
+ }
+ Map<String, String> resultMap = new HashMap<>();
+ for (Map.Entry<String, String> field : mySqlFieldTypes.entrySet()) {
+ String fieldName = field.getKey();
+ String mySqlType = field.getValue();
+ Object objectValue = jsonMap.get(fieldName);
+ if (objectValue == null) {
+ continue;
+ }
+
+ String oldValue = objectValue.toString();
+ String newValue = oldValue;
+ if (MySqlTypeUtils.isSetType(MySqlTypeUtils.getShortType(mySqlType))) {
+ newValue = CanalFieldParser.convertSet(newValue, mySqlType);
+ } else if (MySqlTypeUtils.isEnumType(MySqlTypeUtils.getShortType(mySqlType))) {
+ newValue = CanalFieldParser.convertEnum(newValue, mySqlType);
+ } else if (MySqlTypeUtils.isGeoType(MySqlTypeUtils.getShortType(mySqlType))) {
+ try {
+ byte[] wkb =
+ CanalFieldParser.convertGeoType2WkbArray(
+ oldValue.getBytes(StandardCharsets.ISO_8859_1));
+ newValue = MySqlTypeUtils.convertWkbArray(wkb);
+ } catch (Exception e) {
+ throw new IllegalArgumentException(
+ String.format("Failed to convert %s to geometry JSON.", oldValue), e);
+ }
+ }
+ resultMap.put(fieldName, newValue);
+ }
+ // generate values of computed columns
+ for (ComputedColumn computedColumn : computedColumns) {
+ resultMap.put(
+ computedColumn.columnName(),
+ computedColumn.eval(resultMap.get(computedColumn.fieldReference())));
+ }
+
+ return resultMap;
+ }
+
+ private Map<String, String> keyCaseInsensitive(Map<String, String> origin) {
+ Map<String, String> keyCaseInsensitive = new HashMap<>();
+ for (Map.Entry<String, String> entry : origin.entrySet()) {
+ String fieldName = entry.getKey().toLowerCase();
+ checkArgument(
+ !keyCaseInsensitive.containsKey(fieldName),
+ "Duplicate key appears when converting map keys to case-insensitive form. Original map is:\n%s",
+ origin);
+ keyCaseInsensitive.put(fieldName, entry.getValue());
+ }
+ return keyCaseInsensitive;
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
index ad47b8e80..b4db22601 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
@@ -18,6 +18,7 @@
package org.apache.paimon.flink.action.cdc.mysql;
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
import org.apache.paimon.flink.sink.cdc.UpdatedDataFieldsProcessFunction;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.TableSchema;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
index 8e0f15a72..b1e1c019b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
@@ -23,6 +23,8 @@
package org.apache.paimon.flink.action.cdc.mysql;
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
import org.apache.paimon.flink.sink.cdc.CdcRecord;
import org.apache.paimon.flink.sink.cdc.EventParser;
import org.apache.paimon.types.DataField;
@@ -34,15 +36,12 @@ import org.apache.paimon.utils.Preconditions;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectWriter;
-import com.esri.core.geometry.ogc.OGCGeometry;
import org.apache.kafka.connect.json.JsonConverterConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigDecimal;
-import java.nio.ByteBuffer;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -314,25 +313,7 @@ public class MySqlDebeziumJsonEventParser implements EventParser<String> {
JsonNode jsonNode = recordRow.get(fieldName);
try {
byte[] wkb = jsonNode.get("wkb").binaryValue();
- String geoJson = OGCGeometry.fromBinary(ByteBuffer.wrap(wkb)).asGeoJson();
- JsonNode originGeoNode = objectMapper.readTree(geoJson);
-
- Optional<Integer> srid =
- Optional.ofNullable(
- originGeoNode.has("srid")
- ? originGeoNode.get("srid").intValue()
- : null);
- Map<String, Object> geometryInfo = new HashMap<>();
- String geometryType = originGeoNode.get("type").asText();
- geometryInfo.put("type", geometryType);
- if (geometryType.equalsIgnoreCase("GeometryCollection")) {
- geometryInfo.put("geometries", originGeoNode.get("geometries"));
- } else {
- geometryInfo.put("coordinates", originGeoNode.get("coordinates"));
- }
- geometryInfo.put("srid", srid.orElse(0));
- ObjectWriter objectWriter = objectMapper.writer();
- newValue = objectWriter.writeValueAsString(geometryInfo);
+ newValue = MySqlTypeUtils.convertWkbArray(wkb);
} catch (Exception e) {
throw new IllegalArgumentException(
String.format("Failed to convert %s to geometry JSON.", jsonNode), e);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
index 398d7f801..fe01d0332 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.flink.FlinkCatalogFactory;
import org.apache.paimon.flink.FlinkConnectorOptions;
import org.apache.paimon.flink.action.Action;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
import org.apache.paimon.flink.sink.cdc.EventParser;
import org.apache.paimon.flink.sink.cdc.FlinkCdcSyncDatabaseSinkBuilder;
import org.apache.paimon.options.CatalogOptions;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
index 201312aa6..0a418ec5a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.flink.FlinkCatalogFactory;
import org.apache.paimon.flink.FlinkConnectorOptions;
import org.apache.paimon.flink.action.Action;
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
import org.apache.paimon.flink.sink.cdc.EventParser;
import org.apache.paimon.flink.sink.cdc.FlinkCdcSyncTableSinkBuilder;
import org.apache.paimon.options.CatalogOptions;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java
index c4471b032..71fef4cd8 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java
@@ -23,13 +23,28 @@
package org.apache.paimon.flink.action.cdc.mysql;
+import org.apache.paimon.types.BinaryType;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.TimestampType;
-import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.types.VarBinaryType;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectWriter;
+
+import com.esri.core.geometry.ogc.OGCGeometry;
import javax.annotation.Nullable;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
/** Converts from MySQL type to {@link DataType}. */
public class MySqlTypeUtils {
@@ -111,6 +126,22 @@ public class MySqlTypeUtils {
// The base length of a timestamp is 19, for example "2023-03-23 17:20:00".
private static final int JDBC_TIMESTAMP_BASE_LENGTH = 19;
+ private static final String LEFT_BRACKETS = "(";
+ private static final String RIGHT_BRACKETS = ")";
+ private static final String COMMA = ",";
+
+ private static final List<String> HAVE_SCALE_LIST =
+ Arrays.asList(DECIMAL, NUMERIC, DOUBLE, REAL, FIXED);
+
+ private static final ObjectMapper objectMapper = new ObjectMapper();
+
+ public static DataType toDataType(String mysqlType) {
+ return toDataType(
+ MySqlTypeUtils.getShortType(mysqlType),
+ MySqlTypeUtils.getPrecision(mysqlType),
+ MySqlTypeUtils.getScale(mysqlType));
+ }
+
public static DataType toDataType(
String type, @Nullable Integer length, @Nullable Integer scale) {
switch (type.toUpperCase()) {
@@ -197,10 +228,13 @@ public class MySqlTypeUtils {
+ length
+ " for MySQL DATETIME and TIMESTAMP types");
}
+ // because tidb ddl event does not contain field precision
case CHAR:
- return DataTypes.CHAR(Preconditions.checkNotNull(length));
+ return length == null || length == 0 ? DataTypes.STRING() : DataTypes.CHAR(length);
case VARCHAR:
- return DataTypes.VARCHAR(Preconditions.checkNotNull(length));
+ return length == null || length == 0
+ ? DataTypes.STRING()
+ : DataTypes.VARCHAR(length);
case TINYTEXT:
case TEXT:
case MEDIUMTEXT:
@@ -217,9 +251,13 @@ public class MySqlTypeUtils {
case GEOMETRYCOLLECTION:
return DataTypes.STRING();
case BINARY:
- return DataTypes.BINARY(Preconditions.checkNotNull(length));
+ return length == null || length == 0
+ ? DataTypes.BINARY(BinaryType.DEFAULT_LENGTH)
+ : DataTypes.BINARY(length);
case VARBINARY:
- return DataTypes.VARBINARY(Preconditions.checkNotNull(length));
+ return length == null || length == 0
+ ? DataTypes.VARBINARY(VarBinaryType.DEFAULT_LENGTH)
+ : DataTypes.VARBINARY(length);
case TINYBLOB:
case BLOB:
case MEDIUMBLOB:
@@ -232,4 +270,99 @@ public class MySqlTypeUtils {
String.format("Don't support MySQL type '%s' yet.", type));
}
}
+
+ public static boolean isGeoType(String type) {
+ switch (type.toUpperCase()) {
+ case GEOMETRY:
+ case POINT:
+ case LINESTRING:
+ case POLYGON:
+ case MULTIPOINT:
+ case MULTILINESTRING:
+ case MULTIPOLYGON:
+ case GEOMETRYCOLLECTION:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ public static String convertWkbArray(byte[] wkb) throws JsonProcessingException {
+ String geoJson = OGCGeometry.fromBinary(ByteBuffer.wrap(wkb)).asGeoJson();
+ JsonNode originGeoNode = objectMapper.readTree(geoJson);
+
+ Optional<Integer> srid =
+ Optional.ofNullable(
+ originGeoNode.has("srid") ? originGeoNode.get("srid").intValue() : null);
+ Map<String, Object> geometryInfo = new HashMap<>();
+ String geometryType = originGeoNode.get("type").asText();
+ geometryInfo.put("type", geometryType);
+ if (geometryType.equalsIgnoreCase("GeometryCollection")) {
+ geometryInfo.put("geometries", originGeoNode.get("geometries"));
+ } else {
+ geometryInfo.put("coordinates", originGeoNode.get("coordinates"));
+ }
+ geometryInfo.put("srid", srid.orElse(0));
+ ObjectWriter objectWriter = objectMapper.writer();
+ return objectWriter.writeValueAsString(geometryInfo);
+ }
+
+ public static boolean isScaleType(String typeName) {
+ return HAVE_SCALE_LIST.stream()
+ .anyMatch(type -> getShortType(typeName).toUpperCase().startsWith(type));
+ }
+
+ public static boolean isEnumType(String typeName) {
+ return typeName.toUpperCase().startsWith(ENUM);
+ }
+
+ public static boolean isSetType(String typeName) {
+ return typeName.toUpperCase().startsWith(SET);
+ }
+
+ /* Get type after the brackets are removed.*/
+ public static String getShortType(String typeName) {
+
+ if (typeName.contains(LEFT_BRACKETS) && typeName.contains(RIGHT_BRACKETS)) {
+ return typeName.substring(0, typeName.indexOf(LEFT_BRACKETS)).trim()
+ + typeName.substring(typeName.indexOf(RIGHT_BRACKETS) + 1);
+ } else {
+ return typeName;
+ }
+ }
+
+ public static int getPrecision(String typeName) {
+ if (typeName.contains(LEFT_BRACKETS)
+ && typeName.contains(RIGHT_BRACKETS)
+ && isScaleType(typeName)) {
+ return Integer.parseInt(
+ typeName.substring(typeName.indexOf(LEFT_BRACKETS) + 1, typeName.indexOf(COMMA))
+ .trim());
+ } else if ((typeName.contains(LEFT_BRACKETS)
+ && typeName.contains(RIGHT_BRACKETS)
+ && !isScaleType(typeName)
+ && !isEnumType(typeName)
+ && !isSetType(typeName))) {
+ return Integer.parseInt(
+ typeName.substring(
+ typeName.indexOf(LEFT_BRACKETS) + 1,
+ typeName.indexOf(RIGHT_BRACKETS))
+ .trim());
+ } else {
+ return 0;
+ }
+ }
+
+ public static int getScale(String typeName) {
+ if (typeName.contains(LEFT_BRACKETS)
+ && typeName.contains(RIGHT_BRACKETS)
+ && isScaleType(typeName)) {
+ return Integer.parseInt(
+ typeName.substring(
+ typeName.indexOf(COMMA) + 1, typeName.indexOf(RIGHT_BRACKETS))
+ .trim());
+ } else {
+ return 0;
+ }
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaEventParserTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaEventParserTest.java
new file mode 100644
index 000000000..a8947c4fa
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaEventParserTest.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.kafka;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
+import org.apache.paimon.flink.action.cdc.kafka.canal.CanalJsonEventParser;
+import org.apache.paimon.flink.action.cdc.mysql.Expression;
+import org.apache.paimon.flink.sink.cdc.EventParser;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/** Tests for {@link CanalJsonEventParser}. */
+public class KafkaEventParserTest {
+ private static final String CANAL_JSON_EVENT =
+ "{\"data\":[{\"pt\":\"1\",\"_ID\":\"1\",\"v1\":\"one\","
+ + "\"_geometrycollection\":\"\\u0000\\u0000\\u0000\\u0000\\u0001\\u0007\\u0000\\u0000\\u0000\\u0003"
+ + "\\u0000\\u0000\\u0000\\u0001\\u0001\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000"
+ + "\\u0000$@\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000$@\\u0001\\u0001\\u0000\\u0000\\u0000\\u0000"
+ + "\\u0000\\u0000\\u0000\\u0000\\u0000>@\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000>@\\u0001\\u0002"
+ + "\\u0000\\u0000\\u0000\\u0002\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000"
+ + ".@\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000"
+ + ".@\\u0000\\u0000\\u0000\\u0000\\u0000\\u00004@\\u0000\\u0000\\u0000\\u0000\\u0000\\u00004@\","
+ + "\"_set\":\"3\",\"_enum\":\"1\"}],\"database\":\"paimon_sync_table\",\"es\":1683006706000,"
+ + "\"id\":92,\"isDdl\":false,\"mysqlType\":{\"pt\":\"INT\",\"_ID\":\"INT\",\"v1\":\"VARCHAR(10)\","
+ + "\"_geometrycollection\":\"GEOMETRYCOLLECTION\",\"_set\":\"SET('a','b','c','d')\",\"_enum\":\"ENUM"
+ + "('value1','value2','value3')\"},\"old\":null,\"pkNames\":[\"_id\"],\"sql\":\"\","
+ + "\"sqlType\":{\"pt\":4,\"_id\":4,\"v1\":12},\"table\":\"schema_evolution_1\",\"ts\":1683006706728,"
+ + "\"type\":\"INSERT\"}\n"
+ + ":null,\"database\":\"paimon_sync_table\",\"es\":1683168078000,\"id\":40,\"isDdl\":true,"
+ + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\"/* Query from "
+ + "DMS-WEBSQL-0-Qid_18293315143325386Y by user 1486767996652600 */ ALTER TABLE schema_evolution_1 "
+ + "MODIFY COLUMN v2 BIGINT\",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168078956,"
+ + "\"type\":\"ALTER\"}";
+ private static final String CANAL_JSON_DDL_ADD_EVENT =
+ "{\"data\":null,\"database\":\"paimon_sync_table\","
+ + "\"es\":1683008289000,\"id\":13,\"isDdl\":true,\"mysqlType\":null,\"old\":null,\"pkNames\":null,"
+ + "\"sql\":\" ALTER TABLE "
+ + "schema_evolution_1 ADD COLUMN v2 INT\",\"sqlType\":null,\"table\":\"schema_evolution_1\","
+ + "\"ts\":1683008289401,\"type\":\"ALTER\"}";
+
+ private static final String CANAL_JSON_DDL_MODIFY_EVENT =
+ "{\"data\":null,\"database\":\"paimon_sync_table\",\"es\":1683168155000,\"id\":54,\"isDdl\":true,"
+ + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\" ALTER TABLE schema_evolution_1 MODIFY "
+ + "COLUMN v1 VARCHAR(20)\",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168154943,"
+ + "\"type\":\"ALTER\"}";
+
+ private static final String CANAL_JSON_DDL_DROP_EVENT =
+ "{\"data\":null,\"database\":\"paimon_sync_table\",\"es\":1683168155000,\"id\":54,\"isDdl\":true,"
+ + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\" ALTER TABLE schema_evolution_1 DROP "
+ + "COLUMN v1 \",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168154943,"
+ + "\"type\":\"ALTER\"}";
+
+ private static final String CANAL_JSON_DDL_CHANGE_EVENT =
+ "{\"data\":null,\"database\":\"paimon_sync_table\",\"es\":1683168155000,\"id\":54,\"isDdl\":true,"
+ + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\" ALTER TABLE schema_evolution_1 CHANGE "
+ + "COLUMN `$% ^,& *(` cg VARCHAR(20) \",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168154943,"
+ + "\"type\":\"ALTER\"}";
+
+ private static final String CANAL_JSON_DDL_MULTI_ADD_EVENT =
+ "{\"data\":null,\"database\":\"paimon_sync_table\","
+ + "\"es\":1683614798000,\"id\":2431,\"isDdl\":true,\"mysqlType\":null,\"old\":null,\"pkNames\":null,"
+ + "\"sql\":\" ALTER TABLE "
+ + "schema_evolution_multiple \\nADD v4 INT,\\nMODIFY COLUMN v1 VARCHAR(30),\\nADD COLUMN (v5 DOUBLE, v6 "
+ + "DECIMAL(5, 3), `$% ^,& *(` VARCHAR(10) COMMENT 'test'),\\n MODIFY v2 BIGINT\",\"sqlType\":null,"
+ + "\"table\":\"schema_evolution_multiple\",\"ts\":1683614799031,\"type\":\"ALTER\"}\n";
+
+ private static final String DEBEZIUM_JSON_EVENT =
+ "{\"before\":null,\"after\":{\"id\":101,\"name\":\"scooter\","
+ + "\"description\":\"Small 2-wheel scooter\",\"weight\":3.140000104904175},\"source\":{\"version\":\"1.1.1"
+ + ".Final\",\"connector\":\"mysql\",\"name\":\"dbserver1\",\"ts_ms\":0,\"snapshot\":\"true\","
+ + "\"db\":\"inventory\",\"table\":\"products\",\"server_id\":0,\"gtid\":null,\"file\":\"mysql-bin.000003\","
+ + "\"pos\":154,\"row\":0,\"thread\":null,\"query\":null},\"op\":\"c\",\"ts_ms\":1589355606100,"
+ + "\"transaction\":null}\n";
+
+ @Test
+ public void testCanalJsonEventParser() {
+ boolean caseSensitive = false;
+ EventParser<String> parser =
+ new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+ parser.setRawEvent(CANAL_JSON_EVENT);
+ List<DataField> dataFields = new ArrayList<>();
+ dataFields.add(new DataField(0, "pt", DataTypes.INT()));
+ dataFields.add(new DataField(1, "_id", DataTypes.INT()));
+ dataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+ dataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+ dataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+ dataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+ List<DataField> updatedDataFields = parser.getUpdatedDataFields().orElse(null);
+ assert updatedDataFields == null;
+ List<GenericRow> result =
+ parser.getRecords().stream()
+ .map(record -> record.toGenericRow(dataFields).get())
+ .collect(Collectors.toList());
+ BinaryString[] binaryStrings =
+ new BinaryString[] {BinaryString.fromString("a"), BinaryString.fromString("b")};
+ GenericArray genericArray = new GenericArray(binaryStrings);
+
+ List<GenericRow> expect =
+ Collections.singletonList(
+ GenericRow.of(
+ 1,
+ 1,
+ BinaryString.fromString("one"),
+ BinaryString.fromString(
+ "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"),
+ genericArray,
+ BinaryString.fromString("value1")));
+ assertThat(result).isEqualTo(expect);
+ }
+
+ @Test
+ public void testCanalJsonEventParserDdl() {
+ boolean caseSensitive = false;
+ EventParser<String> parser =
+ new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+ parser.setRawEvent(CANAL_JSON_EVENT);
+ List<DataField> expectDataFields = new ArrayList<>();
+ expectDataFields.add(new DataField(0, "pt", DataTypes.INT()));
+ expectDataFields.add(new DataField(1, "_id", DataTypes.INT()));
+ expectDataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+ expectDataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+ expectDataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+ expectDataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+ expectDataFields.add(new DataField(6, "v2", DataTypes.INT()));
+ List<DataField> updatedDataFields = parser.getUpdatedDataFields().orElse(null);
+ assert updatedDataFields == null;
+ parser.setRawEvent(CANAL_JSON_DDL_ADD_EVENT);
+ List<DataField> updatedDataFieldsAdd = parser.getUpdatedDataFields().orElse(null);
+ assertThat(updatedDataFieldsAdd).isEqualTo(expectDataFields);
+
+ expectDataFields.remove(2);
+ expectDataFields.add(2, new DataField(2, "v1", DataTypes.VARCHAR(20)));
+ parser.setRawEvent(CANAL_JSON_DDL_MODIFY_EVENT);
+ List<DataField> updatedDataFieldsModify = parser.getUpdatedDataFields().orElse(null);
+ assertThat(updatedDataFieldsModify).isEqualTo(expectDataFields);
+ expectDataFields.remove(2);
+ expectDataFields.add(2, new DataField(2, "v1", DataTypes.VARCHAR(30)));
+ expectDataFields.remove(6);
+ expectDataFields.add(new DataField(6, "v2", DataTypes.BIGINT()));
+
+ expectDataFields.add(new DataField(7, "v4", DataTypes.INT()));
+ expectDataFields.add(new DataField(8, "v5", DataTypes.DOUBLE()));
+ expectDataFields.add(new DataField(9, "v6", DataTypes.DECIMAL(5, 3)));
+ expectDataFields.add(new DataField(10, "$% ^,& *(", DataTypes.VARCHAR(10)));
+ parser.setRawEvent(CANAL_JSON_DDL_MULTI_ADD_EVENT);
+ List<DataField> updatedDataFieldsMulti = parser.getUpdatedDataFields().orElse(null);
+ assertThat(updatedDataFieldsMulti).isEqualTo(expectDataFields);
+ parser.setRawEvent(CANAL_JSON_DDL_DROP_EVENT);
+ List<DataField> updatedDataFieldsDrop = parser.getUpdatedDataFields().orElse(null);
+ for (int i = 0; i < expectDataFields.size(); i++) {
+ expectDataFields.set(
+ i,
+ new DataField(
+ i, expectDataFields.get(i).name(), expectDataFields.get(i).type()));
+ }
+ expectDataFields.remove(2);
+ assertThat(updatedDataFieldsDrop).isEqualTo(expectDataFields);
+ parser.setRawEvent(CANAL_JSON_DDL_CHANGE_EVENT);
+ List<DataField> updatedDataFieldsChange = parser.getUpdatedDataFields().orElse(null);
+ expectDataFields.remove(9);
+ for (int i = 0; i < expectDataFields.size(); i++) {
+ expectDataFields.set(
+ i,
+ new DataField(
+ i, expectDataFields.get(i).name(), expectDataFields.get(i).type()));
+ }
+ expectDataFields.add(new DataField(10, "cg", DataTypes.VARCHAR(20)));
+ assertThat(updatedDataFieldsChange).isEqualTo(expectDataFields);
+ }
+
+ @Test
+ public void testCanalJsonEventParserParseDebeziumJson() {
+ boolean caseSensitive = true;
+ EventParser<String> parser =
+ new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+
+ RuntimeException e =
+ assertThrows(
+ RuntimeException.class,
+ () -> parser.setRawEvent(DEBEZIUM_JSON_EVENT),
+ "Expecting RuntimeException");
+ assertThat(e)
+ .hasMessage(
+ "java.lang.NullPointerException: CanalJsonEventParser only supports canal-json format,please make sure that your topic's format is accurate.");
+ }
+
+ @Test
+ public void testCaseSensitive() {
+ boolean caseSensitive = false;
+ EventParser<String> parserCaseInsensitive =
+ new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+ EventParser<String> parserCaseSensitive =
+ new CanalJsonEventParser(!caseSensitive, new TableNameConverter(!caseSensitive));
+ parserCaseInsensitive.setRawEvent(CANAL_JSON_EVENT);
+ List<DataField> dataFields = new ArrayList<>();
+ dataFields.add(new DataField(0, "pt", DataTypes.INT()));
+ dataFields.add(new DataField(1, "_id", DataTypes.INT()));
+ dataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+ dataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+ dataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+ dataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+ List<DataField> updatedDataFields =
+ parserCaseInsensitive.getUpdatedDataFields().orElse(null);
+ assert updatedDataFields == null;
+ List<GenericRow> result =
+ parserCaseInsensitive.getRecords().stream()
+ .map(record -> record.toGenericRow(dataFields).get())
+ .collect(Collectors.toList());
+ BinaryString[] binaryStrings =
+ new BinaryString[] {BinaryString.fromString("a"), BinaryString.fromString("b")};
+ GenericArray genericArray = new GenericArray(binaryStrings);
+
+ List<GenericRow> expect =
+ Collections.singletonList(
+ GenericRow.of(
+ 1,
+ 1,
+ BinaryString.fromString("one"),
+ BinaryString.fromString(
+ "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"),
+ genericArray,
+ BinaryString.fromString("value1")));
+ assertThat(result).isEqualTo(expect);
+ parserCaseSensitive.setRawEvent(CANAL_JSON_EVENT);
+ Optional<GenericRow> genericRow =
+ parserCaseSensitive.getRecords().get(0).toGenericRow(dataFields);
+ assert !genericRow.isPresent();
+ dataFields.remove(1);
+ dataFields.add(1, new DataField(1, "_ID", DataTypes.INT()));
+ List<GenericRow> resultCaseSensitive =
+ parserCaseSensitive.getRecords().stream()
+ .map(record -> record.toGenericRow(dataFields).get())
+ .collect(Collectors.toList());
+ assertThat(resultCaseSensitive).isEqualTo(expect);
+ }
+
+ @Test
+ public void testCanalJsonEventParserAndComputeColumn() {
+ boolean caseSensitive = false;
+ EventParser<String> parser =
+ new CanalJsonEventParser(
+ caseSensitive,
+ new TableNameConverter(caseSensitive),
+ Collections.singletonList(
+ new ComputedColumn("v1", Expression.substring("v1", "1"))));
+ parser.setRawEvent(CANAL_JSON_EVENT);
+ List<DataField> dataFields = new ArrayList<>();
+ dataFields.add(new DataField(0, "pt", DataTypes.INT()));
+ dataFields.add(new DataField(1, "_id", DataTypes.INT()));
+ dataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+ dataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+ dataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+ dataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+ List<DataField> updatedDataFields = parser.getUpdatedDataFields().orElse(null);
+ assert updatedDataFields == null;
+ List<GenericRow> result =
+ parser.getRecords().stream()
+ .map(record -> record.toGenericRow(dataFields).get())
+ .collect(Collectors.toList());
+ BinaryString[] binaryStrings =
+ new BinaryString[] {BinaryString.fromString("a"), BinaryString.fromString("b")};
+ GenericArray genericArray = new GenericArray(binaryStrings);
+
+ List<GenericRow> expect =
+ Collections.singletonList(
+ GenericRow.of(
+ 1,
+ 1,
+ BinaryString.fromString("ne"),
+ BinaryString.fromString(
+ "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"),
+ genericArray,
+ BinaryString.fromString("value1")));
+ assertThat(result).isEqualTo(expect);
+ }
+}