You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/05/26 04:06:22 UTC

[incubator-paimon] branch master updated: [flink] Add KafkaCanalEventParser (#1226)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new ae2b99634 [flink] Add KafkaCanalEventParser (#1226)
ae2b99634 is described below

commit ae2b996341be6e7666a5b1b0bb263f07cdd8d197
Author: HZY <48...@users.noreply.github.com>
AuthorDate: Fri May 26 12:06:17 2023 +0800

    [flink] Add KafkaCanalEventParser (#1226)
---
 .../java/org/apache/paimon/utils/TypeUtils.java    |  10 +-
 paimon-flink/paimon-flink-common/pom.xml           |   6 +
 .../action/cdc/{mysql => }/ComputedColumn.java     |   7 +-
 .../action/cdc/{mysql => }/TableNameConverter.java |   6 +-
 .../action/cdc/kafka/canal/CanalFieldParser.java   | 126 +++++++
 .../cdc/kafka/canal/CanalJsonEventParser.java      | 381 +++++++++++++++++++++
 .../flink/action/cdc/mysql/MySqlActionUtils.java   |   1 +
 .../cdc/mysql/MySqlDebeziumJsonEventParser.java    |  25 +-
 .../action/cdc/mysql/MySqlSyncDatabaseAction.java  |   1 +
 .../action/cdc/mysql/MySqlSyncTableAction.java     |   1 +
 .../flink/action/cdc/mysql/MySqlTypeUtils.java     | 143 +++++++-
 .../action/cdc/kafka/KafkaEventParserTest.java     | 307 +++++++++++++++++
 12 files changed, 979 insertions(+), 35 deletions(-)

diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
index b853d4881..cfcfe7e26 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/TypeUtils.java
@@ -106,8 +106,14 @@ public class TypeUtils {
                 if (d == ((float) d)) {
                     return (float) d;
                 } else {
-                    throw new NumberFormatException(
-                            s + " cannot be cast to float due to precision loss");
+                    // Compatible canal-cdc
+                    Float f = Float.valueOf(s);
+                    if (f.toString().length() != s.length()) {
+                        throw new NumberFormatException(
+                                s + " cannot be cast to float due to precision loss");
+                    } else {
+                        return f;
+                    }
                 }
             case DOUBLE:
                 return Double.valueOf(s);
diff --git a/paimon-flink/paimon-flink-common/pom.xml b/paimon-flink/paimon-flink-common/pom.xml
index c12b9587a..2c3e066c0 100644
--- a/paimon-flink/paimon-flink-common/pom.xml
+++ b/paimon-flink/paimon-flink-common/pom.xml
@@ -38,6 +38,7 @@ under the License.
         <flink.cdc.version>2.3.0</flink.cdc.version>
         <frocksdbjni.version>6.20.3-ververica-2.0</frocksdbjni.version>
         <geometry.version>2.2.0</geometry.version>
+        <druid.version>1.2.15</druid.version>
     </properties>
 
     <dependencies>
@@ -267,6 +268,11 @@ under the License.
             <version>${testcontainers.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>druid</artifactId>
+            <version>${druid.version}</version>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/ComputedColumn.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/ComputedColumn.java
similarity index 90%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/ComputedColumn.java
rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/ComputedColumn.java
index e489366f3..df8142120 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/ComputedColumn.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/ComputedColumn.java
@@ -16,8 +16,9 @@
  * limitations under the License.
  */
 
-package org.apache.paimon.flink.action.cdc.mysql;
+package org.apache.paimon.flink.action.cdc;
 
+import org.apache.paimon.flink.action.cdc.mysql.Expression;
 import org.apache.paimon.types.DataType;
 
 import javax.annotation.Nullable;
@@ -48,13 +49,13 @@ public class ComputedColumn implements Serializable {
         return expression.outputType();
     }
 
-    String fieldReference() {
+    public String fieldReference() {
         return expression.fieldReference();
     }
 
     /** Compute column's value from given argument. Return null if input is null. */
     @Nullable
-    String eval(@Nullable String input) {
+    public String eval(@Nullable String input) {
         if (input == null) {
             return null;
         }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/TableNameConverter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java
similarity index 88%
rename from paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/TableNameConverter.java
rename to paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java
index 94996b48a..692b10252 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/TableNameConverter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/TableNameConverter.java
@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.paimon.flink.action.cdc.mysql;
+package org.apache.paimon.flink.action.cdc;
 
 import java.io.Serializable;
 
@@ -29,11 +29,11 @@ public class TableNameConverter implements Serializable {
     private final String prefix;
     private final String suffix;
 
-    TableNameConverter(boolean caseSensitive) {
+    public TableNameConverter(boolean caseSensitive) {
         this(caseSensitive, "", "");
     }
 
-    TableNameConverter(boolean caseSensitive, String prefix, String suffix) {
+    public TableNameConverter(boolean caseSensitive, String prefix, String suffix) {
         this.caseSensitive = caseSensitive;
         this.prefix = prefix;
         this.suffix = suffix;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalFieldParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalFieldParser.java
new file mode 100644
index 000000000..b65c7396b
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalFieldParser.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.kafka.canal;
+
+/** Converts some special types such as enum、set、geometry. */
+public class CanalFieldParser {
+
+    protected static byte[] convertGeoType2WkbArray(byte[] mysqlGeomBytes) {
+        if (mysqlGeomBytes == null) {
+            return null;
+        }
+        int sridLength = 4;
+        boolean hasSrid = false;
+        for (int i = 0; i < sridLength; ++i) {
+            if (mysqlGeomBytes[i] != 0) {
+                hasSrid = true;
+                break;
+            }
+        }
+        byte[] wkb;
+        if (hasSrid) {
+            wkb = new byte[mysqlGeomBytes.length];
+            // byteOrder + geometry
+            System.arraycopy(mysqlGeomBytes, 4, wkb, 0, 5);
+            // srid
+            System.arraycopy(mysqlGeomBytes, 0, wkb, 5, 4);
+            // geometry
+            System.arraycopy(mysqlGeomBytes, 9, wkb, 9, wkb.length - 9);
+
+            // set srid flag
+            if (wkb[0] == 0) {
+                // big endian
+                wkb[1] = (byte) (wkb[1] + 32);
+            } else {
+                wkb[4] = (byte) (wkb[4] + 32);
+            }
+        } else {
+            wkb = new byte[mysqlGeomBytes.length - 4];
+            System.arraycopy(mysqlGeomBytes, 4, wkb, 0, wkb.length);
+        }
+        return wkb;
+    }
+
+    protected static String convertSet(String value, String mysqlType) {
+        if (value == null) {
+            return null;
+        }
+        // mysql set type value can be filled with more than one, value is a bit string conversion
+        // from the long
+        int indexes = Integer.parseInt(value);
+        return getSetValuesByIndex(mysqlType, indexes);
+    }
+
+    protected static String convertEnum(String value, String mysqlType) {
+        if (value == null) {
+            return null;
+        }
+        int elementIndex = Integer.parseInt(value);
+        // enum('a','b','c')
+        return getEnumValueByIndex(mysqlType, elementIndex);
+    }
+
+    protected static String getEnumValueByIndex(String mysqlType, int elementIndex) {
+        String[] options = extractEnumValueByIndex(mysqlType);
+
+        return options[elementIndex - 1];
+    }
+
+    protected static String getSetValuesByIndex(String mysqlType, int indexes) {
+        String[] options = extractSetValuesByIndex(mysqlType);
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("[");
+        int index = 0;
+        boolean first = true;
+        int optionLen = options.length;
+
+        while (indexes != 0L) {
+            if (indexes % 2L != 0) {
+                if (first) {
+                    first = false;
+                } else {
+                    sb.append(',');
+                }
+                if (index < optionLen) {
+                    sb.append(options[index]);
+                } else {
+                    throw new RuntimeException(
+                            String.format(
+                                    "extractSetValues from mysqlType[%s],index:%d failed",
+                                    mysqlType, indexes));
+                }
+            }
+            ++index;
+            indexes = indexes >>> 1;
+        }
+        sb.append("]");
+        return sb.toString();
+    }
+
+    private static String[] extractSetValuesByIndex(String mysqlType) {
+        // set('x','y')
+        return mysqlType.substring(5, mysqlType.length() - 2).split("'\\s*,\\s*'");
+    }
+
+    private static String[] extractEnumValueByIndex(String mysqlType) {
+        // enum('x','y')
+        return mysqlType.substring(6, mysqlType.length() - 2).split("'\\s*,\\s*'");
+    }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalJsonEventParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalJsonEventParser.java
new file mode 100644
index 000000000..6c2441a3a
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/kafka/canal/CanalJsonEventParser.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.kafka.canal;
+
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
+import org.apache.paimon.flink.action.cdc.mysql.MySqlTypeUtils;
+import org.apache.paimon.flink.sink.cdc.CdcRecord;
+import org.apache.paimon.flink.sink.cdc.EventParser;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.RowKind;
+import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.utils.StringUtils;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.NullNode;
+
+import com.alibaba.druid.DbType;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLName;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableAddColumn;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableDropColumnItem;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableItem;
+import com.alibaba.druid.sql.ast.statement.SQLAlterTableStatement;
+import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlAlterTableChangeColumn;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlAlterTableModifyColumn;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** {@link EventParser} for Canal-json. */
+public class CanalJsonEventParser implements EventParser<String> {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CanalJsonEventParser.class);
+
+    private static final String FIELD_DATA = "data";
+    private static final String FIELD_OLD = "old";
+    private static final String TYPE = "type";
+    private static final String OP_INSERT = "INSERT";
+    private static final String OP_UPDATE = "UPDATE";
+    private static final String OP_DELETE = "DELETE";
+    private static final String OP_CREATE = "CREATE";
+
+    private final ObjectMapper objectMapper = new ObjectMapper();
+
+    private JsonNode root;
+    private Map<String, String> mySqlFieldTypes;
+    private Map<String, DataType> paimonFieldTypes;
+
+    private final boolean caseSensitive;
+    private final TableNameConverter tableNameConverter;
+    private final List<ComputedColumn> computedColumns;
+
+    public CanalJsonEventParser(boolean caseSensitive, List<ComputedColumn> computedColumns) {
+        this(caseSensitive, new TableNameConverter(caseSensitive), computedColumns);
+    }
+
+    public CanalJsonEventParser(boolean caseSensitive, TableNameConverter tableNameConverter) {
+        this(caseSensitive, tableNameConverter, Collections.emptyList());
+    }
+
+    public CanalJsonEventParser(
+            boolean caseSensitive,
+            TableNameConverter tableNameConverter,
+            List<ComputedColumn> computedColumns) {
+        this.caseSensitive = caseSensitive;
+        this.tableNameConverter = tableNameConverter;
+        this.computedColumns = computedColumns;
+    }
+
+    @Override
+    public void setRawEvent(String rawEvent) {
+        try {
+            root = objectMapper.readValue(rawEvent, JsonNode.class);
+            Preconditions.checkNotNull(
+                    root.get("mysqlType"),
+                    "CanalJsonEventParser only supports canal-json format,"
+                            + "please make sure that your topic's format is accurate.");
+            JsonNode mysqlType = root.get("mysqlType");
+
+            if (!isUpdatedDataFields()) {
+                updateFieldTypes(mysqlType);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public String tableName() {
+        String tableName = root.get("table").asText();
+        return tableNameConverter.convert(tableName);
+    }
+
+    private void updateFieldTypes(JsonNode schema) {
+        mySqlFieldTypes = new LinkedHashMap<>();
+        paimonFieldTypes = new LinkedHashMap<>();
+        Iterator<String> iterator = schema.fieldNames();
+        while (iterator.hasNext()) {
+            String fieldName = iterator.next();
+            String fieldType = schema.get(fieldName).asText();
+            mySqlFieldTypes.put(fieldName, fieldType);
+            paimonFieldTypes.put(fieldName, MySqlTypeUtils.toDataType(fieldType));
+        }
+    }
+
+    @Override
+    public boolean isUpdatedDataFields() {
+        if (root.get("isDdl") == null) {
+            return false;
+        } else {
+            return "true".equals(root.get("isDdl").asText());
+        }
+    }
+
+    @Override
+    public Optional<List<DataField>> getUpdatedDataFields() {
+        String sql = root.get("sql").asText();
+
+        if (StringUtils.isEmpty(sql)) {
+            return Optional.empty();
+        }
+
+        List<DataField> result = new ArrayList<>();
+        int id = 0;
+        for (Map.Entry<String, DataType> fieldType : paimonFieldTypes.entrySet()) {
+            result.add(
+                    new DataField(
+                            id++,
+                            caseSensitive ? fieldType.getKey() : fieldType.getKey().toLowerCase(),
+                            fieldType.getValue()));
+        }
+        SQLStatement sqlStatement = SQLUtils.parseSingleStatement(sql, DbType.mysql);
+        if (sqlStatement instanceof SQLAlterTableStatement) {
+            SQLAlterTableStatement sqlAlterTableStatement = (SQLAlterTableStatement) sqlStatement;
+            for (int i = 0; i < sqlAlterTableStatement.getItems().size(); i++) {
+                SQLAlterTableItem sqlAlterTableItem = sqlAlterTableStatement.getItems().get(i);
+                if (sqlAlterTableItem instanceof SQLAlterTableAddColumn) {
+                    SQLAlterTableAddColumn sqlAlterTableAddColumn =
+                            (SQLAlterTableAddColumn) sqlAlterTableItem;
+                    List<SQLColumnDefinition> columns = sqlAlterTableAddColumn.getColumns();
+                    for (SQLColumnDefinition column : columns) {
+                        String columnName = column.getColumnName().replace("`", "");
+                        String columnType = column.getDataType().toString();
+                        DataType dataType = MySqlTypeUtils.toDataType(columnType);
+                        boolean notNull = column.toString().toUpperCase().contains("NOT NULL");
+                        dataType = notNull ? dataType.notNull() : dataType.nullable();
+                        result =
+                                result.stream()
+                                        .filter(dataField -> !dataField.name().equals(columnName))
+                                        .collect(Collectors.toList());
+                        result.add(
+                                new DataField(
+                                        id++,
+                                        caseSensitive ? columnName : columnName.toLowerCase(),
+                                        dataType));
+                        mySqlFieldTypes.put(columnName, columnType);
+                        paimonFieldTypes.put(columnName, MySqlTypeUtils.toDataType(columnType));
+                    }
+                } else if (sqlAlterTableItem instanceof SQLAlterTableDropColumnItem) {
+                    SQLAlterTableDropColumnItem sqlAlterTableDropColumnItem =
+                            (SQLAlterTableDropColumnItem) sqlAlterTableItem;
+                    List<String> columnNames =
+                            sqlAlterTableDropColumnItem.getColumns().stream()
+                                    .map(SQLName::getSimpleName)
+                                    .collect(Collectors.toList());
+                    columnNames.forEach(
+                            columnName -> {
+                                mySqlFieldTypes.remove(columnName);
+                                paimonFieldTypes.remove(columnName);
+                            });
+
+                    result =
+                            result.stream()
+                                    .filter(dataField -> !columnNames.contains(dataField.name()))
+                                    .collect(Collectors.toList());
+                } else if (sqlAlterTableItem instanceof MySqlAlterTableModifyColumn) {
+                    MySqlAlterTableModifyColumn mySqlAlterTableModifyColumn =
+                            (MySqlAlterTableModifyColumn) sqlAlterTableItem;
+                    SQLColumnDefinition newColumnDefinition =
+                            mySqlAlterTableModifyColumn.getNewColumnDefinition();
+                    String columnName = newColumnDefinition.getColumnName().replace("`", "");
+                    String columnType = newColumnDefinition.getDataType().toString();
+                    DataType dataType = MySqlTypeUtils.toDataType(columnType);
+                    boolean notNull =
+                            newColumnDefinition.toString().toUpperCase().contains("NOT NULL");
+                    dataType = notNull ? dataType.notNull() : dataType.nullable();
+                    DataType finalDataType = dataType;
+                    result =
+                            result.stream()
+                                    .map(
+                                            dataField -> {
+                                                if (dataField.name().equals(columnName)) {
+                                                    dataField =
+                                                            new DataField(
+                                                                    dataField.id(),
+                                                                    caseSensitive
+                                                                            ? columnName
+                                                                            : columnName
+                                                                                    .toLowerCase(),
+                                                                    finalDataType);
+                                                }
+                                                return dataField;
+                                            })
+                                    .collect(Collectors.toList());
+                    mySqlFieldTypes.put(columnName, columnType);
+                    paimonFieldTypes.put(columnName, MySqlTypeUtils.toDataType(columnType));
+                } else if (sqlAlterTableItem instanceof MySqlAlterTableChangeColumn) {
+                    MySqlAlterTableChangeColumn mySqlAlterTableChangeColumn =
+                            (MySqlAlterTableChangeColumn) sqlAlterTableItem;
+                    SQLColumnDefinition newColumnDefinition =
+                            mySqlAlterTableChangeColumn.getNewColumnDefinition();
+                    String oldColumnName =
+                            mySqlAlterTableChangeColumn
+                                    .getColumnName()
+                                    .getSimpleName()
+                                    .replace("`", "");
+                    String columnName = newColumnDefinition.getColumnName().replace("`", "");
+                    String columnType = newColumnDefinition.getDataType().toString();
+                    DataType dataType = MySqlTypeUtils.toDataType(columnType);
+                    boolean notNull =
+                            newColumnDefinition.toString().toUpperCase().contains("NOT NULL");
+                    dataType = notNull ? dataType.notNull() : dataType.nullable();
+                    result =
+                            result.stream()
+                                    .filter(dataField -> !dataField.name().equals(oldColumnName))
+                                    .collect(Collectors.toList());
+                    result.add(
+                            new DataField(
+                                    id,
+                                    caseSensitive ? columnName : columnName.toLowerCase(),
+                                    dataType));
+                    mySqlFieldTypes.remove(oldColumnName);
+                    paimonFieldTypes.remove(oldColumnName);
+                    mySqlFieldTypes.put(columnName, columnType);
+                    paimonFieldTypes.put(columnName, MySqlTypeUtils.toDataType(columnType));
+                }
+            }
+        }
+
+        return Optional.of(result);
+    }
+
+    @Override
+    public List<CdcRecord> getRecords() {
+        List<CdcRecord> records = new ArrayList<>();
+        String type = root.get(TYPE).asText();
+        if (OP_UPDATE.equals(type)) {
+            ArrayNode data = (ArrayNode) root.get(FIELD_DATA);
+            ArrayNode old =
+                    root.get(FIELD_OLD) instanceof NullNode
+                            ? null
+                            : (ArrayNode) root.get(FIELD_OLD);
+            for (int i = 0; i < data.size(); i++) {
+                Map<String, String> after = extractRow(data.get(i));
+                if (old != null) {
+                    Map<String, String> before = extractRow(old.get(i));
+                    // fields in "old" (before) means the fields are changed
+                    // fields not in "old" (before) means the fields are not changed
+                    // so we just copy the not changed fields into before
+                    for (Map.Entry<String, String> entry : after.entrySet()) {
+                        if (!before.containsKey(entry.getKey())) {
+                            before.put(entry.getKey(), entry.getValue());
+                        }
+                    }
+                    before = caseSensitive ? before : keyCaseInsensitive(before);
+                    records.add(new CdcRecord(RowKind.DELETE, before));
+                }
+                after = caseSensitive ? after : keyCaseInsensitive(after);
+                records.add(new CdcRecord(RowKind.INSERT, after));
+            }
+        } else if (OP_INSERT.equals(type)) {
+            ArrayNode data = (ArrayNode) root.get(FIELD_DATA);
+            for (int i = 0; i < data.size(); i++) {
+                Map<String, String> after = extractRow(data.get(i));
+                after = caseSensitive ? after : keyCaseInsensitive(after);
+                records.add(new CdcRecord(RowKind.INSERT, after));
+            }
+        } else if (OP_DELETE.equals(type)) {
+            ArrayNode data = (ArrayNode) root.get(FIELD_DATA);
+            for (int i = 0; i < data.size(); i++) {
+                Map<String, String> after = extractRow(data.get(i));
+                after = caseSensitive ? after : keyCaseInsensitive(after);
+                records.add(new CdcRecord(RowKind.DELETE, after));
+            }
+        }
+
+        return records;
+    }
+
+    private Map<String, String> extractRow(JsonNode recordRow) {
+        Map<String, Object> jsonMap =
+                objectMapper.convertValue(recordRow, new TypeReference<Map<String, Object>>() {});
+        if (jsonMap == null) {
+            return new HashMap<>();
+        }
+        Map<String, String> resultMap = new HashMap<>();
+        for (Map.Entry<String, String> field : mySqlFieldTypes.entrySet()) {
+            String fieldName = field.getKey();
+            String mySqlType = field.getValue();
+            Object objectValue = jsonMap.get(fieldName);
+            if (objectValue == null) {
+                continue;
+            }
+
+            String oldValue = objectValue.toString();
+            String newValue = oldValue;
+            if (MySqlTypeUtils.isSetType(MySqlTypeUtils.getShortType(mySqlType))) {
+                newValue = CanalFieldParser.convertSet(newValue, mySqlType);
+            } else if (MySqlTypeUtils.isEnumType(MySqlTypeUtils.getShortType(mySqlType))) {
+                newValue = CanalFieldParser.convertEnum(newValue, mySqlType);
+            } else if (MySqlTypeUtils.isGeoType(MySqlTypeUtils.getShortType(mySqlType))) {
+                try {
+                    byte[] wkb =
+                            CanalFieldParser.convertGeoType2WkbArray(
+                                    oldValue.getBytes(StandardCharsets.ISO_8859_1));
+                    newValue = MySqlTypeUtils.convertWkbArray(wkb);
+                } catch (Exception e) {
+                    throw new IllegalArgumentException(
+                            String.format("Failed to convert %s to geometry JSON.", oldValue), e);
+                }
+            }
+            resultMap.put(fieldName, newValue);
+        }
+        // generate values of computed columns
+        for (ComputedColumn computedColumn : computedColumns) {
+            resultMap.put(
+                    computedColumn.columnName(),
+                    computedColumn.eval(resultMap.get(computedColumn.fieldReference())));
+        }
+
+        return resultMap;
+    }
+
+    private Map<String, String> keyCaseInsensitive(Map<String, String> origin) {
+        Map<String, String> keyCaseInsensitive = new HashMap<>();
+        for (Map.Entry<String, String> entry : origin.entrySet()) {
+            String fieldName = entry.getKey().toLowerCase();
+            checkArgument(
+                    !keyCaseInsensitive.containsKey(fieldName),
+                    "Duplicate key appears when converting map keys to case-insensitive form. Original map is:\n%s",
+                    origin);
+            keyCaseInsensitive.put(fieldName, entry.getValue());
+        }
+        return keyCaseInsensitive;
+    }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
index ad47b8e80..b4db22601 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
@@ -18,6 +18,7 @@
 
 package org.apache.paimon.flink.action.cdc.mysql;
 
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
 import org.apache.paimon.flink.sink.cdc.UpdatedDataFieldsProcessFunction;
 import org.apache.paimon.schema.Schema;
 import org.apache.paimon.schema.TableSchema;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
index 8e0f15a72..b1e1c019b 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
@@ -23,6 +23,8 @@
 
 package org.apache.paimon.flink.action.cdc.mysql;
 
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
 import org.apache.paimon.flink.sink.cdc.CdcRecord;
 import org.apache.paimon.flink.sink.cdc.EventParser;
 import org.apache.paimon.types.DataField;
@@ -34,15 +36,12 @@ import org.apache.paimon.utils.Preconditions;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectWriter;
 
-import com.esri.core.geometry.ogc.OGCGeometry;
 import org.apache.kafka.connect.json.JsonConverterConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.math.BigDecimal;
-import java.nio.ByteBuffer;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -314,25 +313,7 @@ public class MySqlDebeziumJsonEventParser implements EventParser<String> {
                 JsonNode jsonNode = recordRow.get(fieldName);
                 try {
                     byte[] wkb = jsonNode.get("wkb").binaryValue();
-                    String geoJson = OGCGeometry.fromBinary(ByteBuffer.wrap(wkb)).asGeoJson();
-                    JsonNode originGeoNode = objectMapper.readTree(geoJson);
-
-                    Optional<Integer> srid =
-                            Optional.ofNullable(
-                                    originGeoNode.has("srid")
-                                            ? originGeoNode.get("srid").intValue()
-                                            : null);
-                    Map<String, Object> geometryInfo = new HashMap<>();
-                    String geometryType = originGeoNode.get("type").asText();
-                    geometryInfo.put("type", geometryType);
-                    if (geometryType.equalsIgnoreCase("GeometryCollection")) {
-                        geometryInfo.put("geometries", originGeoNode.get("geometries"));
-                    } else {
-                        geometryInfo.put("coordinates", originGeoNode.get("coordinates"));
-                    }
-                    geometryInfo.put("srid", srid.orElse(0));
-                    ObjectWriter objectWriter = objectMapper.writer();
-                    newValue = objectWriter.writeValueAsString(geometryInfo);
+                    newValue = MySqlTypeUtils.convertWkbArray(wkb);
                 } catch (Exception e) {
                     throw new IllegalArgumentException(
                             String.format("Failed to convert %s to geometry JSON.", jsonNode), e);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
index 398d7f801..fe01d0332 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.flink.FlinkCatalogFactory;
 import org.apache.paimon.flink.FlinkConnectorOptions;
 import org.apache.paimon.flink.action.Action;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
 import org.apache.paimon.flink.sink.cdc.EventParser;
 import org.apache.paimon.flink.sink.cdc.FlinkCdcSyncDatabaseSinkBuilder;
 import org.apache.paimon.options.CatalogOptions;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
index 201312aa6..0a418ec5a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
@@ -23,6 +23,7 @@ import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.flink.FlinkCatalogFactory;
 import org.apache.paimon.flink.FlinkConnectorOptions;
 import org.apache.paimon.flink.action.Action;
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
 import org.apache.paimon.flink.sink.cdc.EventParser;
 import org.apache.paimon.flink.sink.cdc.FlinkCdcSyncTableSinkBuilder;
 import org.apache.paimon.options.CatalogOptions;
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java
index c4471b032..71fef4cd8 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlTypeUtils.java
@@ -23,13 +23,28 @@
 
 package org.apache.paimon.flink.action.cdc.mysql;
 
+import org.apache.paimon.types.BinaryType;
 import org.apache.paimon.types.DataType;
 import org.apache.paimon.types.DataTypes;
 import org.apache.paimon.types.TimestampType;
-import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.types.VarBinaryType;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectWriter;
+
+import com.esri.core.geometry.ogc.OGCGeometry;
 
 import javax.annotation.Nullable;
 
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
 /** Converts from MySQL type to {@link DataType}. */
 public class MySqlTypeUtils {
 
@@ -111,6 +126,22 @@ public class MySqlTypeUtils {
     // The base length of a timestamp is 19, for example "2023-03-23 17:20:00".
     private static final int JDBC_TIMESTAMP_BASE_LENGTH = 19;
 
+    private static final String LEFT_BRACKETS = "(";
+    private static final String RIGHT_BRACKETS = ")";
+    private static final String COMMA = ",";
+
+    private static final List<String> HAVE_SCALE_LIST =
+            Arrays.asList(DECIMAL, NUMERIC, DOUBLE, REAL, FIXED);
+
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+
+    public static DataType toDataType(String mysqlType) {
+        return toDataType(
+                MySqlTypeUtils.getShortType(mysqlType),
+                MySqlTypeUtils.getPrecision(mysqlType),
+                MySqlTypeUtils.getScale(mysqlType));
+    }
+
     public static DataType toDataType(
             String type, @Nullable Integer length, @Nullable Integer scale) {
         switch (type.toUpperCase()) {
@@ -197,10 +228,13 @@ public class MySqlTypeUtils {
                                     + length
                                     + " for MySQL DATETIME and TIMESTAMP types");
                 }
+                // because tidb ddl event does not contain field precision
             case CHAR:
-                return DataTypes.CHAR(Preconditions.checkNotNull(length));
+                return length == null || length == 0 ? DataTypes.STRING() : DataTypes.CHAR(length);
             case VARCHAR:
-                return DataTypes.VARCHAR(Preconditions.checkNotNull(length));
+                return length == null || length == 0
+                        ? DataTypes.STRING()
+                        : DataTypes.VARCHAR(length);
             case TINYTEXT:
             case TEXT:
             case MEDIUMTEXT:
@@ -217,9 +251,13 @@ public class MySqlTypeUtils {
             case GEOMETRYCOLLECTION:
                 return DataTypes.STRING();
             case BINARY:
-                return DataTypes.BINARY(Preconditions.checkNotNull(length));
+                return length == null || length == 0
+                        ? DataTypes.BINARY(BinaryType.DEFAULT_LENGTH)
+                        : DataTypes.BINARY(length);
             case VARBINARY:
-                return DataTypes.VARBINARY(Preconditions.checkNotNull(length));
+                return length == null || length == 0
+                        ? DataTypes.VARBINARY(VarBinaryType.DEFAULT_LENGTH)
+                        : DataTypes.VARBINARY(length);
             case TINYBLOB:
             case BLOB:
             case MEDIUMBLOB:
@@ -232,4 +270,99 @@ public class MySqlTypeUtils {
                         String.format("Don't support MySQL type '%s' yet.", type));
         }
     }
+
+    public static boolean isGeoType(String type) {
+        switch (type.toUpperCase()) {
+            case GEOMETRY:
+            case POINT:
+            case LINESTRING:
+            case POLYGON:
+            case MULTIPOINT:
+            case MULTILINESTRING:
+            case MULTIPOLYGON:
+            case GEOMETRYCOLLECTION:
+                return true;
+            default:
+                return false;
+        }
+    }
+
+    public static String convertWkbArray(byte[] wkb) throws JsonProcessingException {
+        String geoJson = OGCGeometry.fromBinary(ByteBuffer.wrap(wkb)).asGeoJson();
+        JsonNode originGeoNode = objectMapper.readTree(geoJson);
+
+        Optional<Integer> srid =
+                Optional.ofNullable(
+                        originGeoNode.has("srid") ? originGeoNode.get("srid").intValue() : null);
+        Map<String, Object> geometryInfo = new HashMap<>();
+        String geometryType = originGeoNode.get("type").asText();
+        geometryInfo.put("type", geometryType);
+        if (geometryType.equalsIgnoreCase("GeometryCollection")) {
+            geometryInfo.put("geometries", originGeoNode.get("geometries"));
+        } else {
+            geometryInfo.put("coordinates", originGeoNode.get("coordinates"));
+        }
+        geometryInfo.put("srid", srid.orElse(0));
+        ObjectWriter objectWriter = objectMapper.writer();
+        return objectWriter.writeValueAsString(geometryInfo);
+    }
+
+    public static boolean isScaleType(String typeName) {
+        return HAVE_SCALE_LIST.stream()
+                .anyMatch(type -> getShortType(typeName).toUpperCase().startsWith(type));
+    }
+
+    public static boolean isEnumType(String typeName) {
+        return typeName.toUpperCase().startsWith(ENUM);
+    }
+
+    public static boolean isSetType(String typeName) {
+        return typeName.toUpperCase().startsWith(SET);
+    }
+
+    /* Get type after the brackets are removed.*/
+    public static String getShortType(String typeName) {
+
+        if (typeName.contains(LEFT_BRACKETS) && typeName.contains(RIGHT_BRACKETS)) {
+            return typeName.substring(0, typeName.indexOf(LEFT_BRACKETS)).trim()
+                    + typeName.substring(typeName.indexOf(RIGHT_BRACKETS) + 1);
+        } else {
+            return typeName;
+        }
+    }
+
+    public static int getPrecision(String typeName) {
+        if (typeName.contains(LEFT_BRACKETS)
+                && typeName.contains(RIGHT_BRACKETS)
+                && isScaleType(typeName)) {
+            return Integer.parseInt(
+                    typeName.substring(typeName.indexOf(LEFT_BRACKETS) + 1, typeName.indexOf(COMMA))
+                            .trim());
+        } else if ((typeName.contains(LEFT_BRACKETS)
+                && typeName.contains(RIGHT_BRACKETS)
+                && !isScaleType(typeName)
+                && !isEnumType(typeName)
+                && !isSetType(typeName))) {
+            return Integer.parseInt(
+                    typeName.substring(
+                                    typeName.indexOf(LEFT_BRACKETS) + 1,
+                                    typeName.indexOf(RIGHT_BRACKETS))
+                            .trim());
+        } else {
+            return 0;
+        }
+    }
+
+    public static int getScale(String typeName) {
+        if (typeName.contains(LEFT_BRACKETS)
+                && typeName.contains(RIGHT_BRACKETS)
+                && isScaleType(typeName)) {
+            return Integer.parseInt(
+                    typeName.substring(
+                                    typeName.indexOf(COMMA) + 1, typeName.indexOf(RIGHT_BRACKETS))
+                            .trim());
+        } else {
+            return 0;
+        }
+    }
 }
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaEventParserTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaEventParserTest.java
new file mode 100644
index 000000000..a8947c4fa
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaEventParserTest.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.kafka;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericArray;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.flink.action.cdc.ComputedColumn;
+import org.apache.paimon.flink.action.cdc.TableNameConverter;
+import org.apache.paimon.flink.action.cdc.kafka.canal.CanalJsonEventParser;
+import org.apache.paimon.flink.action.cdc.mysql.Expression;
+import org.apache.paimon.flink.sink.cdc.EventParser;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/** Tests for {@link CanalJsonEventParser}. */
+public class KafkaEventParserTest {
+    private static final String CANAL_JSON_EVENT =
+            "{\"data\":[{\"pt\":\"1\",\"_ID\":\"1\",\"v1\":\"one\","
+                    + "\"_geometrycollection\":\"\\u0000\\u0000\\u0000\\u0000\\u0001\\u0007\\u0000\\u0000\\u0000\\u0003"
+                    + "\\u0000\\u0000\\u0000\\u0001\\u0001\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000"
+                    + "\\u0000$@\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000$@\\u0001\\u0001\\u0000\\u0000\\u0000\\u0000"
+                    + "\\u0000\\u0000\\u0000\\u0000\\u0000>@\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000>@\\u0001\\u0002"
+                    + "\\u0000\\u0000\\u0000\\u0002\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000"
+                    + ".@\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000"
+                    + ".@\\u0000\\u0000\\u0000\\u0000\\u0000\\u00004@\\u0000\\u0000\\u0000\\u0000\\u0000\\u00004@\","
+                    + "\"_set\":\"3\",\"_enum\":\"1\"}],\"database\":\"paimon_sync_table\",\"es\":1683006706000,"
+                    + "\"id\":92,\"isDdl\":false,\"mysqlType\":{\"pt\":\"INT\",\"_ID\":\"INT\",\"v1\":\"VARCHAR(10)\","
+                    + "\"_geometrycollection\":\"GEOMETRYCOLLECTION\",\"_set\":\"SET('a','b','c','d')\",\"_enum\":\"ENUM"
+                    + "('value1','value2','value3')\"},\"old\":null,\"pkNames\":[\"_id\"],\"sql\":\"\","
+                    + "\"sqlType\":{\"pt\":4,\"_id\":4,\"v1\":12},\"table\":\"schema_evolution_1\",\"ts\":1683006706728,"
+                    + "\"type\":\"INSERT\"}\n"
+                    + ":null,\"database\":\"paimon_sync_table\",\"es\":1683168078000,\"id\":40,\"isDdl\":true,"
+                    + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\"/* Query from "
+                    + "DMS-WEBSQL-0-Qid_18293315143325386Y by user 1486767996652600 */ ALTER TABLE schema_evolution_1 "
+                    + "MODIFY COLUMN v2 BIGINT\",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168078956,"
+                    + "\"type\":\"ALTER\"}";
+    private static final String CANAL_JSON_DDL_ADD_EVENT =
+            "{\"data\":null,\"database\":\"paimon_sync_table\","
+                    + "\"es\":1683008289000,\"id\":13,\"isDdl\":true,\"mysqlType\":null,\"old\":null,\"pkNames\":null,"
+                    + "\"sql\":\" ALTER TABLE "
+                    + "schema_evolution_1 ADD COLUMN v2 INT\",\"sqlType\":null,\"table\":\"schema_evolution_1\","
+                    + "\"ts\":1683008289401,\"type\":\"ALTER\"}";
+
+    private static final String CANAL_JSON_DDL_MODIFY_EVENT =
+            "{\"data\":null,\"database\":\"paimon_sync_table\",\"es\":1683168155000,\"id\":54,\"isDdl\":true,"
+                    + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\" ALTER TABLE schema_evolution_1 MODIFY "
+                    + "COLUMN v1 VARCHAR(20)\",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168154943,"
+                    + "\"type\":\"ALTER\"}";
+
+    private static final String CANAL_JSON_DDL_DROP_EVENT =
+            "{\"data\":null,\"database\":\"paimon_sync_table\",\"es\":1683168155000,\"id\":54,\"isDdl\":true,"
+                    + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\" ALTER TABLE schema_evolution_1 DROP "
+                    + "COLUMN v1 \",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168154943,"
+                    + "\"type\":\"ALTER\"}";
+
+    private static final String CANAL_JSON_DDL_CHANGE_EVENT =
+            "{\"data\":null,\"database\":\"paimon_sync_table\",\"es\":1683168155000,\"id\":54,\"isDdl\":true,"
+                    + "\"mysqlType\":null,\"old\":null,\"pkNames\":null,\"sql\":\" ALTER TABLE schema_evolution_1 CHANGE "
+                    + "COLUMN `$% ^,& *(` cg VARCHAR(20) \",\"sqlType\":null,\"table\":\"schema_evolution_1\",\"ts\":1683168154943,"
+                    + "\"type\":\"ALTER\"}";
+
+    private static final String CANAL_JSON_DDL_MULTI_ADD_EVENT =
+            "{\"data\":null,\"database\":\"paimon_sync_table\","
+                    + "\"es\":1683614798000,\"id\":2431,\"isDdl\":true,\"mysqlType\":null,\"old\":null,\"pkNames\":null,"
+                    + "\"sql\":\" ALTER TABLE "
+                    + "schema_evolution_multiple \\nADD v4 INT,\\nMODIFY COLUMN v1 VARCHAR(30),\\nADD COLUMN (v5 DOUBLE, v6 "
+                    + "DECIMAL(5, 3), `$% ^,& *(` VARCHAR(10) COMMENT 'test'),\\n            MODIFY v2 BIGINT\",\"sqlType\":null,"
+                    + "\"table\":\"schema_evolution_multiple\",\"ts\":1683614799031,\"type\":\"ALTER\"}\n";
+
+    private static final String DEBEZIUM_JSON_EVENT =
+            "{\"before\":null,\"after\":{\"id\":101,\"name\":\"scooter\","
+                    + "\"description\":\"Small 2-wheel scooter\",\"weight\":3.140000104904175},\"source\":{\"version\":\"1.1.1"
+                    + ".Final\",\"connector\":\"mysql\",\"name\":\"dbserver1\",\"ts_ms\":0,\"snapshot\":\"true\","
+                    + "\"db\":\"inventory\",\"table\":\"products\",\"server_id\":0,\"gtid\":null,\"file\":\"mysql-bin.000003\","
+                    + "\"pos\":154,\"row\":0,\"thread\":null,\"query\":null},\"op\":\"c\",\"ts_ms\":1589355606100,"
+                    + "\"transaction\":null}\n";
+
+    @Test
+    public void testCanalJsonEventParser() {
+        boolean caseSensitive = false;
+        EventParser<String> parser =
+                new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+        parser.setRawEvent(CANAL_JSON_EVENT);
+        List<DataField> dataFields = new ArrayList<>();
+        dataFields.add(new DataField(0, "pt", DataTypes.INT()));
+        dataFields.add(new DataField(1, "_id", DataTypes.INT()));
+        dataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+        dataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+        dataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+        dataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+        List<DataField> updatedDataFields = parser.getUpdatedDataFields().orElse(null);
+        assert updatedDataFields == null;
+        List<GenericRow> result =
+                parser.getRecords().stream()
+                        .map(record -> record.toGenericRow(dataFields).get())
+                        .collect(Collectors.toList());
+        BinaryString[] binaryStrings =
+                new BinaryString[] {BinaryString.fromString("a"), BinaryString.fromString("b")};
+        GenericArray genericArray = new GenericArray(binaryStrings);
+
+        List<GenericRow> expect =
+                Collections.singletonList(
+                        GenericRow.of(
+                                1,
+                                1,
+                                BinaryString.fromString("one"),
+                                BinaryString.fromString(
+                                        "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"),
+                                genericArray,
+                                BinaryString.fromString("value1")));
+        assertThat(result).isEqualTo(expect);
+    }
+
+    @Test
+    public void testCanalJsonEventParserDdl() {
+        boolean caseSensitive = false;
+        EventParser<String> parser =
+                new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+        parser.setRawEvent(CANAL_JSON_EVENT);
+        List<DataField> expectDataFields = new ArrayList<>();
+        expectDataFields.add(new DataField(0, "pt", DataTypes.INT()));
+        expectDataFields.add(new DataField(1, "_id", DataTypes.INT()));
+        expectDataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+        expectDataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+        expectDataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+        expectDataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+        expectDataFields.add(new DataField(6, "v2", DataTypes.INT()));
+        List<DataField> updatedDataFields = parser.getUpdatedDataFields().orElse(null);
+        assert updatedDataFields == null;
+        parser.setRawEvent(CANAL_JSON_DDL_ADD_EVENT);
+        List<DataField> updatedDataFieldsAdd = parser.getUpdatedDataFields().orElse(null);
+        assertThat(updatedDataFieldsAdd).isEqualTo(expectDataFields);
+
+        expectDataFields.remove(2);
+        expectDataFields.add(2, new DataField(2, "v1", DataTypes.VARCHAR(20)));
+        parser.setRawEvent(CANAL_JSON_DDL_MODIFY_EVENT);
+        List<DataField> updatedDataFieldsModify = parser.getUpdatedDataFields().orElse(null);
+        assertThat(updatedDataFieldsModify).isEqualTo(expectDataFields);
+        expectDataFields.remove(2);
+        expectDataFields.add(2, new DataField(2, "v1", DataTypes.VARCHAR(30)));
+        expectDataFields.remove(6);
+        expectDataFields.add(new DataField(6, "v2", DataTypes.BIGINT()));
+
+        expectDataFields.add(new DataField(7, "v4", DataTypes.INT()));
+        expectDataFields.add(new DataField(8, "v5", DataTypes.DOUBLE()));
+        expectDataFields.add(new DataField(9, "v6", DataTypes.DECIMAL(5, 3)));
+        expectDataFields.add(new DataField(10, "$% ^,& *(", DataTypes.VARCHAR(10)));
+        parser.setRawEvent(CANAL_JSON_DDL_MULTI_ADD_EVENT);
+        List<DataField> updatedDataFieldsMulti = parser.getUpdatedDataFields().orElse(null);
+        assertThat(updatedDataFieldsMulti).isEqualTo(expectDataFields);
+        parser.setRawEvent(CANAL_JSON_DDL_DROP_EVENT);
+        List<DataField> updatedDataFieldsDrop = parser.getUpdatedDataFields().orElse(null);
+        for (int i = 0; i < expectDataFields.size(); i++) {
+            expectDataFields.set(
+                    i,
+                    new DataField(
+                            i, expectDataFields.get(i).name(), expectDataFields.get(i).type()));
+        }
+        expectDataFields.remove(2);
+        assertThat(updatedDataFieldsDrop).isEqualTo(expectDataFields);
+        parser.setRawEvent(CANAL_JSON_DDL_CHANGE_EVENT);
+        List<DataField> updatedDataFieldsChange = parser.getUpdatedDataFields().orElse(null);
+        expectDataFields.remove(9);
+        for (int i = 0; i < expectDataFields.size(); i++) {
+            expectDataFields.set(
+                    i,
+                    new DataField(
+                            i, expectDataFields.get(i).name(), expectDataFields.get(i).type()));
+        }
+        expectDataFields.add(new DataField(10, "cg", DataTypes.VARCHAR(20)));
+        assertThat(updatedDataFieldsChange).isEqualTo(expectDataFields);
+    }
+
+    @Test
+    public void testCanalJsonEventParserParseDebeziumJson() {
+        boolean caseSensitive = true;
+        EventParser<String> parser =
+                new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+
+        RuntimeException e =
+                assertThrows(
+                        RuntimeException.class,
+                        () -> parser.setRawEvent(DEBEZIUM_JSON_EVENT),
+                        "Expecting RuntimeException");
+        assertThat(e)
+                .hasMessage(
+                        "java.lang.NullPointerException: CanalJsonEventParser only supports canal-json format,please make sure that your topic's format is accurate.");
+    }
+
+    @Test
+    public void testCaseSensitive() {
+        boolean caseSensitive = false;
+        EventParser<String> parserCaseInsensitive =
+                new CanalJsonEventParser(caseSensitive, new TableNameConverter(caseSensitive));
+        EventParser<String> parserCaseSensitive =
+                new CanalJsonEventParser(!caseSensitive, new TableNameConverter(!caseSensitive));
+        parserCaseInsensitive.setRawEvent(CANAL_JSON_EVENT);
+        List<DataField> dataFields = new ArrayList<>();
+        dataFields.add(new DataField(0, "pt", DataTypes.INT()));
+        dataFields.add(new DataField(1, "_id", DataTypes.INT()));
+        dataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+        dataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+        dataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+        dataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+        List<DataField> updatedDataFields =
+                parserCaseInsensitive.getUpdatedDataFields().orElse(null);
+        assert updatedDataFields == null;
+        List<GenericRow> result =
+                parserCaseInsensitive.getRecords().stream()
+                        .map(record -> record.toGenericRow(dataFields).get())
+                        .collect(Collectors.toList());
+        BinaryString[] binaryStrings =
+                new BinaryString[] {BinaryString.fromString("a"), BinaryString.fromString("b")};
+        GenericArray genericArray = new GenericArray(binaryStrings);
+
+        List<GenericRow> expect =
+                Collections.singletonList(
+                        GenericRow.of(
+                                1,
+                                1,
+                                BinaryString.fromString("one"),
+                                BinaryString.fromString(
+                                        "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"),
+                                genericArray,
+                                BinaryString.fromString("value1")));
+        assertThat(result).isEqualTo(expect);
+        parserCaseSensitive.setRawEvent(CANAL_JSON_EVENT);
+        Optional<GenericRow> genericRow =
+                parserCaseSensitive.getRecords().get(0).toGenericRow(dataFields);
+        assert !genericRow.isPresent();
+        dataFields.remove(1);
+        dataFields.add(1, new DataField(1, "_ID", DataTypes.INT()));
+        List<GenericRow> resultCaseSensitive =
+                parserCaseSensitive.getRecords().stream()
+                        .map(record -> record.toGenericRow(dataFields).get())
+                        .collect(Collectors.toList());
+        assertThat(resultCaseSensitive).isEqualTo(expect);
+    }
+
+    @Test
+    public void testCanalJsonEventParserAndComputeColumn() {
+        boolean caseSensitive = false;
+        EventParser<String> parser =
+                new CanalJsonEventParser(
+                        caseSensitive,
+                        new TableNameConverter(caseSensitive),
+                        Collections.singletonList(
+                                new ComputedColumn("v1", Expression.substring("v1", "1"))));
+        parser.setRawEvent(CANAL_JSON_EVENT);
+        List<DataField> dataFields = new ArrayList<>();
+        dataFields.add(new DataField(0, "pt", DataTypes.INT()));
+        dataFields.add(new DataField(1, "_id", DataTypes.INT()));
+        dataFields.add(new DataField(2, "v1", DataTypes.VARCHAR(10)));
+        dataFields.add(new DataField(3, "_geometrycollection", DataTypes.STRING()));
+        dataFields.add(new DataField(4, "_set", DataTypes.ARRAY(DataTypes.STRING())));
+        dataFields.add(new DataField(5, "_enum", DataTypes.STRING()));
+        List<DataField> updatedDataFields = parser.getUpdatedDataFields().orElse(null);
+        assert updatedDataFields == null;
+        List<GenericRow> result =
+                parser.getRecords().stream()
+                        .map(record -> record.toGenericRow(dataFields).get())
+                        .collect(Collectors.toList());
+        BinaryString[] binaryStrings =
+                new BinaryString[] {BinaryString.fromString("a"), BinaryString.fromString("b")};
+        GenericArray genericArray = new GenericArray(binaryStrings);
+
+        List<GenericRow> expect =
+                Collections.singletonList(
+                        GenericRow.of(
+                                1,
+                                1,
+                                BinaryString.fromString("ne"),
+                                BinaryString.fromString(
+                                        "{\"geometries\":[{\"type\":\"Point\",\"coordinates\":[10,10]},{\"type\":\"Point\",\"coordinates\":[30,30]},{\"type\":\"LineString\",\"coordinates\":[[15,15],[20,20]]}],\"type\":\"GeometryCollection\",\"srid\":0}"),
+                                genericArray,
+                                BinaryString.fromString("value1")));
+        assertThat(result).isEqualTo(expect);
+    }
+}