You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/11/15 09:29:38 UTC

(incubator-paimon) branch master updated: [cdc] Refactor Kafka CDC RecordParser (#2312)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 7a79cbba1 [cdc] Refactor Kafka CDC RecordParser (#2312)
7a79cbba1 is described below

commit 7a79cbba106e3cd024b8562903615dee2e8f7e93
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Wed Nov 15 17:29:34 2023 +0800

    [cdc] Refactor Kafka CDC RecordParser (#2312)
---
 docs/content/cdc-ingestion/kafka-cdc.md            |   8 +-
 .../org/apache/paimon/utils/JsonSerdeUtil.java     |  32 +--
 .../flink/action/cdc/CdcActionCommonUtils.java     |  20 +-
 .../cdc/MessageQueueSyncTableActionBase.java       |   8 +-
 .../flink/action/cdc/format/RecordParser.java      | 231 ++++++++++++---------
 .../action/cdc/format/canal/CanalFieldParser.java  |   9 -
 .../action/cdc/format/canal/CanalRecordParser.java | 162 ++++++++-------
 .../cdc/format/debezium/DebeziumRecordParser.java  |  82 ++++----
 .../cdc/format/maxwell/MaxwellRecordParser.java    |  26 +--
 .../action/cdc/format/ogg/OggRecordParser.java     |  85 ++++----
 .../action/cdc/mongodb/MongoDBSyncTableAction.java |   5 +-
 .../action/cdc/mysql/MySqlSyncDatabaseAction.java  |   3 +-
 .../action/cdc/mysql/MySqlSyncTableAction.java     |   3 +-
 .../flink/sink/cdc/RichCdcMultiplexRecord.java     |  16 +-
 .../cdc/RichCdcMultiplexRecordEventParser.java     |  13 ++
 .../kafka/KafkaCanalSyncDatabaseActionITCase.java  |  32 ++-
 .../cdc/kafka/KafkaCanalSyncTableActionITCase.java |  61 ++++++
 .../canal/database/incomplete/canal-data-1.txt     |  20 ++
 .../kafka/canal/table/incomplete/canal-data-1.txt  |  20 ++
 .../kafka/canal/table/nonpk/canal-data-1.txt       |  20 ++
 20 files changed, 517 insertions(+), 339 deletions(-)

diff --git a/docs/content/cdc-ingestion/kafka-cdc.md b/docs/content/cdc-ingestion/kafka-cdc.md
index 12db1c34c..9cc51b625 100644
--- a/docs/content/cdc-ingestion/kafka-cdc.md
+++ b/docs/content/cdc-ingestion/kafka-cdc.md
@@ -63,7 +63,13 @@ If a message in a Kafka topic is a change event captured from another database u
 </table>
 
 {{< hint info >}}
-In Oracle GoldenGate and Maxwell, the data format synchronized to Kafka does not include field data type information. As a result, Paimon sets the data type for all fields to "String" by default.
+The JSON sources possibly missing some information. For example, Ogg and Maxwell format standards don't contain field 
+types; When you write JSON sources into Flink Kafka sink, it will only reserve data and row type and drop other information. 
+The synchronization job will try best to handle the problem as follows:
+1. If missing field types, Paimon will use 'STRING' type as default. 
+2. If missing database name or table name, you cannot do database synchronization, but you can still do table synchronization.
+3. If missing primary keys, the job might create non primary key table. You can set primary keys when submit job in table 
+synchronization.
 {{< /hint >}}
 
 ## Synchronizing Tables
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java
index 803f4f3da..9ce917d07 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java
@@ -39,12 +39,13 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.Arra
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ser.std.StdSerializer;
 
+import javax.annotation.Nullable;
+
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 /** A utility class that provide abilities for JSON serialization and deserialization. */
@@ -101,22 +102,25 @@ public class JsonSerdeUtil {
      * @param root The root node from which the specific node is to be retrieved.
      * @param fieldName The name of the field to retrieve.
      * @param clazz The class of the node to be returned.
-     * @return The node cast to the specified type.
-     * @throws IllegalArgumentException if the node is not present or if it's not of the expected
-     *     type.
+     * @return The node cast to the specified type or null if not present.
+     * @throws IllegalArgumentException if the node is not of the expected type.
      */
+    @Nullable
     public static <T extends JsonNode> T getNodeAs(
             JsonNode root, String fieldName, Class<T> clazz) {
-        return Optional.ofNullable(root)
-                .map(r -> r.get(fieldName))
-                .filter(clazz::isInstance)
-                .map(clazz::cast)
-                .orElseThrow(
-                        () ->
-                                new IllegalArgumentException(
-                                        String.format(
-                                                "Expected node '%s' to be of type %s but was either not found or of a different type.",
-                                                fieldName, clazz.getName())));
+        JsonNode node = root.get(fieldName);
+        if (node == null) {
+            return null;
+        }
+
+        if (clazz.isInstance(node)) {
+            return clazz.cast(node);
+        }
+
+        throw new IllegalArgumentException(
+                String.format(
+                        "Expected node '%s' to be of type %s but was %s.",
+                        fieldName, clazz.getName(), node.getClass().getName()));
     }
 
     public static <T> T fromJson(String json, Class<T> clazz) {
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java
index b5e59636b..ced474cb7 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java
@@ -145,28 +145,14 @@ public class CdcActionCommonUtils {
         return columnLowerCase;
     }
 
-    public static Schema buildPaimonSchema(
-            List<String> specifiedPartitionKeys,
-            List<String> specifiedPrimaryKeys,
-            List<ComputedColumn> computedColumns,
-            Map<String, String> tableConfig,
-            Schema sourceSchema) {
-        return buildPaimonSchema(
-                specifiedPartitionKeys,
-                specifiedPrimaryKeys,
-                computedColumns,
-                tableConfig,
-                sourceSchema,
-                new CdcMetadataConverter[] {});
-    }
-
     public static Schema buildPaimonSchema(
             List<String> specifiedPartitionKeys,
             List<String> specifiedPrimaryKeys,
             List<ComputedColumn> computedColumns,
             Map<String, String> tableConfig,
             Schema sourceSchema,
-            CdcMetadataConverter[] metadataConverters) {
+            CdcMetadataConverter[] metadataConverters,
+            boolean requirePrimaryKeys) {
         Schema.Builder builder = Schema.newBuilder();
 
         // options
@@ -208,7 +194,7 @@ public class CdcActionCommonUtils {
             builder.primaryKey(specifiedPrimaryKeys);
         } else if (!sourceSchema.primaryKeys().isEmpty()) {
             builder.primaryKey(sourceSchema.primaryKeys());
-        } else {
+        } else if (requirePrimaryKeys) {
             throw new IllegalArgumentException(
                     "Primary keys are not specified. "
                             + "Also, can't infer primary keys from source table schemas because "
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java
index 970f37324..854131eb3 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java
@@ -169,7 +169,9 @@ public abstract class MessageQueueSyncTableActionBase extends ActionBase {
                                 primaryKeys,
                                 computedColumns,
                                 tableConfig,
-                                retrievedSchema);
+                                retrievedSchema,
+                                new CdcMetadataConverter[] {},
+                                false);
                 assertSchemaCompatible(fileStoreTable.schema(), fromMq.fields());
             } catch (MessageQueueSchemaUtils.SchemaRetrievalException e) {
                 LOG.info(
@@ -191,7 +193,9 @@ public abstract class MessageQueueSyncTableActionBase extends ActionBase {
                             primaryKeys,
                             computedColumns,
                             tableConfig,
-                            retrievedSchema);
+                            retrievedSchema,
+                            new CdcMetadataConverter[] {},
+                            false);
 
             catalog.createTable(identifier, fromMq, false);
             fileStoreTable = (FileStoreTable) catalog.getTable(identifier).copy(tableConfig);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java
index b5465ac5e..96042fcb1 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java
@@ -27,18 +27,17 @@ import org.apache.paimon.types.DataType;
 import org.apache.paimon.types.DataTypes;
 import org.apache.paimon.types.RowKind;
 import org.apache.paimon.utils.JsonSerdeUtil;
-import org.apache.paimon.utils.StringUtils;
 
-import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
 
-import org.apache.commons.lang3.BooleanUtils;
 import org.apache.flink.api.common.functions.FlatMapFunction;
 import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.annotation.Nullable;
 
@@ -58,6 +57,7 @@ import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.columnDupl
 import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.listCaseConvert;
 import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.mapKeyCaseConvert;
 import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.recordKeyDuplicateErrMsg;
+import static org.apache.paimon.utils.JsonSerdeUtil.getNodeAs;
 import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
 
 /**
@@ -70,17 +70,17 @@ import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
  */
 public abstract class RecordParser implements FlatMapFunction<String, RichCdcMultiplexRecord> {
 
+    private static final Logger LOG = LoggerFactory.getLogger(RecordParser.class);
+
     protected static final String FIELD_TABLE = "table";
     protected static final String FIELD_DATABASE = "database";
     protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-    protected final boolean caseSensitive;
+
+    private final boolean caseSensitive;
     protected final TypeMapping typeMapping;
     protected final List<ComputedColumn> computedColumns;
-    protected List<String> primaryKeys;
-    protected LinkedHashMap<String, String> fieldTypes;
+
     protected JsonNode root;
-    protected String databaseName;
-    protected String tableName;
 
     public RecordParser(
             boolean caseSensitive, TypeMapping typeMapping, List<ComputedColumn> computedColumns) {
@@ -91,138 +91,126 @@ public abstract class RecordParser implements FlatMapFunction<String, RichCdcMul
 
     @Nullable
     public Schema buildSchema(String record) {
-        this.parseRootJson(record);
-        if (BooleanUtils.isTrue(this.isDDL())) {
-            return null;
-        }
-
-        tableName = extractStringFromRootJson(FIELD_TABLE);
-        this.validateFormat();
-        this.extractPrimaryKeys();
-        this.extractFieldTypesFromDatabaseSchema();
-        LinkedHashMap<String, DataType> paimonFieldTypes = this.setPaimonFieldType();
-
-        Schema.Builder builder = Schema.newBuilder();
-        Set<String> existedFields = new HashSet<>();
-        Function<String, String> columnDuplicateErrMsg = columnDuplicateErrMsg(tableName);
-        for (Map.Entry<String, DataType> entry : paimonFieldTypes.entrySet()) {
-            builder.column(
-                    columnCaseConvertAndDuplicateCheck(
-                            entry.getKey(), existedFields, caseSensitive, columnDuplicateErrMsg),
-                    entry.getValue());
+        try {
+            setRoot(record);
+            if (isDDL()) {
+                return null;
+            }
+
+            LinkedHashMap<String, DataType> paimonFieldTypes = extractPaimonFieldTypes();
+            Schema.Builder builder = Schema.newBuilder();
+            Set<String> existedFields = new HashSet<>();
+            Function<String, String> columnDuplicateErrMsg = columnDuplicateErrMsg(getTableName());
+            for (Map.Entry<String, DataType> entry : paimonFieldTypes.entrySet()) {
+                builder.column(
+                        columnCaseConvertAndDuplicateCheck(
+                                entry.getKey(),
+                                existedFields,
+                                caseSensitive,
+                                columnDuplicateErrMsg),
+                        entry.getValue());
+            }
+
+            builder.primaryKey(listCaseConvert(extractPrimaryKeys(), caseSensitive));
+
+            return builder.build();
+        } catch (Exception e) {
+            logInvalidJsonString(record);
+            throw e;
         }
-
-        builder.primaryKey(listCaseConvert(primaryKeys, caseSensitive));
-
-        return builder.build();
     }
 
     protected abstract List<RichCdcMultiplexRecord> extractRecords();
 
-    protected abstract void validateFormat();
-
     protected abstract String primaryField();
 
     protected abstract String dataField();
 
-    protected Boolean isDDL() {
+    protected boolean isDDL() {
         return false;
     }
 
-    protected String extractStringFromRootJson(String key) {
-        JsonNode node = root.get(key);
-        return isNull(node) ? null : node.asText();
-    }
+    // get field -> type mapping from given data node
+    protected LinkedHashMap<String, DataType> extractPaimonFieldTypes() {
+        JsonNode record = getAndCheck(dataField());
 
-    protected Boolean extractBooleanFromRootJson(String key) {
-        JsonNode node = root.get(key);
-        return isNull(node) ? null : node.asBoolean();
+        return fillDefaultStringTypes(record);
     }
 
-    protected LinkedHashMap<String, DataType> setPaimonFieldType() {
+    // use STRING type in default when we cannot get origin data types (most cases)
+    protected LinkedHashMap<String, DataType> fillDefaultStringTypes(JsonNode record) {
         LinkedHashMap<String, DataType> fieldTypes = new LinkedHashMap<>();
-        JsonNode record = root.get(dataField());
-        if (isNull(record)) {
-            return fieldTypes;
-        }
-        Map<String, Object> linkedHashMap =
-                OBJECT_MAPPER.convertValue(
-                        record, new TypeReference<LinkedHashMap<String, Object>>() {});
-        linkedHashMap.forEach(
-                (column, value) ->
-                        fieldTypes.put(applyCaseSensitiveFieldName(column), DataTypes.STRING()));
+        record.fieldNames().forEachRemaining(name -> fieldTypes.put(name, DataTypes.STRING()));
         return fieldTypes;
     }
 
     @Override
     public void flatMap(String value, Collector<RichCdcMultiplexRecord> out) throws Exception {
-        root = OBJECT_MAPPER.readValue(value, JsonNode.class);
-        this.validateFormat();
-
-        databaseName = extractStringFromRootJson(FIELD_DATABASE);
-        tableName = extractStringFromRootJson(FIELD_TABLE);
-
-        extractRecords().forEach(out::collect);
+        try {
+            setRoot(value);
+            extractRecords().forEach(out::collect);
+        } catch (Exception e) {
+            logInvalidJsonString(value);
+            throw e;
+        }
     }
 
-    protected void extractFieldTypesFromDatabaseSchema() {}
-
     protected Map<String, String> extractRowData(
             JsonNode record, LinkedHashMap<String, DataType> paimonFieldTypes) {
-        Map<String, String> linkedHashMap =
-                OBJECT_MAPPER.convertValue(
-                        record, new TypeReference<LinkedHashMap<String, String>>() {});
-        if (linkedHashMap == null) {
-            return Collections.emptyMap();
-        }
+        paimonFieldTypes.putAll(fillDefaultStringTypes(record));
+        Map<String, String> recordMap =
+                OBJECT_MAPPER.convertValue(record, new TypeReference<Map<String, String>>() {});
 
-        Map<String, String> resultMap = new HashMap<>();
-        linkedHashMap.forEach(
-                (key, value) -> {
-                    paimonFieldTypes.put(applyCaseSensitiveFieldName(key), DataTypes.STRING());
-                    resultMap.put(key, value);
-                });
+        Map<String, String> rowData = new HashMap<>(recordMap);
+        evalComputedColumns(rowData, paimonFieldTypes);
+        return rowData;
+    }
 
-        // generate values for computed columns
+    // generate values for computed columns
+    protected void evalComputedColumns(
+            Map<String, String> rowData, LinkedHashMap<String, DataType> paimonFieldTypes) {
         computedColumns.forEach(
                 computedColumn -> {
-                    resultMap.put(
+                    rowData.put(
                             computedColumn.columnName(),
-                            computedColumn.eval(resultMap.get(computedColumn.fieldReference())));
-                    paimonFieldTypes.put(
-                            applyCaseSensitiveFieldName(computedColumn.columnName()),
-                            computedColumn.columnType());
+                            computedColumn.eval(rowData.get(computedColumn.fieldReference())));
+                    paimonFieldTypes.put(computedColumn.columnName(), computedColumn.columnType());
                 });
-
-        return resultMap;
     }
 
-    protected void extractPrimaryKeys() {
-        ArrayNode pkNames = JsonSerdeUtil.getNodeAs(root, primaryField(), ArrayNode.class);
-        primaryKeys =
-                StreamSupport.stream(pkNames.spliterator(), false)
-                        .map(pk -> applyCaseSensitiveFieldName(pk.asText()))
-                        .collect(Collectors.toList());
-    }
+    private List<String> extractPrimaryKeys() {
+        ArrayNode pkNames = getNodeAs(root, primaryField(), ArrayNode.class);
+        if (pkNames == null) {
+            return Collections.emptyList();
+        }
 
-    protected String applyCaseSensitiveFieldName(String rawName) {
-        return StringUtils.caseSensitiveConversion(rawName, caseSensitive);
+        return StreamSupport.stream(pkNames.spliterator(), false)
+                .map(JsonNode::asText)
+                .collect(Collectors.toList());
     }
 
     protected void processRecord(
             JsonNode jsonNode, RowKind rowKind, List<RichCdcMultiplexRecord> records) {
         LinkedHashMap<String, DataType> paimonFieldTypes = new LinkedHashMap<>(jsonNode.size());
-        this.extractPrimaryKeys();
-        this.extractFieldTypesFromDatabaseSchema();
         Map<String, String> rowData = this.extractRowData(jsonNode, paimonFieldTypes);
-        rowData = mapKeyCaseConvert(rowData, caseSensitive, recordKeyDuplicateErrMsg(rowData));
         records.add(createRecord(rowKind, rowData, paimonFieldTypes));
     }
 
-    protected RichCdcMultiplexRecord createRecord(
+    /** Handle case sensitivity here. */
+    private RichCdcMultiplexRecord createRecord(
             RowKind rowKind,
             Map<String, String> data,
             LinkedHashMap<String, DataType> paimonFieldTypes) {
+        String databaseName = getDatabaseName();
+        String tableName = getTableName();
+        paimonFieldTypes =
+                mapKeyCaseConvert(
+                        paimonFieldTypes,
+                        caseSensitive,
+                        columnDuplicateErrMsg(tableName == null ? "UNKNOWN" : tableName));
+        data = mapKeyCaseConvert(data, caseSensitive, recordKeyDuplicateErrMsg(data));
+        List<String> primaryKeys = listCaseConvert(extractPrimaryKeys(), caseSensitive);
+
         return new RichCdcMultiplexRecord(
                 databaseName,
                 tableName,
@@ -231,12 +219,8 @@ public abstract class RecordParser implements FlatMapFunction<String, RichCdcMul
                 new CdcRecord(rowKind, data));
     }
 
-    protected final void parseRootJson(String record) {
-        try {
-            root = OBJECT_MAPPER.readValue(record, JsonNode.class);
-        } catch (JsonProcessingException e) {
-            throw new RuntimeException("Error processing JSON: " + record, e);
-        }
+    private void setRoot(String record) {
+        root = JsonSerdeUtil.fromJson(record, JsonNode.class);
     }
 
     protected JsonNode mergeOldRecord(JsonNode data, JsonNode oldNode) {
@@ -248,4 +232,51 @@ public abstract class RecordParser implements FlatMapFunction<String, RichCdcMul
                                         .set(fieldName, oldNode.get(fieldName)));
         return oldFullRecordNode;
     }
+
+    @Nullable
+    protected String getTableName() {
+        JsonNode node = root.get(FIELD_TABLE);
+        return isNull(node) ? null : node.asText();
+    }
+
+    @Nullable
+    protected String getDatabaseName() {
+        JsonNode node = root.get(FIELD_DATABASE);
+        return isNull(node) ? null : node.asText();
+    }
+
+    private void logInvalidJsonString(String json) {
+        LOG.info("Invalid Json:\n{}", json);
+    }
+
+    protected void checkNotNull(JsonNode node, String key) {
+        if (isNull(node)) {
+            throw new RuntimeException(
+                    String.format("Invalid %s format: missing '%s' field.", format(), key));
+        }
+    }
+
+    protected void checkNotNull(
+            JsonNode node, String key, String conditionKey, String conditionValue) {
+        if (isNull(node)) {
+            throw new RuntimeException(
+                    String.format(
+                            "Invalid %s format: missing '%s' field when '%s' is '%s'.",
+                            format(), key, conditionKey, conditionValue));
+        }
+    }
+
+    protected JsonNode getAndCheck(String key) {
+        JsonNode node = root.get(key);
+        checkNotNull(node, key);
+        return node;
+    }
+
+    protected JsonNode getAndCheck(String key, String conditionKey, String conditionValue) {
+        JsonNode node = root.get(key);
+        checkNotNull(node, key, conditionKey, conditionValue);
+        return node;
+    }
+
+    protected abstract String format();
 }
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java
index 185e02750..6dcca098e 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java
@@ -22,9 +22,6 @@ package org.apache.paimon.flink.action.cdc.format.canal;
 public class CanalFieldParser {
 
     protected static byte[] convertGeoType2WkbArray(byte[] mysqlGeomBytes) {
-        if (mysqlGeomBytes == null) {
-            return null;
-        }
         int sridLength = 4;
         boolean hasSrid = false;
         for (int i = 0; i < sridLength; ++i) {
@@ -58,9 +55,6 @@ public class CanalFieldParser {
     }
 
     protected static String convertSet(String value, String mysqlType) {
-        if (value == null) {
-            return null;
-        }
         // mysql set type value can be filled with more than one, value is a bit string conversion
         // from the long
         int indexes = Integer.parseInt(value);
@@ -68,9 +62,6 @@ public class CanalFieldParser {
     }
 
     protected static String convertEnum(String value, String mysqlType) {
-        if (value == null) {
-            return null;
-        }
         int elementIndex = Integer.parseInt(value);
         // enum('a','b','c')
         return getEnumValueByIndex(mysqlType, elementIndex);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java
index 0a0e48c20..4f2dc6e49 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java
@@ -25,13 +25,15 @@ import org.apache.paimon.flink.action.cdc.mysql.MySqlTypeUtils;
 import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord;
 import org.apache.paimon.types.DataType;
 import org.apache.paimon.types.RowKind;
-import org.apache.paimon.utils.JsonSerdeUtil;
 
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
 
-import org.apache.commons.lang3.BooleanUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
 
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
@@ -40,11 +42,12 @@ import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
+import static org.apache.paimon.utils.JsonSerdeUtil.getNodeAs;
 import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
 
 /**
  * The {@code CanalRecordParser} class is responsible for parsing records from the Canal-JSON
@@ -61,6 +64,8 @@ import static org.apache.paimon.utils.Preconditions.checkArgument;
  */
 public class CanalRecordParser extends RecordParser {
 
+    private static final Logger LOG = LoggerFactory.getLogger(CanalRecordParser.class);
+
     private static final String FIELD_IS_DDL = "isDdl";
     private static final String FIELD_MYSQL_TYPE = "mysqlType";
     private static final String FIELD_TYPE = "type";
@@ -71,8 +76,9 @@ public class CanalRecordParser extends RecordParser {
     private static final String OP_ROW = "ROW";
 
     @Override
-    protected Boolean isDDL() {
-        return extractBooleanFromRootJson(FIELD_IS_DDL);
+    protected boolean isDDL() {
+        JsonNode node = root.get(FIELD_IS_DDL);
+        return !isNull(node) && node.asBoolean();
     }
 
     public CanalRecordParser(
@@ -80,33 +86,25 @@ public class CanalRecordParser extends RecordParser {
         super(caseSensitive, typeMapping, computedColumns);
     }
 
-    @Override
-    protected void extractFieldTypesFromDatabaseSchema() {
-        JsonNode schema = root.get(FIELD_MYSQL_TYPE);
-        LinkedHashMap<String, String> fieldTypes = new LinkedHashMap<>();
-
-        schema.fieldNames()
-                .forEachRemaining(
-                        fieldName -> {
-                            String fieldType = schema.get(fieldName).asText();
-                            fieldTypes.put(fieldName, fieldType);
-                        });
-        this.fieldTypes = fieldTypes;
-    }
-
     @Override
     public List<RichCdcMultiplexRecord> extractRecords() {
-        if (BooleanUtils.isTrue(this.isDDL())) {
+        if (isDDL()) {
             return Collections.emptyList();
         }
+
         List<RichCdcMultiplexRecord> records = new ArrayList<>();
-        ArrayNode arrayData = JsonSerdeUtil.getNodeAs(root, dataField(), ArrayNode.class);
-        String type = extractStringFromRootJson(FIELD_TYPE);
+
+        ArrayNode arrayData = getNodeAs(root, dataField(), ArrayNode.class);
+        checkNotNull(arrayData, dataField());
+
+        String type = getAndCheck(FIELD_TYPE).asText();
+
         for (JsonNode data : arrayData) {
             switch (type) {
                 case OP_UPDATE:
-                    ArrayNode oldArrayData =
-                            JsonSerdeUtil.getNodeAs(root, FIELD_OLD, ArrayNode.class);
+                    ArrayNode oldArrayData = getNodeAs(root, FIELD_OLD, ArrayNode.class);
+                    checkNotNull(oldArrayData, FIELD_OLD, FIELD_TYPE, type);
+
                     Map<JsonNode, JsonNode> matchedOldRecords =
                             matchOldRecords(arrayData, oldArrayData);
                     JsonNode old = matchedOldRecords.get(data);
@@ -128,33 +126,41 @@ public class CanalRecordParser extends RecordParser {
     }
 
     @Override
-    protected LinkedHashMap<String, DataType> setPaimonFieldType() {
-        LinkedHashMap<String, DataType> paimonFieldTypes = new LinkedHashMap<>();
-        fieldTypes.forEach(
-                (name, type) ->
-                        paimonFieldTypes.put(
-                                applyCaseSensitiveFieldName(name),
-                                MySqlTypeUtils.toDataType(type, typeMapping)));
-        return paimonFieldTypes;
+    protected LinkedHashMap<String, DataType> extractPaimonFieldTypes() {
+        LinkedHashMap<String, String> originalFieldTypes = tryExtractOriginalFieldTypes();
+        if (originalFieldTypes != null) {
+            return toPaimonFieldTypes(originalFieldTypes);
+        }
+
+        // fall back to STRING type
+        ArrayNode records = getNodeAs(root, dataField(), ArrayNode.class);
+        checkNotNull(records, dataField());
+        JsonNode record = records.get(0);
+
+        return fillDefaultStringTypes(record);
     }
 
-    @Override
-    protected void validateFormat() {
-        String errorMessageTemplate =
-                "Didn't find '%s' node in json. Only supports canal-json format,"
-                        + "please make sure your topic's format is correct.";
-
-        checkArgument(!isNull(root.get(FIELD_DATABASE)), errorMessageTemplate, FIELD_DATABASE);
-        checkArgument(!isNull(root.get(FIELD_TABLE)), errorMessageTemplate, FIELD_TABLE);
-        checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
-        checkArgument(!isNull(root.get(FIELD_IS_DDL)), errorMessageTemplate, FIELD_IS_DDL);
-
-        if (!extractBooleanFromRootJson(FIELD_IS_DDL)) {
-            checkArgument(
-                    !isNull(root.get(FIELD_MYSQL_TYPE)), errorMessageTemplate, FIELD_MYSQL_TYPE);
-            checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
-            checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
+    @Nullable
+    private LinkedHashMap<String, String> tryExtractOriginalFieldTypes() {
+        JsonNode schema = root.get(FIELD_MYSQL_TYPE);
+        if (isNull(schema)) {
+            LOG.debug(
+                    "Cannot get original field types because '{}' field is missing.",
+                    FIELD_MYSQL_TYPE);
+            return null;
         }
+
+        return OBJECT_MAPPER.convertValue(
+                schema, new TypeReference<LinkedHashMap<String, String>>() {});
+    }
+
+    private LinkedHashMap<String, DataType> toPaimonFieldTypes(
+            LinkedHashMap<String, String> originalFieldTypes) {
+        LinkedHashMap<String, DataType> paimonFieldTypes = new LinkedHashMap<>();
+        originalFieldTypes.forEach(
+                (name, type) ->
+                        paimonFieldTypes.put(name, MySqlTypeUtils.toDataType(type, typeMapping)));
+        return paimonFieldTypes;
     }
 
     @Override
@@ -170,38 +176,34 @@ public class CanalRecordParser extends RecordParser {
     @Override
     protected Map<String, String> extractRowData(
             JsonNode record, LinkedHashMap<String, DataType> paimonFieldTypes) {
-        fieldTypes.forEach(
-                (name, type) ->
-                        paimonFieldTypes.put(
-                                applyCaseSensitiveFieldName(name),
-                                MySqlTypeUtils.toDataType(type, typeMapping)));
-        Map<String, Object> jsonMap =
+        LinkedHashMap<String, String> originalFieldTypes = tryExtractOriginalFieldTypes();
+        Map<String, Object> recordMap =
                 OBJECT_MAPPER.convertValue(record, new TypeReference<Map<String, Object>>() {});
-        if (jsonMap == null) {
-            return new HashMap<>();
+        Map<String, String> rowData = new HashMap<>();
+
+        if (originalFieldTypes != null) {
+            paimonFieldTypes.putAll(toPaimonFieldTypes(originalFieldTypes));
+            for (Map.Entry<String, Object> entry : recordMap.entrySet()) {
+                String fieldName = entry.getKey();
+                String originalType = originalFieldTypes.get(fieldName);
+                String newValue =
+                        transformValue(Objects.toString(entry.getValue(), null), originalType);
+                rowData.put(fieldName, newValue);
+            }
+        } else {
+            paimonFieldTypes.putAll(fillDefaultStringTypes(record));
+            for (Map.Entry<String, Object> entry : recordMap.entrySet()) {
+                rowData.put(entry.getKey(), Objects.toString(entry.getValue(), null));
+            }
         }
 
-        Map<String, String> resultMap =
-                fieldTypes.entrySet().stream()
-                        .filter(entry -> jsonMap.get(entry.getKey()) != null)
-                        .collect(
-                                Collectors.toMap(
-                                        Map.Entry::getKey,
-                                        entry ->
-                                                transformValue(
-                                                        jsonMap.get(entry.getKey()).toString(),
-                                                        entry.getValue())));
-
-        // generate values for computed columns
-        for (ComputedColumn computedColumn : computedColumns) {
-            resultMap.put(
-                    computedColumn.columnName(),
-                    computedColumn.eval(resultMap.get(computedColumn.fieldReference())));
-            paimonFieldTypes.put(
-                    applyCaseSensitiveFieldName(computedColumn.columnName()),
-                    computedColumn.columnType());
-        }
-        return resultMap;
+        evalComputedColumns(rowData, paimonFieldTypes);
+        return rowData;
+    }
+
+    @Override
+    protected String format() {
+        return "canal-json";
     }
 
     private Map<JsonNode, JsonNode> matchOldRecords(ArrayNode newData, ArrayNode oldData) {
@@ -210,7 +212,11 @@ public class CanalRecordParser extends RecordParser {
                 .collect(Collectors.toMap(newData::get, oldData::get));
     }
 
-    private String transformValue(String oldValue, String mySqlType) {
+    private String transformValue(@Nullable String oldValue, String mySqlType) {
+        if (oldValue == null) {
+            return null;
+        }
+
         String shortType = MySqlTypeUtils.getShortType(mySqlType);
 
         if (MySqlTypeUtils.isSetType(shortType)) {
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java
index b1d858bf4..8bdce72b1 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java
@@ -24,10 +24,13 @@ import org.apache.paimon.flink.action.cdc.format.RecordParser;
 import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord;
 import org.apache.paimon.types.RowKind;
 
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+
+import javax.annotation.Nullable;
+
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.apache.flink.shaded.guava30.com.google.common.base.Preconditions.checkArgument;
 import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
 
 /**
@@ -67,22 +70,20 @@ public class DebeziumRecordParser extends RecordParser {
 
     @Override
     public List<RichCdcMultiplexRecord> extractRecords() {
-        String operation = extractStringFromRootJson(FIELD_TYPE);
+        String operation = getAndCheck(FIELD_TYPE).asText();
         List<RichCdcMultiplexRecord> records = new ArrayList<>();
         switch (operation) {
             case OP_INSERT:
             case OP_READE:
-                processRecord(root.get(dataField()), RowKind.INSERT, records);
+                processRecord(getData(), RowKind.INSERT, records);
                 break;
             case OP_UPDATE:
                 processRecord(
-                        mergeOldRecord(root.get(dataField()), root.get(FIELD_BEFORE)),
-                        RowKind.DELETE,
-                        records);
-                processRecord(root.get(dataField()), RowKind.INSERT, records);
+                        mergeOldRecord(getData(), getBefore(operation)), RowKind.DELETE, records);
+                processRecord(getData(), RowKind.INSERT, records);
                 break;
             case OP_DELETE:
-                processRecord(root.get(FIELD_BEFORE), RowKind.DELETE, records);
+                processRecord(getBefore(operation), RowKind.DELETE, records);
                 break;
             default:
                 throw new UnsupportedOperationException("Unknown record operation: " + operation);
@@ -90,33 +91,12 @@ public class DebeziumRecordParser extends RecordParser {
         return records;
     }
 
-    @Override
-    protected void validateFormat() {
-        String errorMessageTemplate =
-                "Didn't find '%s' node in json. Please make sure your topic's format is correct.";
-        checkArgument(
-                !isNull(root.get(FIELD_SOURCE).get(FIELD_TABLE)),
-                errorMessageTemplate,
-                FIELD_TABLE);
-        checkArgument(
-                !isNull(root.get(FIELD_SOURCE).get(FIELD_DB)),
-                errorMessageTemplate,
-                FIELD_DATABASE);
-        checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
-        String operation = root.get(FIELD_TYPE).asText();
-        switch (operation) {
-            case OP_INSERT:
-            case OP_READE:
-                checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
-                break;
-            case OP_UPDATE:
-            case OP_DELETE:
-                checkArgument(!isNull(root.get(FIELD_BEFORE)), errorMessageTemplate, FIELD_BEFORE);
-                break;
-            default:
-                throw new IllegalArgumentException("Unsupported operation type: " + operation);
-        }
-        checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
+    private JsonNode getData() {
+        return getAndCheck(dataField());
+    }
+
+    private JsonNode getBefore(String op) {
+        return getAndCheck(FIELD_BEFORE, FIELD_TYPE, op);
     }
 
     @Override
@@ -129,15 +109,31 @@ public class DebeziumRecordParser extends RecordParser {
         return FIELD_AFTER;
     }
 
+    @Nullable
+    @Override
+    protected String getTableName() {
+        return getFromSourceField(FIELD_TABLE);
+    }
+
+    @Nullable
     @Override
-    protected String extractStringFromRootJson(String key) {
-        if (key.equals(FIELD_TABLE)) {
-            tableName = root.get(FIELD_SOURCE).get(FIELD_TABLE).asText();
-            return tableName;
-        } else if (key.equals(FIELD_DATABASE)) {
-            databaseName = root.get(FIELD_SOURCE).get(FIELD_DB).asText();
-            return databaseName;
+    protected String getDatabaseName() {
+        return getFromSourceField(FIELD_DB);
+    }
+
+    @Override
+    protected String format() {
+        return "debezium-json";
+    }
+
+    @Nullable
+    private String getFromSourceField(String key) {
+        JsonNode node = root.get(FIELD_SOURCE);
+        if (isNull(node)) {
+            return null;
         }
-        return root.get(key) != null ? root.get(key).asText() : null;
+
+        node = node.get(key);
+        return isNull(node) ? null : node.asText();
     }
 }
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java
index fa75b2b86..7379f81a5 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java
@@ -29,9 +29,6 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-
 /**
  * The {@code MaxwellRecordParser} class extends the abstract {@link RecordParser} and is designed
  * to parse records from Maxwell's JSON change data capture (CDC) format. Maxwell is a CDC solution
@@ -60,15 +57,16 @@ public class MaxwellRecordParser extends RecordParser {
 
     @Override
     public List<RichCdcMultiplexRecord> extractRecords() {
-        String operation = extractStringFromRootJson(FIELD_TYPE);
-        JsonNode data = root.get(dataField());
+        String operation = getAndCheck(FIELD_TYPE).asText();
+        JsonNode data = getAndCheck(dataField());
         List<RichCdcMultiplexRecord> records = new ArrayList<>();
         switch (operation) {
             case OP_INSERT:
                 processRecord(data, RowKind.INSERT, records);
                 break;
             case OP_UPDATE:
-                processRecord(mergeOldRecord(data, root.get(FIELD_OLD)), RowKind.DELETE, records);
+                JsonNode old = getAndCheck(FIELD_OLD, FIELD_TYPE, operation);
+                processRecord(mergeOldRecord(data, old), RowKind.DELETE, records);
                 processRecord(data, RowKind.INSERT, records);
                 break;
             case OP_DELETE:
@@ -80,17 +78,6 @@ public class MaxwellRecordParser extends RecordParser {
         return records;
     }
 
-    @Override
-    protected void validateFormat() {
-        String errorMessageTemplate =
-                "Didn't find '%s' node in json. Please make sure your topic's format is correct.";
-        checkArgument(!isNull(root.get(FIELD_TABLE)), errorMessageTemplate, FIELD_TABLE);
-        checkArgument(!isNull(root.get(FIELD_DATABASE)), errorMessageTemplate, FIELD_DATABASE);
-        checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
-        checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
-        checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
-    }
-
     @Override
     protected String primaryField() {
         return "primary_key_columns";
@@ -100,4 +87,9 @@ public class MaxwellRecordParser extends RecordParser {
     protected String dataField() {
         return "data";
     }
+
+    @Override
+    protected String format() {
+        return "maxwell-json";
+    }
 }
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java
index 1eaf1d9f4..019ae1562 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java
@@ -18,6 +18,7 @@
 
 package org.apache.paimon.flink.action.cdc.format.ogg;
 
+import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.flink.action.cdc.ComputedColumn;
 import org.apache.paimon.flink.action.cdc.TypeMapping;
 import org.apache.paimon.flink.action.cdc.format.RecordParser;
@@ -26,11 +27,12 @@ import org.apache.paimon.types.RowKind;
 
 import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
 
+import javax.annotation.Nullable;
+
 import java.util.ArrayList;
 import java.util.List;
 
 import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
 
 /**
  * The {@code OggRecordParser} class extends the abstract {@link RecordParser} and is responsible
@@ -64,17 +66,17 @@ public class OggRecordParser extends RecordParser {
     @Override
     public List<RichCdcMultiplexRecord> extractRecords() {
         List<RichCdcMultiplexRecord> records = new ArrayList<>();
-        String operation = extractStringFromRootJson(FIELD_TYPE);
+        String operation = getAndCheck(FIELD_TYPE).asText();
         switch (operation) {
             case OP_UPDATE:
-                processRecord(root.get(FIELD_BEFORE), RowKind.DELETE, records);
-                processRecord(root.get(dataField()), RowKind.INSERT, records);
+                processRecord(getBefore(operation), RowKind.DELETE, records);
+                processRecord(getData(), RowKind.INSERT, records);
                 break;
             case OP_INSERT:
-                processRecord(root.get(dataField()), RowKind.INSERT, records);
+                processRecord(getData(), RowKind.INSERT, records);
                 break;
             case OP_DELETE:
-                processRecord(root.get(FIELD_BEFORE), RowKind.DELETE, records);
+                processRecord(getBefore(operation), RowKind.DELETE, records);
                 break;
             default:
                 throw new UnsupportedOperationException("Unknown record operation: " + operation);
@@ -82,41 +84,12 @@ public class OggRecordParser extends RecordParser {
         return records;
     }
 
-    @Override
-    protected void validateFormat() {
-        String errorMessageTemplate =
-                "Didn't find '%s' node in json. Please make sure your topic's format is correct.";
-
-        checkArgument(!isNull(root.get(FIELD_TABLE)), errorMessageTemplate, FIELD_TABLE);
-        checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
-        checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
-
-        String fieldType = root.get(FIELD_TYPE).asText();
-
-        switch (fieldType) {
-            case OP_UPDATE:
-                checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
-                checkArgument(!isNull(root.get(FIELD_BEFORE)), errorMessageTemplate, FIELD_BEFORE);
-                break;
-            case OP_INSERT:
-                checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
-                break;
-            case OP_DELETE:
-                checkArgument(!isNull(root.get(FIELD_BEFORE)), errorMessageTemplate, FIELD_BEFORE);
-                break;
-        }
+    private JsonNode getData() {
+        return getAndCheck(dataField());
     }
 
-    @Override
-    protected String extractStringFromRootJson(String key) {
-        if (key.equals(FIELD_TABLE)) {
-            extractDatabaseAndTableNames();
-            return tableName;
-        } else if (key.equals(FIELD_DATABASE)) {
-            extractDatabaseAndTableNames();
-            return databaseName;
-        }
-        return root.get(key) != null ? root.get(key).asText() : null;
+    private JsonNode getBefore(String op) {
+        return getAndCheck(FIELD_BEFORE, FIELD_TYPE, op);
     }
 
     @Override
@@ -129,14 +102,32 @@ public class OggRecordParser extends RecordParser {
         return "after";
     }
 
-    private void extractDatabaseAndTableNames() {
-        JsonNode tableNode = root.get(FIELD_TABLE);
-        if (tableNode != null) {
-            String[] dbt = tableNode.asText().split("\\.", 2);
-            if (dbt.length == 2) {
-                databaseName = dbt[0];
-                tableName = dbt[1];
-            }
+    @Nullable
+    @Override
+    protected String getTableName() {
+        Identifier id = getTableId();
+        return id == null ? null : id.getObjectName();
+    }
+
+    @Nullable
+    @Override
+    protected String getDatabaseName() {
+        Identifier id = getTableId();
+        return id == null ? null : id.getDatabaseName();
+    }
+
+    @Nullable
+    private Identifier getTableId() {
+        JsonNode node = root.get(FIELD_TABLE);
+        if (isNull(node)) {
+            return null;
         }
+
+        return Identifier.fromString(node.asText());
+    }
+
+    @Override
+    protected String format() {
+        return "ogg-json";
     }
 }
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java
index 91e3892c6..9802b14be 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java
@@ -22,6 +22,7 @@ import org.apache.paimon.annotation.VisibleForTesting;
 import org.apache.paimon.catalog.Identifier;
 import org.apache.paimon.flink.FlinkConnectorOptions;
 import org.apache.paimon.flink.action.ActionBase;
+import org.apache.paimon.flink.action.cdc.CdcMetadataConverter;
 import org.apache.paimon.flink.action.cdc.ComputedColumn;
 import org.apache.paimon.flink.sink.cdc.CdcSinkBuilder;
 import org.apache.paimon.flink.sink.cdc.EventParser;
@@ -140,7 +141,9 @@ public class MongoDBSyncTableAction extends ActionBase {
                         Collections.emptyList(),
                         computedColumns,
                         tableConfig,
-                        mongodbSchema);
+                        mongodbSchema,
+                        new CdcMetadataConverter[] {},
+                        true);
         // Check if table exists before trying to get or create it
         if (catalog.tableExists(identifier)) {
             fileStoreTable = (FileStoreTable) catalog.getTable(identifier);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
index 68c4fa440..931e38c5e 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
@@ -246,7 +246,8 @@ public class MySqlSyncDatabaseAction extends ActionBase {
                             Collections.emptyList(),
                             tableConfig,
                             tableInfo.schema(),
-                            metadataConverters);
+                            metadataConverters,
+                            true);
             try {
                 table = (FileStoreTable) catalog.getTable(identifier);
                 table = table.copy(tableConfig);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
index bca841d2a..b2d38f508 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
@@ -194,7 +194,8 @@ public class MySqlSyncTableAction extends ActionBase {
                         computedColumns,
                         tableConfig,
                         tableInfo.schema(),
-                        metadataConverters);
+                        metadataConverters,
+                        true);
         try {
             fileStoreTable = (FileStoreTable) catalog.getTable(identifier);
             fileStoreTable = fileStoreTable.copy(tableConfig);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java
index 17ffa55e0..0fe0c9eab 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java
@@ -20,6 +20,8 @@ package org.apache.paimon.flink.sink.cdc;
 
 import org.apache.paimon.types.DataType;
 
+import javax.annotation.Nullable;
+
 import java.io.Serializable;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -30,15 +32,15 @@ public class RichCdcMultiplexRecord implements Serializable {
 
     private static final long serialVersionUID = 1L;
 
-    private final String databaseName;
-    private final String tableName;
+    @Nullable private final String databaseName;
+    @Nullable private final String tableName;
     private final LinkedHashMap<String, DataType> fieldTypes;
     private final List<String> primaryKeys;
     private final CdcRecord cdcRecord;
 
     public RichCdcMultiplexRecord(
-            String databaseName,
-            String tableName,
+            @Nullable String databaseName,
+            @Nullable String tableName,
             LinkedHashMap<String, DataType> fieldTypes,
             List<String> primaryKeys,
             CdcRecord cdcRecord) {
@@ -49,10 +51,12 @@ public class RichCdcMultiplexRecord implements Serializable {
         this.cdcRecord = cdcRecord;
     }
 
+    @Nullable
     public String databaseName() {
         return databaseName;
     }
 
+    @Nullable
     public String tableName() {
         return tableName;
     }
@@ -83,8 +87,8 @@ public class RichCdcMultiplexRecord implements Serializable {
             return false;
         }
         RichCdcMultiplexRecord that = (RichCdcMultiplexRecord) o;
-        return databaseName.equals(that.databaseName)
-                && tableName.equals(that.tableName)
+        return Objects.equals(databaseName, that.databaseName)
+                && Objects.equals(tableName, that.tableName)
                 && Objects.equals(fieldTypes, that.fieldTypes)
                 && Objects.equals(primaryKeys, that.primaryKeys)
                 && Objects.equals(cdcRecord, that.cdcRecord);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java
index cdbcd2918..b0d5a77a0 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java
@@ -85,6 +85,13 @@ public class RichCdcMultiplexRecordEventParser implements EventParser<RichCdcMul
 
     @Override
     public String parseTableName() {
+        // database synchronization needs this, so we validate the record here
+        if (record.databaseName() == null || record.tableName() == null) {
+            throw new IllegalArgumentException(
+                    "Cannot synchronize record when database name or table name is unknown. "
+                            + "Invalid record is:\n"
+                            + record);
+        }
         return tableNameConverter.convert(Identifier.create(record.databaseName(), currentTable));
     }
 
@@ -112,6 +119,12 @@ public class RichCdcMultiplexRecordEventParser implements EventParser<RichCdcMul
     }
 
     private boolean shouldSynchronizeCurrentTable() {
+        // In case the record is incomplete, we let the null value pass validation
+        // and handle the null value when we really need it
+        if (currentTable == null) {
+            return true;
+        }
+
         if (includedTables.contains(currentTable)) {
             return true;
         }
diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java
index 62253a6c8..db7ab79ea 100644
--- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java
+++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java
@@ -20,7 +20,6 @@ package org.apache.paimon.flink.action.cdc.kafka;
 
 import org.apache.paimon.options.CatalogOptions;
 import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.testutils.assertj.AssertionUtils;
 import org.apache.paimon.types.DataType;
 import org.apache.paimon.types.DataTypes;
 import org.apache.paimon.types.RowType;
@@ -38,6 +37,7 @@ import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 
 import static org.apache.paimon.flink.action.cdc.TypeMapping.TypeMappingMode.TO_STRING;
+import static org.apache.paimon.testutils.assertj.AssertionUtils.anyCauseMatches;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
@@ -248,7 +248,7 @@ public class KafkaCanalSyncDatabaseActionITCase extends KafkaActionITCaseBase {
 
         assertThatThrownBy(action::run)
                 .satisfies(
-                        AssertionUtils.anyCauseMatches(
+                        anyCauseMatches(
                                 IllegalArgumentException.class,
                                 "kafka-conf [topic] must be specified."));
     }
@@ -614,4 +614,32 @@ public class KafkaCanalSyncDatabaseActionITCase extends KafkaActionITCaseBase {
                 rowType,
                 Collections.singletonList("k1"));
     }
+
+    @Test
+    @Timeout(60)
+    public void testCannotSynchronizeIncompleteJson() throws Exception {
+        final String topic = "incomplete";
+        createTestTopic(topic, 1, 1);
+
+        writeRecordsToKafka(topic, readLines("kafka/canal/database/incomplete/canal-data-1.txt"));
+
+        Map<String, String> kafkaConfig = getBasicKafkaConfig();
+        kafkaConfig.put("value.format", "canal-json");
+        kafkaConfig.put("topic", topic);
+
+        KafkaSyncDatabaseAction action =
+                syncDatabaseActionBuilder(kafkaConfig)
+                        .withTableConfig(getBasicTableConfig())
+                        .build();
+        action.withStreamExecutionEnvironment(env).build();
+
+        assertThatThrownBy(() -> env.execute())
+                .satisfies(
+                        anyCauseMatches(
+                                IllegalArgumentException.class,
+                                "Cannot synchronize record when database name or table name is unknown. "
+                                        + "Invalid record is:\n"
+                                        + "{databaseName=null, tableName=null, fieldTypes={k=STRING, v0=STRING, v1=STRING}, "
+                                        + "primaryKeys=[], cdcRecord=+I {v0=five, k=5, v1=50}}"));
+    }
 }
diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java
index 6cd3d04e3..4e5f31218 100644
--- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java
+++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java
@@ -1072,4 +1072,65 @@ public class KafkaCanalSyncTableActionITCase extends KafkaActionITCaseBase {
                 rowType,
                 Arrays.asList("_id", "_year"));
     }
+
+    @Test
+    @Timeout(60)
+    public void testSynchronizeIncompleteJson() throws Exception {
+        String topic = "incomplete";
+        createTestTopic(topic, 1, 1);
+        List<String> lines = readLines("kafka/canal/table/incomplete/canal-data-1.txt");
+        writeRecordsToKafka(topic, lines);
+
+        Map<String, String> kafkaConfig = getBasicKafkaConfig();
+        kafkaConfig.put("value.format", "canal-json");
+        kafkaConfig.put("topic", topic);
+        KafkaSyncTableAction action =
+                syncTableActionBuilder(kafkaConfig)
+                        .withPrimaryKeys("k")
+                        .withTableConfig(getBasicTableConfig())
+                        .build();
+
+        runActionWithDefaultEnv(action);
+
+        RowType rowType =
+                RowType.of(
+                        new DataType[] {
+                            DataTypes.STRING().notNull(), DataTypes.STRING(),
+                        },
+                        new String[] {"k", "v"});
+        waitForResult(
+                Arrays.asList("+I[1, Hi]", "+I[2, Hello]"),
+                getFileStoreTable(tableName),
+                rowType,
+                Collections.singletonList("k"));
+    }
+
+    @Test
+    @Timeout(60)
+    public void testSynchronizeNonPkTable() throws Exception {
+        String topic = "non_pk";
+        createTestTopic(topic, 1, 1);
+        List<String> lines = readLines("kafka/canal/table/nonpk/canal-data-1.txt");
+        writeRecordsToKafka(topic, lines);
+
+        Map<String, String> kafkaConfig = getBasicKafkaConfig();
+        kafkaConfig.put("value.format", "canal-json");
+        kafkaConfig.put("topic", topic);
+        KafkaSyncTableAction action =
+                syncTableActionBuilder(kafkaConfig).withTableConfig(getBasicTableConfig()).build();
+
+        runActionWithDefaultEnv(action);
+
+        RowType rowType =
+                RowType.of(
+                        new DataType[] {
+                            DataTypes.VARCHAR(10), DataTypes.INT(),
+                        },
+                        new String[] {"v0", "v1"});
+        waitForResult(
+                Arrays.asList("+I[five, 50]", "+I[five, 50]"),
+                getFileStoreTable(tableName),
+                rowType,
+                Collections.emptyList());
+    }
 }
diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/database/incomplete/canal-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/database/incomplete/canal-data-1.txt
new file mode 100644
index 000000000..061c242ea
--- /dev/null
+++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/database/incomplete/canal-data-1.txt
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"data":[{"k":"5","v0":"five","v1":"50"}],"type":"INSERT"}
+
diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/incomplete/canal-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/incomplete/canal-data-1.txt
new file mode 100644
index 000000000..b6ea31253
--- /dev/null
+++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/incomplete/canal-data-1.txt
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"data":[{"k":"1","v":"Hi"}],"type":"INSERT"}
+{"data":[{"k":"2","v":"Hello"}],"type":"INSERT"}
diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/nonpk/canal-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/nonpk/canal-data-1.txt
new file mode 100644
index 000000000..250b5057c
--- /dev/null
+++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/nonpk/canal-data-1.txt
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"data":[{"v0":"five","v1":"50"}],"database":"non_pk","es":1684770072000,"id":81,"isDdl":false,"mysqlType":{"v0":"VARCHAR(10)","v1":"INT"},"old":null,"pkNames":[],"sql":"","sqlType":{"v0":12,"v1":4},"table":"t","ts":1684770072286,"type":"INSERT"}
+{"data":[{"v0":"five","v1":"50"}],"database":"non_pk","es":1684770073000,"id":84,"isDdl":false,"mysqlType":{"v0":"VARCHAR(10)","v1":"INT"},"old":null,"pkNames":[],"sql":"","sqlType":{"v0":12,"v1":4},"table":"t","ts":1684770073254,"type":"INSERT"}