You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/11/15 09:29:38 UTC
(incubator-paimon) branch master updated: [cdc] Refactor Kafka CDC RecordParser (#2312)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7a79cbba1 [cdc] Refactor Kafka CDC RecordParser (#2312)
7a79cbba1 is described below
commit 7a79cbba106e3cd024b8562903615dee2e8f7e93
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Wed Nov 15 17:29:34 2023 +0800
[cdc] Refactor Kafka CDC RecordParser (#2312)
---
docs/content/cdc-ingestion/kafka-cdc.md | 8 +-
.../org/apache/paimon/utils/JsonSerdeUtil.java | 32 +--
.../flink/action/cdc/CdcActionCommonUtils.java | 20 +-
.../cdc/MessageQueueSyncTableActionBase.java | 8 +-
.../flink/action/cdc/format/RecordParser.java | 231 ++++++++++++---------
.../action/cdc/format/canal/CanalFieldParser.java | 9 -
.../action/cdc/format/canal/CanalRecordParser.java | 162 ++++++++-------
.../cdc/format/debezium/DebeziumRecordParser.java | 82 ++++----
.../cdc/format/maxwell/MaxwellRecordParser.java | 26 +--
.../action/cdc/format/ogg/OggRecordParser.java | 85 ++++----
.../action/cdc/mongodb/MongoDBSyncTableAction.java | 5 +-
.../action/cdc/mysql/MySqlSyncDatabaseAction.java | 3 +-
.../action/cdc/mysql/MySqlSyncTableAction.java | 3 +-
.../flink/sink/cdc/RichCdcMultiplexRecord.java | 16 +-
.../cdc/RichCdcMultiplexRecordEventParser.java | 13 ++
.../kafka/KafkaCanalSyncDatabaseActionITCase.java | 32 ++-
.../cdc/kafka/KafkaCanalSyncTableActionITCase.java | 61 ++++++
.../canal/database/incomplete/canal-data-1.txt | 20 ++
.../kafka/canal/table/incomplete/canal-data-1.txt | 20 ++
.../kafka/canal/table/nonpk/canal-data-1.txt | 20 ++
20 files changed, 517 insertions(+), 339 deletions(-)
diff --git a/docs/content/cdc-ingestion/kafka-cdc.md b/docs/content/cdc-ingestion/kafka-cdc.md
index 12db1c34c..9cc51b625 100644
--- a/docs/content/cdc-ingestion/kafka-cdc.md
+++ b/docs/content/cdc-ingestion/kafka-cdc.md
@@ -63,7 +63,13 @@ If a message in a Kafka topic is a change event captured from another database u
</table>
{{< hint info >}}
-In Oracle GoldenGate and Maxwell, the data format synchronized to Kafka does not include field data type information. As a result, Paimon sets the data type for all fields to "String" by default.
+The JSON sources possibly missing some information. For example, Ogg and Maxwell format standards don't contain field
+types; When you write JSON sources into Flink Kafka sink, it will only reserve data and row type and drop other information.
+The synchronization job will try best to handle the problem as follows:
+1. If missing field types, Paimon will use 'STRING' type as default.
+2. If missing database name or table name, you cannot do database synchronization, but you can still do table synchronization.
+3. If missing primary keys, the job might create non primary key table. You can set primary keys when submit job in table
+synchronization.
{{< /hint >}}
## Synchronizing Tables
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java
index 803f4f3da..9ce917d07 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/JsonSerdeUtil.java
@@ -39,12 +39,13 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.Arra
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.stream.Collectors;
/** A utility class that provide abilities for JSON serialization and deserialization. */
@@ -101,22 +102,25 @@ public class JsonSerdeUtil {
* @param root The root node from which the specific node is to be retrieved.
* @param fieldName The name of the field to retrieve.
* @param clazz The class of the node to be returned.
- * @return The node cast to the specified type.
- * @throws IllegalArgumentException if the node is not present or if it's not of the expected
- * type.
+ * @return The node cast to the specified type or null if not present.
+ * @throws IllegalArgumentException if the node is not of the expected type.
*/
+ @Nullable
public static <T extends JsonNode> T getNodeAs(
JsonNode root, String fieldName, Class<T> clazz) {
- return Optional.ofNullable(root)
- .map(r -> r.get(fieldName))
- .filter(clazz::isInstance)
- .map(clazz::cast)
- .orElseThrow(
- () ->
- new IllegalArgumentException(
- String.format(
- "Expected node '%s' to be of type %s but was either not found or of a different type.",
- fieldName, clazz.getName())));
+ JsonNode node = root.get(fieldName);
+ if (node == null) {
+ return null;
+ }
+
+ if (clazz.isInstance(node)) {
+ return clazz.cast(node);
+ }
+
+ throw new IllegalArgumentException(
+ String.format(
+ "Expected node '%s' to be of type %s but was %s.",
+ fieldName, clazz.getName(), node.getClass().getName()));
}
public static <T> T fromJson(String json, Class<T> clazz) {
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java
index b5e59636b..ced474cb7 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtils.java
@@ -145,28 +145,14 @@ public class CdcActionCommonUtils {
return columnLowerCase;
}
- public static Schema buildPaimonSchema(
- List<String> specifiedPartitionKeys,
- List<String> specifiedPrimaryKeys,
- List<ComputedColumn> computedColumns,
- Map<String, String> tableConfig,
- Schema sourceSchema) {
- return buildPaimonSchema(
- specifiedPartitionKeys,
- specifiedPrimaryKeys,
- computedColumns,
- tableConfig,
- sourceSchema,
- new CdcMetadataConverter[] {});
- }
-
public static Schema buildPaimonSchema(
List<String> specifiedPartitionKeys,
List<String> specifiedPrimaryKeys,
List<ComputedColumn> computedColumns,
Map<String, String> tableConfig,
Schema sourceSchema,
- CdcMetadataConverter[] metadataConverters) {
+ CdcMetadataConverter[] metadataConverters,
+ boolean requirePrimaryKeys) {
Schema.Builder builder = Schema.newBuilder();
// options
@@ -208,7 +194,7 @@ public class CdcActionCommonUtils {
builder.primaryKey(specifiedPrimaryKeys);
} else if (!sourceSchema.primaryKeys().isEmpty()) {
builder.primaryKey(sourceSchema.primaryKeys());
- } else {
+ } else if (requirePrimaryKeys) {
throw new IllegalArgumentException(
"Primary keys are not specified. "
+ "Also, can't infer primary keys from source table schemas because "
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java
index 970f37324..854131eb3 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/MessageQueueSyncTableActionBase.java
@@ -169,7 +169,9 @@ public abstract class MessageQueueSyncTableActionBase extends ActionBase {
primaryKeys,
computedColumns,
tableConfig,
- retrievedSchema);
+ retrievedSchema,
+ new CdcMetadataConverter[] {},
+ false);
assertSchemaCompatible(fileStoreTable.schema(), fromMq.fields());
} catch (MessageQueueSchemaUtils.SchemaRetrievalException e) {
LOG.info(
@@ -191,7 +193,9 @@ public abstract class MessageQueueSyncTableActionBase extends ActionBase {
primaryKeys,
computedColumns,
tableConfig,
- retrievedSchema);
+ retrievedSchema,
+ new CdcMetadataConverter[] {},
+ false);
catalog.createTable(identifier, fromMq, false);
fileStoreTable = (FileStoreTable) catalog.getTable(identifier).copy(tableConfig);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java
index b5465ac5e..96042fcb1 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/RecordParser.java
@@ -27,18 +27,17 @@ import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowKind;
import org.apache.paimon.utils.JsonSerdeUtil;
-import org.apache.paimon.utils.StringUtils;
-import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.commons.lang3.BooleanUtils;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
@@ -58,6 +57,7 @@ import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.columnDupl
import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.listCaseConvert;
import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.mapKeyCaseConvert;
import static org.apache.paimon.flink.action.cdc.CdcActionCommonUtils.recordKeyDuplicateErrMsg;
+import static org.apache.paimon.utils.JsonSerdeUtil.getNodeAs;
import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
/**
@@ -70,17 +70,17 @@ import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
*/
public abstract class RecordParser implements FlatMapFunction<String, RichCdcMultiplexRecord> {
+ private static final Logger LOG = LoggerFactory.getLogger(RecordParser.class);
+
protected static final String FIELD_TABLE = "table";
protected static final String FIELD_DATABASE = "database";
protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- protected final boolean caseSensitive;
+
+ private final boolean caseSensitive;
protected final TypeMapping typeMapping;
protected final List<ComputedColumn> computedColumns;
- protected List<String> primaryKeys;
- protected LinkedHashMap<String, String> fieldTypes;
+
protected JsonNode root;
- protected String databaseName;
- protected String tableName;
public RecordParser(
boolean caseSensitive, TypeMapping typeMapping, List<ComputedColumn> computedColumns) {
@@ -91,138 +91,126 @@ public abstract class RecordParser implements FlatMapFunction<String, RichCdcMul
@Nullable
public Schema buildSchema(String record) {
- this.parseRootJson(record);
- if (BooleanUtils.isTrue(this.isDDL())) {
- return null;
- }
-
- tableName = extractStringFromRootJson(FIELD_TABLE);
- this.validateFormat();
- this.extractPrimaryKeys();
- this.extractFieldTypesFromDatabaseSchema();
- LinkedHashMap<String, DataType> paimonFieldTypes = this.setPaimonFieldType();
-
- Schema.Builder builder = Schema.newBuilder();
- Set<String> existedFields = new HashSet<>();
- Function<String, String> columnDuplicateErrMsg = columnDuplicateErrMsg(tableName);
- for (Map.Entry<String, DataType> entry : paimonFieldTypes.entrySet()) {
- builder.column(
- columnCaseConvertAndDuplicateCheck(
- entry.getKey(), existedFields, caseSensitive, columnDuplicateErrMsg),
- entry.getValue());
+ try {
+ setRoot(record);
+ if (isDDL()) {
+ return null;
+ }
+
+ LinkedHashMap<String, DataType> paimonFieldTypes = extractPaimonFieldTypes();
+ Schema.Builder builder = Schema.newBuilder();
+ Set<String> existedFields = new HashSet<>();
+ Function<String, String> columnDuplicateErrMsg = columnDuplicateErrMsg(getTableName());
+ for (Map.Entry<String, DataType> entry : paimonFieldTypes.entrySet()) {
+ builder.column(
+ columnCaseConvertAndDuplicateCheck(
+ entry.getKey(),
+ existedFields,
+ caseSensitive,
+ columnDuplicateErrMsg),
+ entry.getValue());
+ }
+
+ builder.primaryKey(listCaseConvert(extractPrimaryKeys(), caseSensitive));
+
+ return builder.build();
+ } catch (Exception e) {
+ logInvalidJsonString(record);
+ throw e;
}
-
- builder.primaryKey(listCaseConvert(primaryKeys, caseSensitive));
-
- return builder.build();
}
protected abstract List<RichCdcMultiplexRecord> extractRecords();
- protected abstract void validateFormat();
-
protected abstract String primaryField();
protected abstract String dataField();
- protected Boolean isDDL() {
+ protected boolean isDDL() {
return false;
}
- protected String extractStringFromRootJson(String key) {
- JsonNode node = root.get(key);
- return isNull(node) ? null : node.asText();
- }
+ // get field -> type mapping from given data node
+ protected LinkedHashMap<String, DataType> extractPaimonFieldTypes() {
+ JsonNode record = getAndCheck(dataField());
- protected Boolean extractBooleanFromRootJson(String key) {
- JsonNode node = root.get(key);
- return isNull(node) ? null : node.asBoolean();
+ return fillDefaultStringTypes(record);
}
- protected LinkedHashMap<String, DataType> setPaimonFieldType() {
+ // use STRING type in default when we cannot get origin data types (most cases)
+ protected LinkedHashMap<String, DataType> fillDefaultStringTypes(JsonNode record) {
LinkedHashMap<String, DataType> fieldTypes = new LinkedHashMap<>();
- JsonNode record = root.get(dataField());
- if (isNull(record)) {
- return fieldTypes;
- }
- Map<String, Object> linkedHashMap =
- OBJECT_MAPPER.convertValue(
- record, new TypeReference<LinkedHashMap<String, Object>>() {});
- linkedHashMap.forEach(
- (column, value) ->
- fieldTypes.put(applyCaseSensitiveFieldName(column), DataTypes.STRING()));
+ record.fieldNames().forEachRemaining(name -> fieldTypes.put(name, DataTypes.STRING()));
return fieldTypes;
}
@Override
public void flatMap(String value, Collector<RichCdcMultiplexRecord> out) throws Exception {
- root = OBJECT_MAPPER.readValue(value, JsonNode.class);
- this.validateFormat();
-
- databaseName = extractStringFromRootJson(FIELD_DATABASE);
- tableName = extractStringFromRootJson(FIELD_TABLE);
-
- extractRecords().forEach(out::collect);
+ try {
+ setRoot(value);
+ extractRecords().forEach(out::collect);
+ } catch (Exception e) {
+ logInvalidJsonString(value);
+ throw e;
+ }
}
- protected void extractFieldTypesFromDatabaseSchema() {}
-
protected Map<String, String> extractRowData(
JsonNode record, LinkedHashMap<String, DataType> paimonFieldTypes) {
- Map<String, String> linkedHashMap =
- OBJECT_MAPPER.convertValue(
- record, new TypeReference<LinkedHashMap<String, String>>() {});
- if (linkedHashMap == null) {
- return Collections.emptyMap();
- }
+ paimonFieldTypes.putAll(fillDefaultStringTypes(record));
+ Map<String, String> recordMap =
+ OBJECT_MAPPER.convertValue(record, new TypeReference<Map<String, String>>() {});
- Map<String, String> resultMap = new HashMap<>();
- linkedHashMap.forEach(
- (key, value) -> {
- paimonFieldTypes.put(applyCaseSensitiveFieldName(key), DataTypes.STRING());
- resultMap.put(key, value);
- });
+ Map<String, String> rowData = new HashMap<>(recordMap);
+ evalComputedColumns(rowData, paimonFieldTypes);
+ return rowData;
+ }
- // generate values for computed columns
+ // generate values for computed columns
+ protected void evalComputedColumns(
+ Map<String, String> rowData, LinkedHashMap<String, DataType> paimonFieldTypes) {
computedColumns.forEach(
computedColumn -> {
- resultMap.put(
+ rowData.put(
computedColumn.columnName(),
- computedColumn.eval(resultMap.get(computedColumn.fieldReference())));
- paimonFieldTypes.put(
- applyCaseSensitiveFieldName(computedColumn.columnName()),
- computedColumn.columnType());
+ computedColumn.eval(rowData.get(computedColumn.fieldReference())));
+ paimonFieldTypes.put(computedColumn.columnName(), computedColumn.columnType());
});
-
- return resultMap;
}
- protected void extractPrimaryKeys() {
- ArrayNode pkNames = JsonSerdeUtil.getNodeAs(root, primaryField(), ArrayNode.class);
- primaryKeys =
- StreamSupport.stream(pkNames.spliterator(), false)
- .map(pk -> applyCaseSensitiveFieldName(pk.asText()))
- .collect(Collectors.toList());
- }
+ private List<String> extractPrimaryKeys() {
+ ArrayNode pkNames = getNodeAs(root, primaryField(), ArrayNode.class);
+ if (pkNames == null) {
+ return Collections.emptyList();
+ }
- protected String applyCaseSensitiveFieldName(String rawName) {
- return StringUtils.caseSensitiveConversion(rawName, caseSensitive);
+ return StreamSupport.stream(pkNames.spliterator(), false)
+ .map(JsonNode::asText)
+ .collect(Collectors.toList());
}
protected void processRecord(
JsonNode jsonNode, RowKind rowKind, List<RichCdcMultiplexRecord> records) {
LinkedHashMap<String, DataType> paimonFieldTypes = new LinkedHashMap<>(jsonNode.size());
- this.extractPrimaryKeys();
- this.extractFieldTypesFromDatabaseSchema();
Map<String, String> rowData = this.extractRowData(jsonNode, paimonFieldTypes);
- rowData = mapKeyCaseConvert(rowData, caseSensitive, recordKeyDuplicateErrMsg(rowData));
records.add(createRecord(rowKind, rowData, paimonFieldTypes));
}
- protected RichCdcMultiplexRecord createRecord(
+ /** Handle case sensitivity here. */
+ private RichCdcMultiplexRecord createRecord(
RowKind rowKind,
Map<String, String> data,
LinkedHashMap<String, DataType> paimonFieldTypes) {
+ String databaseName = getDatabaseName();
+ String tableName = getTableName();
+ paimonFieldTypes =
+ mapKeyCaseConvert(
+ paimonFieldTypes,
+ caseSensitive,
+ columnDuplicateErrMsg(tableName == null ? "UNKNOWN" : tableName));
+ data = mapKeyCaseConvert(data, caseSensitive, recordKeyDuplicateErrMsg(data));
+ List<String> primaryKeys = listCaseConvert(extractPrimaryKeys(), caseSensitive);
+
return new RichCdcMultiplexRecord(
databaseName,
tableName,
@@ -231,12 +219,8 @@ public abstract class RecordParser implements FlatMapFunction<String, RichCdcMul
new CdcRecord(rowKind, data));
}
- protected final void parseRootJson(String record) {
- try {
- root = OBJECT_MAPPER.readValue(record, JsonNode.class);
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Error processing JSON: " + record, e);
- }
+ private void setRoot(String record) {
+ root = JsonSerdeUtil.fromJson(record, JsonNode.class);
}
protected JsonNode mergeOldRecord(JsonNode data, JsonNode oldNode) {
@@ -248,4 +232,51 @@ public abstract class RecordParser implements FlatMapFunction<String, RichCdcMul
.set(fieldName, oldNode.get(fieldName)));
return oldFullRecordNode;
}
+
+ @Nullable
+ protected String getTableName() {
+ JsonNode node = root.get(FIELD_TABLE);
+ return isNull(node) ? null : node.asText();
+ }
+
+ @Nullable
+ protected String getDatabaseName() {
+ JsonNode node = root.get(FIELD_DATABASE);
+ return isNull(node) ? null : node.asText();
+ }
+
+ private void logInvalidJsonString(String json) {
+ LOG.info("Invalid Json:\n{}", json);
+ }
+
+ protected void checkNotNull(JsonNode node, String key) {
+ if (isNull(node)) {
+ throw new RuntimeException(
+ String.format("Invalid %s format: missing '%s' field.", format(), key));
+ }
+ }
+
+ protected void checkNotNull(
+ JsonNode node, String key, String conditionKey, String conditionValue) {
+ if (isNull(node)) {
+ throw new RuntimeException(
+ String.format(
+ "Invalid %s format: missing '%s' field when '%s' is '%s'.",
+ format(), key, conditionKey, conditionValue));
+ }
+ }
+
+ protected JsonNode getAndCheck(String key) {
+ JsonNode node = root.get(key);
+ checkNotNull(node, key);
+ return node;
+ }
+
+ protected JsonNode getAndCheck(String key, String conditionKey, String conditionValue) {
+ JsonNode node = root.get(key);
+ checkNotNull(node, key, conditionKey, conditionValue);
+ return node;
+ }
+
+ protected abstract String format();
}
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java
index 185e02750..6dcca098e 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalFieldParser.java
@@ -22,9 +22,6 @@ package org.apache.paimon.flink.action.cdc.format.canal;
public class CanalFieldParser {
protected static byte[] convertGeoType2WkbArray(byte[] mysqlGeomBytes) {
- if (mysqlGeomBytes == null) {
- return null;
- }
int sridLength = 4;
boolean hasSrid = false;
for (int i = 0; i < sridLength; ++i) {
@@ -58,9 +55,6 @@ public class CanalFieldParser {
}
protected static String convertSet(String value, String mysqlType) {
- if (value == null) {
- return null;
- }
// mysql set type value can be filled with more than one, value is a bit string conversion
// from the long
int indexes = Integer.parseInt(value);
@@ -68,9 +62,6 @@ public class CanalFieldParser {
}
protected static String convertEnum(String value, String mysqlType) {
- if (value == null) {
- return null;
- }
int elementIndex = Integer.parseInt(value);
// enum('a','b','c')
return getEnumValueByIndex(mysqlType, elementIndex);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java
index 0a0e48c20..4f2dc6e49 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/canal/CanalRecordParser.java
@@ -25,13 +25,15 @@ import org.apache.paimon.flink.action.cdc.mysql.MySqlTypeUtils;
import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowKind;
-import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.node.ArrayNode;
-import org.apache.commons.lang3.BooleanUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
@@ -40,11 +42,12 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
+import static org.apache.paimon.utils.JsonSerdeUtil.getNodeAs;
import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
/**
* The {@code CanalRecordParser} class is responsible for parsing records from the Canal-JSON
@@ -61,6 +64,8 @@ import static org.apache.paimon.utils.Preconditions.checkArgument;
*/
public class CanalRecordParser extends RecordParser {
+ private static final Logger LOG = LoggerFactory.getLogger(CanalRecordParser.class);
+
private static final String FIELD_IS_DDL = "isDdl";
private static final String FIELD_MYSQL_TYPE = "mysqlType";
private static final String FIELD_TYPE = "type";
@@ -71,8 +76,9 @@ public class CanalRecordParser extends RecordParser {
private static final String OP_ROW = "ROW";
@Override
- protected Boolean isDDL() {
- return extractBooleanFromRootJson(FIELD_IS_DDL);
+ protected boolean isDDL() {
+ JsonNode node = root.get(FIELD_IS_DDL);
+ return !isNull(node) && node.asBoolean();
}
public CanalRecordParser(
@@ -80,33 +86,25 @@ public class CanalRecordParser extends RecordParser {
super(caseSensitive, typeMapping, computedColumns);
}
- @Override
- protected void extractFieldTypesFromDatabaseSchema() {
- JsonNode schema = root.get(FIELD_MYSQL_TYPE);
- LinkedHashMap<String, String> fieldTypes = new LinkedHashMap<>();
-
- schema.fieldNames()
- .forEachRemaining(
- fieldName -> {
- String fieldType = schema.get(fieldName).asText();
- fieldTypes.put(fieldName, fieldType);
- });
- this.fieldTypes = fieldTypes;
- }
-
@Override
public List<RichCdcMultiplexRecord> extractRecords() {
- if (BooleanUtils.isTrue(this.isDDL())) {
+ if (isDDL()) {
return Collections.emptyList();
}
+
List<RichCdcMultiplexRecord> records = new ArrayList<>();
- ArrayNode arrayData = JsonSerdeUtil.getNodeAs(root, dataField(), ArrayNode.class);
- String type = extractStringFromRootJson(FIELD_TYPE);
+
+ ArrayNode arrayData = getNodeAs(root, dataField(), ArrayNode.class);
+ checkNotNull(arrayData, dataField());
+
+ String type = getAndCheck(FIELD_TYPE).asText();
+
for (JsonNode data : arrayData) {
switch (type) {
case OP_UPDATE:
- ArrayNode oldArrayData =
- JsonSerdeUtil.getNodeAs(root, FIELD_OLD, ArrayNode.class);
+ ArrayNode oldArrayData = getNodeAs(root, FIELD_OLD, ArrayNode.class);
+ checkNotNull(oldArrayData, FIELD_OLD, FIELD_TYPE, type);
+
Map<JsonNode, JsonNode> matchedOldRecords =
matchOldRecords(arrayData, oldArrayData);
JsonNode old = matchedOldRecords.get(data);
@@ -128,33 +126,41 @@ public class CanalRecordParser extends RecordParser {
}
@Override
- protected LinkedHashMap<String, DataType> setPaimonFieldType() {
- LinkedHashMap<String, DataType> paimonFieldTypes = new LinkedHashMap<>();
- fieldTypes.forEach(
- (name, type) ->
- paimonFieldTypes.put(
- applyCaseSensitiveFieldName(name),
- MySqlTypeUtils.toDataType(type, typeMapping)));
- return paimonFieldTypes;
+ protected LinkedHashMap<String, DataType> extractPaimonFieldTypes() {
+ LinkedHashMap<String, String> originalFieldTypes = tryExtractOriginalFieldTypes();
+ if (originalFieldTypes != null) {
+ return toPaimonFieldTypes(originalFieldTypes);
+ }
+
+ // fall back to STRING type
+ ArrayNode records = getNodeAs(root, dataField(), ArrayNode.class);
+ checkNotNull(records, dataField());
+ JsonNode record = records.get(0);
+
+ return fillDefaultStringTypes(record);
}
- @Override
- protected void validateFormat() {
- String errorMessageTemplate =
- "Didn't find '%s' node in json. Only supports canal-json format,"
- + "please make sure your topic's format is correct.";
-
- checkArgument(!isNull(root.get(FIELD_DATABASE)), errorMessageTemplate, FIELD_DATABASE);
- checkArgument(!isNull(root.get(FIELD_TABLE)), errorMessageTemplate, FIELD_TABLE);
- checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
- checkArgument(!isNull(root.get(FIELD_IS_DDL)), errorMessageTemplate, FIELD_IS_DDL);
-
- if (!extractBooleanFromRootJson(FIELD_IS_DDL)) {
- checkArgument(
- !isNull(root.get(FIELD_MYSQL_TYPE)), errorMessageTemplate, FIELD_MYSQL_TYPE);
- checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
- checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
+ @Nullable
+ private LinkedHashMap<String, String> tryExtractOriginalFieldTypes() {
+ JsonNode schema = root.get(FIELD_MYSQL_TYPE);
+ if (isNull(schema)) {
+ LOG.debug(
+ "Cannot get original field types because '{}' field is missing.",
+ FIELD_MYSQL_TYPE);
+ return null;
}
+
+ return OBJECT_MAPPER.convertValue(
+ schema, new TypeReference<LinkedHashMap<String, String>>() {});
+ }
+
+ private LinkedHashMap<String, DataType> toPaimonFieldTypes(
+ LinkedHashMap<String, String> originalFieldTypes) {
+ LinkedHashMap<String, DataType> paimonFieldTypes = new LinkedHashMap<>();
+ originalFieldTypes.forEach(
+ (name, type) ->
+ paimonFieldTypes.put(name, MySqlTypeUtils.toDataType(type, typeMapping)));
+ return paimonFieldTypes;
}
@Override
@@ -170,38 +176,34 @@ public class CanalRecordParser extends RecordParser {
@Override
protected Map<String, String> extractRowData(
JsonNode record, LinkedHashMap<String, DataType> paimonFieldTypes) {
- fieldTypes.forEach(
- (name, type) ->
- paimonFieldTypes.put(
- applyCaseSensitiveFieldName(name),
- MySqlTypeUtils.toDataType(type, typeMapping)));
- Map<String, Object> jsonMap =
+ LinkedHashMap<String, String> originalFieldTypes = tryExtractOriginalFieldTypes();
+ Map<String, Object> recordMap =
OBJECT_MAPPER.convertValue(record, new TypeReference<Map<String, Object>>() {});
- if (jsonMap == null) {
- return new HashMap<>();
+ Map<String, String> rowData = new HashMap<>();
+
+ if (originalFieldTypes != null) {
+ paimonFieldTypes.putAll(toPaimonFieldTypes(originalFieldTypes));
+ for (Map.Entry<String, Object> entry : recordMap.entrySet()) {
+ String fieldName = entry.getKey();
+ String originalType = originalFieldTypes.get(fieldName);
+ String newValue =
+ transformValue(Objects.toString(entry.getValue(), null), originalType);
+ rowData.put(fieldName, newValue);
+ }
+ } else {
+ paimonFieldTypes.putAll(fillDefaultStringTypes(record));
+ for (Map.Entry<String, Object> entry : recordMap.entrySet()) {
+ rowData.put(entry.getKey(), Objects.toString(entry.getValue(), null));
+ }
}
- Map<String, String> resultMap =
- fieldTypes.entrySet().stream()
- .filter(entry -> jsonMap.get(entry.getKey()) != null)
- .collect(
- Collectors.toMap(
- Map.Entry::getKey,
- entry ->
- transformValue(
- jsonMap.get(entry.getKey()).toString(),
- entry.getValue())));
-
- // generate values for computed columns
- for (ComputedColumn computedColumn : computedColumns) {
- resultMap.put(
- computedColumn.columnName(),
- computedColumn.eval(resultMap.get(computedColumn.fieldReference())));
- paimonFieldTypes.put(
- applyCaseSensitiveFieldName(computedColumn.columnName()),
- computedColumn.columnType());
- }
- return resultMap;
+ evalComputedColumns(rowData, paimonFieldTypes);
+ return rowData;
+ }
+
+ @Override
+ protected String format() {
+ return "canal-json";
}
private Map<JsonNode, JsonNode> matchOldRecords(ArrayNode newData, ArrayNode oldData) {
@@ -210,7 +212,11 @@ public class CanalRecordParser extends RecordParser {
.collect(Collectors.toMap(newData::get, oldData::get));
}
- private String transformValue(String oldValue, String mySqlType) {
+ private String transformValue(@Nullable String oldValue, String mySqlType) {
+ if (oldValue == null) {
+ return null;
+ }
+
String shortType = MySqlTypeUtils.getShortType(mySqlType);
if (MySqlTypeUtils.isSetType(shortType)) {
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java
index b1d858bf4..8bdce72b1 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/debezium/DebeziumRecordParser.java
@@ -24,10 +24,13 @@ import org.apache.paimon.flink.action.cdc.format.RecordParser;
import org.apache.paimon.flink.sink.cdc.RichCdcMultiplexRecord;
import org.apache.paimon.types.RowKind;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+
+import javax.annotation.Nullable;
+
import java.util.ArrayList;
import java.util.List;
-import static org.apache.flink.shaded.guava30.com.google.common.base.Preconditions.checkArgument;
import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
/**
@@ -67,22 +70,20 @@ public class DebeziumRecordParser extends RecordParser {
@Override
public List<RichCdcMultiplexRecord> extractRecords() {
- String operation = extractStringFromRootJson(FIELD_TYPE);
+ String operation = getAndCheck(FIELD_TYPE).asText();
List<RichCdcMultiplexRecord> records = new ArrayList<>();
switch (operation) {
case OP_INSERT:
case OP_READE:
- processRecord(root.get(dataField()), RowKind.INSERT, records);
+ processRecord(getData(), RowKind.INSERT, records);
break;
case OP_UPDATE:
processRecord(
- mergeOldRecord(root.get(dataField()), root.get(FIELD_BEFORE)),
- RowKind.DELETE,
- records);
- processRecord(root.get(dataField()), RowKind.INSERT, records);
+ mergeOldRecord(getData(), getBefore(operation)), RowKind.DELETE, records);
+ processRecord(getData(), RowKind.INSERT, records);
break;
case OP_DELETE:
- processRecord(root.get(FIELD_BEFORE), RowKind.DELETE, records);
+ processRecord(getBefore(operation), RowKind.DELETE, records);
break;
default:
throw new UnsupportedOperationException("Unknown record operation: " + operation);
@@ -90,33 +91,12 @@ public class DebeziumRecordParser extends RecordParser {
return records;
}
- @Override
- protected void validateFormat() {
- String errorMessageTemplate =
- "Didn't find '%s' node in json. Please make sure your topic's format is correct.";
- checkArgument(
- !isNull(root.get(FIELD_SOURCE).get(FIELD_TABLE)),
- errorMessageTemplate,
- FIELD_TABLE);
- checkArgument(
- !isNull(root.get(FIELD_SOURCE).get(FIELD_DB)),
- errorMessageTemplate,
- FIELD_DATABASE);
- checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
- String operation = root.get(FIELD_TYPE).asText();
- switch (operation) {
- case OP_INSERT:
- case OP_READE:
- checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
- break;
- case OP_UPDATE:
- case OP_DELETE:
- checkArgument(!isNull(root.get(FIELD_BEFORE)), errorMessageTemplate, FIELD_BEFORE);
- break;
- default:
- throw new IllegalArgumentException("Unsupported operation type: " + operation);
- }
- checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
+ private JsonNode getData() {
+ return getAndCheck(dataField());
+ }
+
+ private JsonNode getBefore(String op) {
+ return getAndCheck(FIELD_BEFORE, FIELD_TYPE, op);
}
@Override
@@ -129,15 +109,31 @@ public class DebeziumRecordParser extends RecordParser {
return FIELD_AFTER;
}
+ @Nullable
+ @Override
+ protected String getTableName() {
+ return getFromSourceField(FIELD_TABLE);
+ }
+
+ @Nullable
@Override
- protected String extractStringFromRootJson(String key) {
- if (key.equals(FIELD_TABLE)) {
- tableName = root.get(FIELD_SOURCE).get(FIELD_TABLE).asText();
- return tableName;
- } else if (key.equals(FIELD_DATABASE)) {
- databaseName = root.get(FIELD_SOURCE).get(FIELD_DB).asText();
- return databaseName;
+ protected String getDatabaseName() {
+ return getFromSourceField(FIELD_DB);
+ }
+
+ @Override
+ protected String format() {
+ return "debezium-json";
+ }
+
+ @Nullable
+ private String getFromSourceField(String key) {
+ JsonNode node = root.get(FIELD_SOURCE);
+ if (isNull(node)) {
+ return null;
}
- return root.get(key) != null ? root.get(key).asText() : null;
+
+ node = node.get(key);
+ return isNull(node) ? null : node.asText();
}
}
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java
index fa75b2b86..7379f81a5 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/maxwell/MaxwellRecordParser.java
@@ -29,9 +29,6 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-
/**
* The {@code MaxwellRecordParser} class extends the abstract {@link RecordParser} and is designed
* to parse records from Maxwell's JSON change data capture (CDC) format. Maxwell is a CDC solution
@@ -60,15 +57,16 @@ public class MaxwellRecordParser extends RecordParser {
@Override
public List<RichCdcMultiplexRecord> extractRecords() {
- String operation = extractStringFromRootJson(FIELD_TYPE);
- JsonNode data = root.get(dataField());
+ String operation = getAndCheck(FIELD_TYPE).asText();
+ JsonNode data = getAndCheck(dataField());
List<RichCdcMultiplexRecord> records = new ArrayList<>();
switch (operation) {
case OP_INSERT:
processRecord(data, RowKind.INSERT, records);
break;
case OP_UPDATE:
- processRecord(mergeOldRecord(data, root.get(FIELD_OLD)), RowKind.DELETE, records);
+ JsonNode old = getAndCheck(FIELD_OLD, FIELD_TYPE, operation);
+ processRecord(mergeOldRecord(data, old), RowKind.DELETE, records);
processRecord(data, RowKind.INSERT, records);
break;
case OP_DELETE:
@@ -80,17 +78,6 @@ public class MaxwellRecordParser extends RecordParser {
return records;
}
- @Override
- protected void validateFormat() {
- String errorMessageTemplate =
- "Didn't find '%s' node in json. Please make sure your topic's format is correct.";
- checkArgument(!isNull(root.get(FIELD_TABLE)), errorMessageTemplate, FIELD_TABLE);
- checkArgument(!isNull(root.get(FIELD_DATABASE)), errorMessageTemplate, FIELD_DATABASE);
- checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
- checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
- checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
- }
-
@Override
protected String primaryField() {
return "primary_key_columns";
@@ -100,4 +87,9 @@ public class MaxwellRecordParser extends RecordParser {
protected String dataField() {
return "data";
}
+
+ @Override
+ protected String format() {
+ return "maxwell-json";
+ }
}
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java
index 1eaf1d9f4..019ae1562 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/format/ogg/OggRecordParser.java
@@ -18,6 +18,7 @@
package org.apache.paimon.flink.action.cdc.format.ogg;
+import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.flink.action.cdc.ComputedColumn;
import org.apache.paimon.flink.action.cdc.TypeMapping;
import org.apache.paimon.flink.action.cdc.format.RecordParser;
@@ -26,11 +27,12 @@ import org.apache.paimon.types.RowKind;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import javax.annotation.Nullable;
+
import java.util.ArrayList;
import java.util.List;
import static org.apache.paimon.utils.JsonSerdeUtil.isNull;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
/**
* The {@code OggRecordParser} class extends the abstract {@link RecordParser} and is responsible
@@ -64,17 +66,17 @@ public class OggRecordParser extends RecordParser {
@Override
public List<RichCdcMultiplexRecord> extractRecords() {
List<RichCdcMultiplexRecord> records = new ArrayList<>();
- String operation = extractStringFromRootJson(FIELD_TYPE);
+ String operation = getAndCheck(FIELD_TYPE).asText();
switch (operation) {
case OP_UPDATE:
- processRecord(root.get(FIELD_BEFORE), RowKind.DELETE, records);
- processRecord(root.get(dataField()), RowKind.INSERT, records);
+ processRecord(getBefore(operation), RowKind.DELETE, records);
+ processRecord(getData(), RowKind.INSERT, records);
break;
case OP_INSERT:
- processRecord(root.get(dataField()), RowKind.INSERT, records);
+ processRecord(getData(), RowKind.INSERT, records);
break;
case OP_DELETE:
- processRecord(root.get(FIELD_BEFORE), RowKind.DELETE, records);
+ processRecord(getBefore(operation), RowKind.DELETE, records);
break;
default:
throw new UnsupportedOperationException("Unknown record operation: " + operation);
@@ -82,41 +84,12 @@ public class OggRecordParser extends RecordParser {
return records;
}
- @Override
- protected void validateFormat() {
- String errorMessageTemplate =
- "Didn't find '%s' node in json. Please make sure your topic's format is correct.";
-
- checkArgument(!isNull(root.get(FIELD_TABLE)), errorMessageTemplate, FIELD_TABLE);
- checkArgument(!isNull(root.get(FIELD_TYPE)), errorMessageTemplate, FIELD_TYPE);
- checkArgument(!isNull(root.get(primaryField())), errorMessageTemplate, primaryField());
-
- String fieldType = root.get(FIELD_TYPE).asText();
-
- switch (fieldType) {
- case OP_UPDATE:
- checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
- checkArgument(!isNull(root.get(FIELD_BEFORE)), errorMessageTemplate, FIELD_BEFORE);
- break;
- case OP_INSERT:
- checkArgument(!isNull(root.get(dataField())), errorMessageTemplate, dataField());
- break;
- case OP_DELETE:
- checkArgument(!isNull(root.get(FIELD_BEFORE)), errorMessageTemplate, FIELD_BEFORE);
- break;
- }
+ private JsonNode getData() {
+ return getAndCheck(dataField());
}
- @Override
- protected String extractStringFromRootJson(String key) {
- if (key.equals(FIELD_TABLE)) {
- extractDatabaseAndTableNames();
- return tableName;
- } else if (key.equals(FIELD_DATABASE)) {
- extractDatabaseAndTableNames();
- return databaseName;
- }
- return root.get(key) != null ? root.get(key).asText() : null;
+ private JsonNode getBefore(String op) {
+ return getAndCheck(FIELD_BEFORE, FIELD_TYPE, op);
}
@Override
@@ -129,14 +102,32 @@ public class OggRecordParser extends RecordParser {
return "after";
}
- private void extractDatabaseAndTableNames() {
- JsonNode tableNode = root.get(FIELD_TABLE);
- if (tableNode != null) {
- String[] dbt = tableNode.asText().split("\\.", 2);
- if (dbt.length == 2) {
- databaseName = dbt[0];
- tableName = dbt[1];
- }
+ @Nullable
+ @Override
+ protected String getTableName() {
+ Identifier id = getTableId();
+ return id == null ? null : id.getObjectName();
+ }
+
+ @Nullable
+ @Override
+ protected String getDatabaseName() {
+ Identifier id = getTableId();
+ return id == null ? null : id.getDatabaseName();
+ }
+
+ @Nullable
+ private Identifier getTableId() {
+ JsonNode node = root.get(FIELD_TABLE);
+ if (isNull(node)) {
+ return null;
}
+
+ return Identifier.fromString(node.asText());
+ }
+
+ @Override
+ protected String format() {
+ return "ogg-json";
}
}
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java
index 91e3892c6..9802b14be 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mongodb/MongoDBSyncTableAction.java
@@ -22,6 +22,7 @@ import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.flink.FlinkConnectorOptions;
import org.apache.paimon.flink.action.ActionBase;
+import org.apache.paimon.flink.action.cdc.CdcMetadataConverter;
import org.apache.paimon.flink.action.cdc.ComputedColumn;
import org.apache.paimon.flink.sink.cdc.CdcSinkBuilder;
import org.apache.paimon.flink.sink.cdc.EventParser;
@@ -140,7 +141,9 @@ public class MongoDBSyncTableAction extends ActionBase {
Collections.emptyList(),
computedColumns,
tableConfig,
- mongodbSchema);
+ mongodbSchema,
+ new CdcMetadataConverter[] {},
+ true);
// Check if table exists before trying to get or create it
if (catalog.tableExists(identifier)) {
fileStoreTable = (FileStoreTable) catalog.getTable(identifier);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
index 68c4fa440..931e38c5e 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
@@ -246,7 +246,8 @@ public class MySqlSyncDatabaseAction extends ActionBase {
Collections.emptyList(),
tableConfig,
tableInfo.schema(),
- metadataConverters);
+ metadataConverters,
+ true);
try {
table = (FileStoreTable) catalog.getTable(identifier);
table = table.copy(tableConfig);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
index bca841d2a..b2d38f508 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableAction.java
@@ -194,7 +194,8 @@ public class MySqlSyncTableAction extends ActionBase {
computedColumns,
tableConfig,
tableInfo.schema(),
- metadataConverters);
+ metadataConverters,
+ true);
try {
fileStoreTable = (FileStoreTable) catalog.getTable(identifier);
fileStoreTable = fileStoreTable.copy(tableConfig);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java
index 17ffa55e0..0fe0c9eab 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecord.java
@@ -20,6 +20,8 @@ package org.apache.paimon.flink.sink.cdc;
import org.apache.paimon.types.DataType;
+import javax.annotation.Nullable;
+
import java.io.Serializable;
import java.util.LinkedHashMap;
import java.util.List;
@@ -30,15 +32,15 @@ public class RichCdcMultiplexRecord implements Serializable {
private static final long serialVersionUID = 1L;
- private final String databaseName;
- private final String tableName;
+ @Nullable private final String databaseName;
+ @Nullable private final String tableName;
private final LinkedHashMap<String, DataType> fieldTypes;
private final List<String> primaryKeys;
private final CdcRecord cdcRecord;
public RichCdcMultiplexRecord(
- String databaseName,
- String tableName,
+ @Nullable String databaseName,
+ @Nullable String tableName,
LinkedHashMap<String, DataType> fieldTypes,
List<String> primaryKeys,
CdcRecord cdcRecord) {
@@ -49,10 +51,12 @@ public class RichCdcMultiplexRecord implements Serializable {
this.cdcRecord = cdcRecord;
}
+ @Nullable
public String databaseName() {
return databaseName;
}
+ @Nullable
public String tableName() {
return tableName;
}
@@ -83,8 +87,8 @@ public class RichCdcMultiplexRecord implements Serializable {
return false;
}
RichCdcMultiplexRecord that = (RichCdcMultiplexRecord) o;
- return databaseName.equals(that.databaseName)
- && tableName.equals(that.tableName)
+ return Objects.equals(databaseName, that.databaseName)
+ && Objects.equals(tableName, that.tableName)
&& Objects.equals(fieldTypes, that.fieldTypes)
&& Objects.equals(primaryKeys, that.primaryKeys)
&& Objects.equals(cdcRecord, that.cdcRecord);
diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java
index cdbcd2918..b0d5a77a0 100644
--- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java
+++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/RichCdcMultiplexRecordEventParser.java
@@ -85,6 +85,13 @@ public class RichCdcMultiplexRecordEventParser implements EventParser<RichCdcMul
@Override
public String parseTableName() {
+ // database synchronization needs this, so we validate the record here
+ if (record.databaseName() == null || record.tableName() == null) {
+ throw new IllegalArgumentException(
+ "Cannot synchronize record when database name or table name is unknown. "
+ + "Invalid record is:\n"
+ + record);
+ }
return tableNameConverter.convert(Identifier.create(record.databaseName(), currentTable));
}
@@ -112,6 +119,12 @@ public class RichCdcMultiplexRecordEventParser implements EventParser<RichCdcMul
}
private boolean shouldSynchronizeCurrentTable() {
+ // In case the record is incomplete, we let the null value pass validation
+ // and handle the null value when we really need it
+ if (currentTable == null) {
+ return true;
+ }
+
if (includedTables.contains(currentTable)) {
return true;
}
diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java
index 62253a6c8..db7ab79ea 100644
--- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java
+++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncDatabaseActionITCase.java
@@ -20,7 +20,6 @@ package org.apache.paimon.flink.action.cdc.kafka;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.testutils.assertj.AssertionUtils;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
@@ -38,6 +37,7 @@ import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import static org.apache.paimon.flink.action.cdc.TypeMapping.TypeMappingMode.TO_STRING;
+import static org.apache.paimon.testutils.assertj.AssertionUtils.anyCauseMatches;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -248,7 +248,7 @@ public class KafkaCanalSyncDatabaseActionITCase extends KafkaActionITCaseBase {
assertThatThrownBy(action::run)
.satisfies(
- AssertionUtils.anyCauseMatches(
+ anyCauseMatches(
IllegalArgumentException.class,
"kafka-conf [topic] must be specified."));
}
@@ -614,4 +614,32 @@ public class KafkaCanalSyncDatabaseActionITCase extends KafkaActionITCaseBase {
rowType,
Collections.singletonList("k1"));
}
+
+ @Test
+ @Timeout(60)
+ public void testCannotSynchronizeIncompleteJson() throws Exception {
+ final String topic = "incomplete";
+ createTestTopic(topic, 1, 1);
+
+ writeRecordsToKafka(topic, readLines("kafka/canal/database/incomplete/canal-data-1.txt"));
+
+ Map<String, String> kafkaConfig = getBasicKafkaConfig();
+ kafkaConfig.put("value.format", "canal-json");
+ kafkaConfig.put("topic", topic);
+
+ KafkaSyncDatabaseAction action =
+ syncDatabaseActionBuilder(kafkaConfig)
+ .withTableConfig(getBasicTableConfig())
+ .build();
+ action.withStreamExecutionEnvironment(env).build();
+
+ assertThatThrownBy(() -> env.execute())
+ .satisfies(
+ anyCauseMatches(
+ IllegalArgumentException.class,
+ "Cannot synchronize record when database name or table name is unknown. "
+ + "Invalid record is:\n"
+ + "{databaseName=null, tableName=null, fieldTypes={k=STRING, v0=STRING, v1=STRING}, "
+ + "primaryKeys=[], cdcRecord=+I {v0=five, k=5, v1=50}}"));
+ }
}
diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java
index 6cd3d04e3..4e5f31218 100644
--- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java
+++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/kafka/KafkaCanalSyncTableActionITCase.java
@@ -1072,4 +1072,65 @@ public class KafkaCanalSyncTableActionITCase extends KafkaActionITCaseBase {
rowType,
Arrays.asList("_id", "_year"));
}
+
+ @Test
+ @Timeout(60)
+ public void testSynchronizeIncompleteJson() throws Exception {
+ String topic = "incomplete";
+ createTestTopic(topic, 1, 1);
+ List<String> lines = readLines("kafka/canal/table/incomplete/canal-data-1.txt");
+ writeRecordsToKafka(topic, lines);
+
+ Map<String, String> kafkaConfig = getBasicKafkaConfig();
+ kafkaConfig.put("value.format", "canal-json");
+ kafkaConfig.put("topic", topic);
+ KafkaSyncTableAction action =
+ syncTableActionBuilder(kafkaConfig)
+ .withPrimaryKeys("k")
+ .withTableConfig(getBasicTableConfig())
+ .build();
+
+ runActionWithDefaultEnv(action);
+
+ RowType rowType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.STRING().notNull(), DataTypes.STRING(),
+ },
+ new String[] {"k", "v"});
+ waitForResult(
+ Arrays.asList("+I[1, Hi]", "+I[2, Hello]"),
+ getFileStoreTable(tableName),
+ rowType,
+ Collections.singletonList("k"));
+ }
+
+ @Test
+ @Timeout(60)
+ public void testSynchronizeNonPkTable() throws Exception {
+ String topic = "non_pk";
+ createTestTopic(topic, 1, 1);
+ List<String> lines = readLines("kafka/canal/table/nonpk/canal-data-1.txt");
+ writeRecordsToKafka(topic, lines);
+
+ Map<String, String> kafkaConfig = getBasicKafkaConfig();
+ kafkaConfig.put("value.format", "canal-json");
+ kafkaConfig.put("topic", topic);
+ KafkaSyncTableAction action =
+ syncTableActionBuilder(kafkaConfig).withTableConfig(getBasicTableConfig()).build();
+
+ runActionWithDefaultEnv(action);
+
+ RowType rowType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.VARCHAR(10), DataTypes.INT(),
+ },
+ new String[] {"v0", "v1"});
+ waitForResult(
+ Arrays.asList("+I[five, 50]", "+I[five, 50]"),
+ getFileStoreTable(tableName),
+ rowType,
+ Collections.emptyList());
+ }
}
diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/database/incomplete/canal-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/database/incomplete/canal-data-1.txt
new file mode 100644
index 000000000..061c242ea
--- /dev/null
+++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/database/incomplete/canal-data-1.txt
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"data":[{"k":"5","v0":"five","v1":"50"}],"type":"INSERT"}
+
diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/incomplete/canal-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/incomplete/canal-data-1.txt
new file mode 100644
index 000000000..b6ea31253
--- /dev/null
+++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/incomplete/canal-data-1.txt
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"data":[{"k":"1","v":"Hi"}],"type":"INSERT"}
+{"data":[{"k":"2","v":"Hello"}],"type":"INSERT"}
diff --git a/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/nonpk/canal-data-1.txt b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/nonpk/canal-data-1.txt
new file mode 100644
index 000000000..250b5057c
--- /dev/null
+++ b/paimon-flink/paimon-flink-cdc/src/test/resources/kafka/canal/table/nonpk/canal-data-1.txt
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"data":[{"v0":"five","v1":"50"}],"database":"non_pk","es":1684770072000,"id":81,"isDdl":false,"mysqlType":{"v0":"VARCHAR(10)","v1":"INT"},"old":null,"pkNames":[],"sql":"","sqlType":{"v0":12,"v1":4},"table":"t","ts":1684770072286,"type":"INSERT"}
+{"data":[{"v0":"five","v1":"50"}],"database":"non_pk","es":1684770073000,"id":84,"isDdl":false,"mysqlType":{"v0":"VARCHAR(10)","v1":"INT"},"old":null,"pkNames":[],"sql":"","sqlType":{"v0":12,"v1":4},"table":"t","ts":1684770073254,"type":"INSERT"}