You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/07/03 06:36:00 UTC
[incubator-paimon] branch master updated: [cdc] Support multiple tables sink operator in cdc ingestion (#1261)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 136ad28d7 [cdc] Support multiple tables sink operator in cdc ingestion (#1261)
136ad28d7 is described below
commit 136ad28d72a880eeba0bf4dfba6147b2250b3c0e
Author: Dian Qi <qi...@163.com>
AuthorDate: Mon Jul 3 14:35:55 2023 +0800
[cdc] Support multiple tables sink operator in cdc ingestion (#1261)
---
paimon-flink/paimon-flink-common/pom.xml | 1 +
.../flink/action/cdc/mysql/MySqlActionUtils.java | 6 +-
.../action/cdc/mysql/MySqlDatabaseSyncMode.java | 31 +++
.../cdc/mysql/MySqlDebeziumJsonEventParser.java | 99 +++++++-
.../paimon/flink/action/cdc/mysql/MySqlSchema.java | 15 ++
.../action/cdc/mysql/MySqlSyncDatabaseAction.java | 82 +++++--
.../cdc/CdcDynamicTableParsingProcessFunction.java | 119 +++++++++
.../cdc/CdcMultiplexRecordChannelComputer.java | 81 +++++++
.../apache/paimon/flink/sink/cdc/EventParser.java | 16 ++
.../flink/sink/cdc/FlinkCdcMultiTableSink.java | 153 ++++++++++++
.../sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java | 87 ++++++-
...MultiTableUpdatedDataFieldsProcessFunction.java | 268 +++++++++++++++++++++
.../src/main/resources/META-INF/NOTICE | 10 +
.../cdc/mysql/MySqlSyncDatabaseActionITCase.java | 105 ++++++--
.../cdc/CdcMultiplexRecordChannelComputerTest.java | 194 +++++++++++++++
.../paimon/flink/sink/cdc/TestCdcEventParser.java | 12 +
.../src/test/resources/mysql/setup.sql | 72 +++++-
17 files changed, 1299 insertions(+), 52 deletions(-)
diff --git a/paimon-flink/paimon-flink-common/pom.xml b/paimon-flink/paimon-flink-common/pom.xml
index 52923bbdb..a9b337460 100644
--- a/paimon-flink/paimon-flink-common/pom.xml
+++ b/paimon-flink/paimon-flink-common/pom.xml
@@ -291,6 +291,7 @@ under the License.
<artifactSet>
<includes combine.children="append">
<include>org.apache.paimon:paimon-bundle</include>
+ <include>com.alibaba:druid</include>
</includes>
</artifactSet>
<relocations>
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
index 46dc25347..15b157ab5 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlActionUtils.java
@@ -195,9 +195,11 @@ class MySqlActionUtils {
mySqlConfig
.getOptional(MySqlSourceOptions.SERVER_TIME_ZONE)
.ifPresent(sourceBuilder::serverTimeZone);
+ // MySQL CDC using increment snapshot, splitSize is used instead of fetchSize (as in JDBC
+ // connector). splitSize is the number of records in each snapshot split.
mySqlConfig
- .getOptional(MySqlSourceOptions.SCAN_SNAPSHOT_FETCH_SIZE)
- .ifPresent(sourceBuilder::fetchSize);
+ .getOptional(MySqlSourceOptions.SCAN_INCREMENTAL_SNAPSHOT_CHUNK_SIZE)
+ .ifPresent(sourceBuilder::splitSize);
mySqlConfig
.getOptional(MySqlSourceOptions.CONNECT_TIMEOUT)
.ifPresent(sourceBuilder::connectTimeout);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDatabaseSyncMode.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDatabaseSyncMode.java
new file mode 100644
index 000000000..127c17c02
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDatabaseSyncMode.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.action.cdc.mysql;
+
+import java.io.Serializable;
+
+/**
+ * There are two modes for database sync. 1) STATIC mode, only write record from static tables.
+ * Newly added tables during runtime are not synced. 2) DYNAMIC mode, all records from static tables
+ * and newly added tables are routed into unified operator.
+ */
+public enum MySqlDatabaseSyncMode implements Serializable {
+ STATIC,
+ DYNAMIC
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
index 54240098c..bba93eaa2 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlDebeziumJsonEventParser.java
@@ -27,16 +27,30 @@ import org.apache.paimon.flink.action.cdc.ComputedColumn;
import org.apache.paimon.flink.action.cdc.TableNameConverter;
import org.apache.paimon.flink.sink.cdc.CdcRecord;
import org.apache.paimon.flink.sink.cdc.EventParser;
+import org.apache.paimon.schema.Schema;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowKind;
import org.apache.paimon.utils.DateTimeUtils;
import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.shade.guava30.com.google.common.base.Strings;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.type.TypeReference;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import com.alibaba.druid.sql.SQLUtils;
+import com.alibaba.druid.sql.ast.SQLDataType;
+import com.alibaba.druid.sql.ast.SQLExpr;
+import com.alibaba.druid.sql.ast.SQLName;
+import com.alibaba.druid.sql.ast.SQLStatement;
+import com.alibaba.druid.sql.ast.expr.SQLCharExpr;
+import com.alibaba.druid.sql.ast.expr.SQLIntegerExpr;
+import com.alibaba.druid.sql.ast.statement.SQLColumnDefinition;
+import com.alibaba.druid.sql.ast.statement.SQLTableElement;
+import com.alibaba.druid.sql.dialect.mysql.ast.statement.MySqlCreateTableStatement;
+import com.alibaba.druid.util.JdbcConstants;
+import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.kafka.connect.json.JsonConverterConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,8 +64,10 @@ import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -106,6 +122,12 @@ public class MySqlDebeziumJsonEventParser implements EventParser<String> {
return tableNameConverter.convert(tableName);
}
+ @Override
+ public String parseDatabaseName() {
+ String databaseName = payload.get("source").get("db").asText();
+ return tableNameConverter.convert(databaseName);
+ }
+
private boolean isSchemaChange() {
return payload.get("op") == null;
}
@@ -161,12 +183,87 @@ public class MySqlDebeziumJsonEventParser implements EventParser<String> {
return result;
}
+ @Override
+ public Optional<Schema> parseNewTable(String databaseName) {
+ JsonNode historyRecord = payload.get("historyRecord");
+ if (historyRecord == null) {
+ return Optional.empty();
+ }
+
+ try {
+ String historyRecordString = historyRecord.asText();
+ String ddl = objectMapper.readTree(historyRecordString).get("ddl").asText();
+ if (Strings.isNullOrEmpty(ddl)) {
+ return Optional.empty();
+ }
+
+ SQLStatement statement = SQLUtils.parseSingleStatement(ddl, JdbcConstants.MYSQL);
+
+ if (!(statement instanceof MySqlCreateTableStatement)) {
+ return Optional.empty();
+ }
+
+ MySqlCreateTableStatement createTableStatement = (MySqlCreateTableStatement) statement;
+
+ // TODO: add default table config, partitions, and computed column
+ // for newly added table;
+ MySqlSchema mySqlSchema = buildMySqlSchema(databaseName, createTableStatement);
+ Schema fromMySql =
+ MySqlActionUtils.buildPaimonSchema(
+ mySqlSchema,
+ Collections.emptyList(),
+ mySqlSchema.getPrimaryKeys(),
+ computedColumns,
+ Collections.emptyMap(),
+ caseSensitive);
+
+ return Optional.of(fromMySql);
+
+ } catch (Exception e) {
+ LOG.info("Failed to parse history record for schema changes", e);
+ return Optional.empty();
+ }
+ }
+
+ private MySqlSchema buildMySqlSchema(String database, MySqlCreateTableStatement statement) {
+ LinkedHashMap<String, Tuple2<DataType, String>> fields = new LinkedHashMap<>();
+
+ List<SQLTableElement> columns = statement.getTableElementList();
+ for (SQLTableElement element : columns) {
+ if (element instanceof SQLColumnDefinition) {
+ SQLColumnDefinition column = (SQLColumnDefinition) element;
+ SQLName name = column.getName();
+ SQLDataType dataType = column.getDataType();
+ List<SQLExpr> arguments = dataType.getArguments();
+ Integer precision = null;
+ Integer scale = null;
+ if (arguments.size() >= 1) {
+ precision = (int) (((SQLIntegerExpr) arguments.get(0)).getValue());
+ }
+
+ if (arguments.size() >= 2) {
+ scale = (int) (((SQLIntegerExpr) arguments.get(1)).getValue());
+ }
+
+ SQLCharExpr comment = (SQLCharExpr) column.getComment();
+ fields.put(
+ name.getSimpleName(),
+ Tuple2.of(
+ MySqlTypeUtils.toDataType(
+ column.getDataType().getName(), precision, scale),
+ comment == null ? null : String.valueOf(comment.getValue())));
+ }
+ }
+
+ return new MySqlSchema(
+ database, statement.getTableName(), fields, statement.getPrimaryKeyNames());
+ }
+
@Override
public List<CdcRecord> parseRecords() {
if (isSchemaChange()) {
return Collections.emptyList();
}
-
List<CdcRecord> records = new ArrayList<>();
Map<String, String> before = extractRow(payload.get("before"));
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSchema.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSchema.java
index 901199c78..5501e25bc 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSchema.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSchema.java
@@ -40,6 +40,17 @@ public class MySqlSchema {
private final LinkedHashMap<String, Tuple2<DataType, String>> fields;
private final List<String> primaryKeys;
+ public MySqlSchema(
+ String databaseName,
+ String tableName,
+ LinkedHashMap<String, Tuple2<DataType, String>> fields,
+ List<String> primaryKeys) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ this.fields = fields;
+ this.primaryKeys = primaryKeys;
+ }
+
public MySqlSchema(DatabaseMetaData metaData, String databaseName, String tableName)
throws Exception {
this.databaseName = databaseName;
@@ -128,4 +139,8 @@ public class MySqlSchema {
}
return this;
}
+
+ public List<String> getPrimaryKeys() {
+ return primaryKeys;
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
index 449969678..bc2abf91e 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseAction.java
@@ -20,12 +20,15 @@ package org.apache.paimon.flink.action.cdc.mysql;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.flink.FlinkCatalogFactory;
import org.apache.paimon.flink.FlinkConnectorOptions;
import org.apache.paimon.flink.action.Action;
import org.apache.paimon.flink.action.ActionBase;
import org.apache.paimon.flink.action.cdc.TableNameConverter;
import org.apache.paimon.flink.sink.cdc.EventParser;
import org.apache.paimon.flink.sink.cdc.FlinkCdcSyncDatabaseSinkBuilder;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.FileStoreTable;
@@ -48,11 +51,13 @@ import java.sql.ResultSet;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import static org.apache.paimon.flink.action.Action.checkRequiredArgument;
import static org.apache.paimon.flink.action.Action.optionalConfigMap;
@@ -103,9 +108,12 @@ public class MySqlSyncDatabaseAction extends ActionBase {
private final boolean ignoreIncompatible;
private final String tablePrefix;
private final String tableSuffix;
- @Nullable private final Pattern includingPattern;
+ private final Pattern includingPattern;
@Nullable private final Pattern excludingPattern;
private final Map<String, String> tableConfig;
+ private final String includingTables;
+ private final Options catalogOptions;
+ private final MySqlDatabaseSyncMode mode;
public MySqlSyncDatabaseAction(
Map<String, String> mySqlConfig,
@@ -124,7 +132,8 @@ public class MySqlSyncDatabaseAction extends ActionBase {
null,
null,
catalogConfig,
- tableConfig);
+ tableConfig,
+ MySqlDatabaseSyncMode.STATIC);
}
public MySqlSyncDatabaseAction(
@@ -137,16 +146,21 @@ public class MySqlSyncDatabaseAction extends ActionBase {
@Nullable String includingTables,
@Nullable String excludingTables,
Map<String, String> catalogConfig,
- Map<String, String> tableConfig) {
+ Map<String, String> tableConfig,
+ MySqlDatabaseSyncMode mode) {
super(warehouse, catalogConfig);
this.mySqlConfig = Configuration.fromMap(mySqlConfig);
this.database = database;
this.ignoreIncompatible = ignoreIncompatible;
this.tablePrefix = tablePrefix == null ? "" : tablePrefix;
this.tableSuffix = tableSuffix == null ? "" : tableSuffix;
- this.includingPattern = includingTables == null ? null : Pattern.compile(includingTables);
+ this.includingTables = includingTables == null ? ".*" : includingTables;
+ this.includingPattern = Pattern.compile(this.includingTables);
this.excludingPattern = excludingTables == null ? null : Pattern.compile(excludingTables);
+ this.catalogOptions = Options.fromMap(catalogConfig);
+ catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse);
this.tableConfig = tableConfig;
+ this.mode = mode;
}
public void build(StreamExecutionEnvironment env) throws Exception {
@@ -162,11 +176,13 @@ public class MySqlSyncDatabaseAction extends ActionBase {
validateCaseInsensitive();
}
- List<MySqlSchema> mySqlSchemas = getMySqlSchemaList();
+ List<String> excludedTables = new LinkedList<>();
+ List<MySqlSchema> mySqlSchemas = getMySqlSchemaList(excludedTables);
+ String mySqlDatabase = mySqlConfig.get(MySqlSourceOptions.DATABASE_NAME);
checkArgument(
mySqlSchemas.size() > 0,
"No tables found in MySQL database "
- + mySqlConfig.get(MySqlSourceOptions.DATABASE_NAME)
+ + mySqlDatabase
+ ", or MySQL database does not exist.");
catalog.createDatabase(database, true);
@@ -207,9 +223,21 @@ public class MySqlSyncDatabaseAction extends ActionBase {
!monitoredTables.isEmpty(),
"No tables to be synchronized. Possible cause is the schemas of all tables in specified "
+ "MySQL database are not compatible with those of existed Paimon tables. Please check the log.");
-
- mySqlConfig.set(
- MySqlSourceOptions.TABLE_NAME, "(" + String.join("|", monitoredTables) + ")");
+ String tableList;
+
+ if (mode == MySqlDatabaseSyncMode.DYNAMIC) {
+ // First excluding all tables that failed the excludingPattern and those does not
+ // have a primary key. Then including other table using regex so that newly
+ // added table DDLs and DMLs during job runtime will be captured
+ tableList =
+ excludedTables.stream()
+ .map(t -> String.format("(?!(%s))", t))
+ .collect(Collectors.joining(""))
+ + includingTables;
+ } else {
+ tableList = "(" + String.join("|", monitoredTables) + ")";
+ }
+ mySqlConfig.set(MySqlSourceOptions.TABLE_NAME, tableList);
MySqlSource<String> source = MySqlActionUtils.buildMySqlSource(mySqlConfig);
String serverTimeZone = mySqlConfig.get(MySqlSourceOptions.SERVER_TIME_ZONE);
@@ -217,17 +245,29 @@ public class MySqlSyncDatabaseAction extends ActionBase {
EventParser.Factory<String> parserFactory =
() -> new MySqlDebeziumJsonEventParser(zoneId, caseSensitive, tableNameConverter);
+ String database = this.database;
+ MySqlDatabaseSyncMode mode = this.mode;
+ // To make the sync database workflow serializable
+ Options catalogOptions = this.catalogOptions;
FlinkCdcSyncDatabaseSinkBuilder<String> sinkBuilder =
new FlinkCdcSyncDatabaseSinkBuilder<String>()
.withInput(
env.fromSource(
source, WatermarkStrategy.noWatermarks(), "MySQL Source"))
.withParserFactory(parserFactory)
- .withTables(fileStoreTables);
+ .withDatabase(database)
+ .withCatalogLoader(
+ () -> {
+ return FlinkCatalogFactory.createPaimonCatalog(catalogOptions);
+ })
+ .withTables(fileStoreTables)
+ .withMode(mode);
+
String sinkParallelism = tableConfig.get(FlinkConnectorOptions.SINK_PARALLELISM.key());
if (sinkParallelism != null) {
sinkBuilder.withParallelism(Integer.parseInt(sinkParallelism));
}
+
sinkBuilder.build();
}
@@ -249,7 +289,7 @@ public class MySqlSyncDatabaseAction extends ActionBase {
tableSuffix));
}
- private List<MySqlSchema> getMySqlSchemaList() throws Exception {
+ private List<MySqlSchema> getMySqlSchemaList(List<String> excludedTables) throws Exception {
String databaseName = mySqlConfig.get(MySqlSourceOptions.DATABASE_NAME);
List<MySqlSchema> mySqlSchemaList = new ArrayList<>();
try (Connection conn = MySqlActionUtils.getConnection(mySqlConfig)) {
@@ -259,12 +299,15 @@ public class MySqlSyncDatabaseAction extends ActionBase {
while (tables.next()) {
String tableName = tables.getString("TABLE_NAME");
if (!shouldMonitorTable(tableName)) {
+ excludedTables.add(tableName);
continue;
}
MySqlSchema mySqlSchema = new MySqlSchema(metaData, databaseName, tableName);
if (mySqlSchema.primaryKeys().size() > 0) {
// only tables with primary keys will be considered
mySqlSchemaList.add(mySqlSchema);
+ } else {
+ excludedTables.add(tableName);
}
}
}
@@ -273,10 +316,7 @@ public class MySqlSyncDatabaseAction extends ActionBase {
}
private boolean shouldMonitorTable(String mySqlTableName) {
- boolean shouldMonitor = true;
- if (includingPattern != null) {
- shouldMonitor = includingPattern.matcher(mySqlTableName).matches();
- }
+ boolean shouldMonitor = includingPattern.matcher(mySqlTableName).matches();
if (excludingPattern != null) {
shouldMonitor = shouldMonitor && !excludingPattern.matcher(mySqlTableName).matches();
}
@@ -335,6 +375,13 @@ public class MySqlSyncDatabaseAction extends ActionBase {
String tableSuffix = params.get("table-suffix");
String includingTables = params.get("including-tables");
String excludingTables = params.get("excluding-tables");
+ String mode = params.get("mode");
+ MySqlDatabaseSyncMode syncMode;
+ if ("dynamic".equalsIgnoreCase(mode)) {
+ syncMode = MySqlDatabaseSyncMode.DYNAMIC;
+ } else {
+ syncMode = MySqlDatabaseSyncMode.STATIC;
+ }
Map<String, String> mySqlConfig = optionalConfigMap(params, "mysql-conf");
Map<String, String> catalogConfig = optionalConfigMap(params, "catalog-conf");
@@ -350,7 +397,8 @@ public class MySqlSyncDatabaseAction extends ActionBase {
includingTables,
excludingTables,
catalogConfig,
- tableConfig));
+ tableConfig,
+ syncMode));
}
private static void printHelp() {
@@ -403,7 +451,7 @@ public class MySqlSyncDatabaseAction extends ActionBase {
"'hostname', 'username', 'password' and 'database-name' "
+ "are required configurations, others are optional. "
+ "Note that 'database-name' should be the exact name "
- + "of the MySQL databse you want to synchronize. "
+ + "of the MySQL database you want to synchronize. "
+ "It can't be a regular expression.");
System.out.println(
"For a complete list of supported configurations, "
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java
new file mode 100644
index 000000000..556be5762
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcDynamicTableParsingProcessFunction.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink.cdc;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.types.DataField;
+
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
+
+import java.util.List;
+
+/**
+ * A {@link ProcessFunction} to parse CDC change event to either a list of {@link DataField}s or
+ * {@link CdcRecord} and send them to different side outputs according to table name. This process
+ * function will capture newly added tables when syncing entire database and in cases where the
+ * newly added tables are including by attesting table filters.
+ *
+ * <p>This {@link ProcessFunction} can handle records for different tables at the same time.
+ *
+ * @param <T> CDC change event type
+ */
+public class CdcDynamicTableParsingProcessFunction<T> extends ProcessFunction<T, Void> {
+
+ public static final OutputTag<CdcMultiplexRecord> DYNAMIC_OUTPUT_TAG =
+ new OutputTag<>("paimon-dynamic-table", TypeInformation.of(CdcMultiplexRecord.class));
+
+ public static final OutputTag<Tuple2<Identifier, List<DataField>>>
+ DYNAMIC_SCHEMA_CHANGE_OUTPUT_TAG =
+ new OutputTag<>(
+ "paimon-dynamic-table-schema-change",
+ TypeInformation.of(
+ new TypeHint<Tuple2<Identifier, List<DataField>>>() {}));
+
+ private final EventParser.Factory<T> parserFactory;
+ private final String database;
+ private final Catalog.Loader catalogLoader;
+
+ private transient EventParser<T> parser;
+ private transient Catalog catalog;
+
+ public CdcDynamicTableParsingProcessFunction(
+ String database, Catalog.Loader catalogLoader, EventParser.Factory<T> parserFactory) {
+ // for now, only support single database
+ this.database = database;
+ this.catalogLoader = catalogLoader;
+ this.parserFactory = parserFactory;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ parser = parserFactory.create();
+ catalog = catalogLoader.load();
+ }
+
+ @Override
+ public void processElement(T raw, Context context, Collector<Void> collector) throws Exception {
+ parser.setRawEvent(raw);
+
+ // CDC Ingestion only supports single database at this time being.
+ // In the future, there will be a mapping between source databases
+ // and target paimon databases
+ // TODO: support multiple databases
+ // String databaseName = parser.parseDatabaseName();
+ String tableName = parser.parseTableName();
+
+ // check for newly added table
+ parser.parseNewTable(database)
+ .ifPresent(
+ schema -> {
+ Identifier identifier =
+ new Identifier(database, parser.parseTableName());
+ try {
+ catalog.createTable(identifier, schema, true);
+ } catch (Throwable ignored) {
+ }
+ });
+
+ List<DataField> schemaChange = parser.parseSchemaChange();
+ if (schemaChange.size() > 0) {
+ context.output(
+ DYNAMIC_SCHEMA_CHANGE_OUTPUT_TAG,
+ Tuple2.of(Identifier.create(database, tableName), schemaChange));
+ }
+
+ parser.parseRecords()
+ .forEach(
+ record ->
+ context.output(
+ DYNAMIC_OUTPUT_TAG,
+ wrapRecord(database, tableName, record)));
+ }
+
+ private CdcMultiplexRecord wrapRecord(String databaseName, String tableName, CdcRecord record) {
+ return CdcMultiplexRecord.fromCdcRecord(databaseName, tableName, record);
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java
new file mode 100644
index 000000000..9006b85cd
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputer.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink.cdc;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.flink.sink.ChannelComputer;
+import org.apache.paimon.table.FileStoreTable;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** {@link ChannelComputer} for {@link CdcRecord}. */
+public class CdcMultiplexRecordChannelComputer implements ChannelComputer<CdcMultiplexRecord> {
+
+ private static final long serialVersionUID = 1L;
+ private final Catalog.Loader catalogLoader;
+
+ private transient int numChannels;
+
+ private Map<Identifier, CdcRecordChannelComputer> channelComputers;
+ private Catalog catalog;
+
+ public CdcMultiplexRecordChannelComputer(Catalog.Loader catalogLoader) {
+ this.catalogLoader = catalogLoader;
+ }
+
+ @Override
+ public void setup(int numChannels) {
+ this.numChannels = numChannels;
+ this.catalog = catalogLoader.load();
+ this.channelComputers = new HashMap<>();
+ }
+
+ @Override
+ public int channel(CdcMultiplexRecord multiplexRecord) {
+ return Math.floorMod(
+ Objects.hash(multiplexRecord.databaseName(), multiplexRecord.tableName())
+ + computeChannelComputer(multiplexRecord).channel(multiplexRecord.record()),
+ numChannels);
+ }
+
+ private ChannelComputer<CdcRecord> computeChannelComputer(CdcMultiplexRecord record) {
+ return channelComputers.computeIfAbsent(
+ Identifier.create(record.databaseName(), record.tableName()),
+ id -> {
+ FileStoreTable table;
+ try {
+ table = (FileStoreTable) catalog.getTable(id);
+ } catch (Catalog.TableNotExistException e) {
+ throw new RuntimeException("Failed to get table " + id.getFullName());
+ }
+ CdcRecordChannelComputer channelComputer =
+ new CdcRecordChannelComputer(table.schema());
+ channelComputer.setup(numChannels);
+ return channelComputer;
+ });
+ }
+
+ @Override
+ public String toString() {
+ return "shuffle by table";
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/EventParser.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/EventParser.java
index c8518ccc9..136ff32b7 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/EventParser.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/EventParser.java
@@ -19,10 +19,12 @@
package org.apache.paimon.flink.sink.cdc;
import org.apache.paimon.annotation.Experimental;
+import org.apache.paimon.schema.Schema;
import org.apache.paimon.types.DataField;
import java.io.Serializable;
import java.util.List;
+import java.util.Optional;
/**
* Parse a CDC change event to a list of {@link DataField}s or {@link CdcRecord}.
@@ -40,6 +42,10 @@ public interface EventParser<T> {
throw new UnsupportedOperationException("Table name is not supported in this parser.");
}
+ default String parseDatabaseName() {
+ throw new UnsupportedOperationException("Database name is not supported in this parser.");
+ }
+
/**
* Parse new schema if this event contains schema change.
*
@@ -54,6 +60,16 @@ public interface EventParser<T> {
*/
List<CdcRecord> parseRecords();
+ /**
+ * Parse newly added table schema from event.
+ *
+ * @param databaseName database of the new table
+ * @return empty if there is no newly added table
+ */
+ default Optional<Schema> parseNewTable(String databaseName) {
+ return Optional.empty();
+ }
+
/** Factory to create an {@link EventParser}. */
interface Factory<T> extends Serializable {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java
new file mode 100644
index 000000000..1291ce15b
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcMultiTableSink.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink.cdc;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.flink.VersionedSerializerWrapper;
+import org.apache.paimon.flink.sink.CommittableStateManager;
+import org.apache.paimon.flink.sink.Committer;
+import org.apache.paimon.flink.sink.CommitterOperator;
+import org.apache.paimon.flink.sink.FlinkSink;
+import org.apache.paimon.flink.sink.MultiTableCommittable;
+import org.apache.paimon.flink.sink.MultiTableCommittableTypeInfo;
+import org.apache.paimon.flink.sink.RestoreAndFailCommittableStateManager;
+import org.apache.paimon.flink.sink.StoreMultiCommitter;
+import org.apache.paimon.flink.sink.StoreSinkWrite;
+import org.apache.paimon.flink.sink.StoreSinkWriteImpl;
+import org.apache.paimon.flink.sink.WrappedManifestCommittableSerializer;
+import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils;
+import org.apache.paimon.manifest.WrappedManifestCommittable;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.utils.Preconditions;
+import org.apache.paimon.utils.SerializableFunction;
+
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.CheckpointConfig;
+import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.DiscardingSink;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+/**
+ * A {@link FlinkSink} which accepts {@link CdcRecord} and waits for a schema change if necessary.
+ */
+public class FlinkCdcMultiTableSink implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private static final String WRITER_NAME = "CDC MultiplexWriter";
+ private static final String GLOBAL_COMMITTER_NAME = "Multiplex Global Committer";
+
+ private final boolean isOverwrite = false;
+ private final Catalog.Loader catalogLoader;
+
+ public FlinkCdcMultiTableSink(Catalog.Loader catalogLoader) {
+ this.catalogLoader = catalogLoader;
+ }
+
+ private StoreSinkWrite.Provider createWriteProvider() {
+ // for now, no compaction for multiplexed sink
+ return (table, commitUser, state, ioManager, memoryPool) ->
+ new StoreSinkWriteImpl(
+ table, commitUser, state, ioManager, isOverwrite, false, true, memoryPool);
+ }
+
+ public DataStreamSink<?> sinkFrom(DataStream<CdcMultiplexRecord> input) {
+ // This commitUser is valid only for new jobs.
+ // After the job starts, this commitUser will be recorded into the states of write and
+ // commit operators.
+ // When the job restarts, commitUser will be recovered from states and this value is
+ // ignored.
+ String initialCommitUser = UUID.randomUUID().toString();
+ return sinkFrom(input, initialCommitUser, createWriteProvider());
+ }
+
+ public DataStreamSink<?> sinkFrom(
+ DataStream<CdcMultiplexRecord> input,
+ String commitUser,
+ StoreSinkWrite.Provider sinkProvider) {
+ StreamExecutionEnvironment env = input.getExecutionEnvironment();
+ ReadableConfig conf = StreamExecutionEnvironmentUtils.getConfiguration(env);
+ CheckpointConfig checkpointConfig = env.getCheckpointConfig();
+
+ assertCheckpointConfiguration(env);
+
+ MultiTableCommittableTypeInfo typeInfo = new MultiTableCommittableTypeInfo();
+ SingleOutputStreamOperator<MultiTableCommittable> written =
+ input.transform(
+ WRITER_NAME,
+ typeInfo,
+ createWriteOperator(sinkProvider, commitUser))
+ .setParallelism(input.getParallelism());
+
+ SingleOutputStreamOperator<?> committed =
+ written.transform(
+ GLOBAL_COMMITTER_NAME,
+ typeInfo,
+ new CommitterOperator<>(
+ true,
+ commitUser,
+ createCommitterFactory(),
+ createCommittableStateManager()))
+ .setParallelism(1)
+ .setMaxParallelism(1);
+ return committed.addSink(new DiscardingSink<>()).name("end").setParallelism(1);
+ }
+
+ private void assertCheckpointConfiguration(StreamExecutionEnvironment env) {
+ Preconditions.checkArgument(
+ !env.getCheckpointConfig().isUnalignedCheckpointsEnabled(),
+ "Paimon sink currently does not support unaligned checkpoints. Please set "
+ + ExecutionCheckpointingOptions.ENABLE_UNALIGNED.key()
+ + " to false.");
+ Preconditions.checkArgument(
+ env.getCheckpointConfig().getCheckpointingMode() == CheckpointingMode.EXACTLY_ONCE,
+ "Paimon sink currently only supports EXACTLY_ONCE checkpoint mode. Please set "
+ + ExecutionCheckpointingOptions.CHECKPOINTING_MODE.key()
+ + " to exactly-once");
+ }
+
+ protected OneInputStreamOperator<CdcMultiplexRecord, MultiTableCommittable> createWriteOperator(
+ StoreSinkWrite.Provider writeProvider, String commitUser) {
+ return new CdcRecordStoreMultiWriteOperator(
+ catalogLoader, writeProvider, commitUser, new Options());
+ }
+
+ // Table committers are dynamically created at runtime
+ protected SerializableFunction<
+ String, Committer<MultiTableCommittable, WrappedManifestCommittable>>
+ createCommitterFactory() {
+ // If checkpoint is enabled for streaming job, we have to
+ // commit new files list even if they're empty.
+ // Otherwise we can't tell if the commit is successful after
+ // a restart.
+ return user -> new StoreMultiCommitter(user, catalogLoader);
+ }
+
+ protected CommittableStateManager<WrappedManifestCommittable> createCommittableStateManager() {
+ return new RestoreAndFailCommittableStateManager<>(
+ () -> new VersionedSerializerWrapper<>(new WrappedManifestCommittableSerializer()));
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java
index ea221e4a6..9bf3f88a3 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java
@@ -18,6 +18,9 @@
package org.apache.paimon.flink.sink.cdc;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.flink.action.cdc.mysql.MySqlDatabaseSyncMode;
+import org.apache.paimon.flink.sink.FlinkStreamPartitioner;
import org.apache.paimon.flink.utils.SingleOutputStreamOperatorUtils;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.table.BucketMode;
@@ -26,6 +29,8 @@ import org.apache.paimon.utils.Preconditions;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.transformations.PartitionTransformation;
import javax.annotation.Nullable;
@@ -41,6 +46,10 @@ import static org.apache.paimon.flink.sink.FlinkStreamPartitioner.partition;
* <p>This builder will create a separate sink for each Paimon sink table. Thus this implementation
* is not very efficient in resource saving.
*
+ * <p>For newly added tables, this builder will create a multiplexed Paimon sink to handle all
+ * tables added during runtime. Note that the topology of the Flink job is likely to change when
+ * there is newly added table and the job resume from a given savepoint.
+ *
* @param <T> CDC change event type
*/
public class FlinkCdcSyncDatabaseSinkBuilder<T> {
@@ -50,6 +59,15 @@ public class FlinkCdcSyncDatabaseSinkBuilder<T> {
private List<FileStoreTable> tables = new ArrayList<>();
@Nullable private Integer parallelism;
+ // Paimon catalog used to check and create tables. There will be two
+ // places where this catalog is used. 1) in processing function,
+ // it will check newly added tables and create the corresponding
+ // Paimon tables. 2) in multiplex sink where it is used to
+ // initialize different writers to multiple tables.
+ private Catalog.Loader catalogLoader;
+ // database to sync, currently only support single database
+ private String database;
+ private MySqlDatabaseSyncMode mode;
public FlinkCdcSyncDatabaseSinkBuilder<T> withInput(DataStream<T> input) {
this.input = input;
@@ -76,6 +94,58 @@ public class FlinkCdcSyncDatabaseSinkBuilder<T> {
Preconditions.checkNotNull(input);
Preconditions.checkNotNull(parserFactory);
+ StreamExecutionEnvironment env = input.getExecutionEnvironment();
+ if (mode == MySqlDatabaseSyncMode.DYNAMIC) {
+ buildDynamicCdcSink(env);
+ } else {
+ buildStaticCdcSink(env);
+ }
+ }
+
+ private void buildDynamicCdcSink(StreamExecutionEnvironment env) {
+ SingleOutputStreamOperator<Void> parsed =
+ input.forward()
+ .process(
+ new CdcDynamicTableParsingProcessFunction<>(
+ database, catalogLoader, parserFactory))
+ .setParallelism(input.getParallelism());
+
+ // for newly-added tables, create a multiplexing operator that handles all their records
+ // and writes to multiple tables
+ DataStream<CdcMultiplexRecord> newlyAddedTableStream =
+ SingleOutputStreamOperatorUtils.getSideOutput(
+ parsed, CdcDynamicTableParsingProcessFunction.DYNAMIC_OUTPUT_TAG);
+ // handles schema change for newly added tables
+ SingleOutputStreamOperatorUtils.getSideOutput(
+ parsed,
+ CdcDynamicTableParsingProcessFunction.DYNAMIC_SCHEMA_CHANGE_OUTPUT_TAG)
+ .process(new MultiTableUpdatedDataFieldsProcessFunction(catalogLoader));
+
+ FlinkStreamPartitioner<CdcMultiplexRecord> partitioner =
+ new FlinkStreamPartitioner<>(new CdcMultiplexRecordChannelComputer(catalogLoader));
+ PartitionTransformation<CdcMultiplexRecord> partitioned =
+ new PartitionTransformation<>(
+ newlyAddedTableStream.getTransformation(), partitioner);
+
+ if (parallelism != null) {
+ partitioned.setParallelism(parallelism);
+ }
+
+ FlinkCdcMultiTableSink sink = new FlinkCdcMultiTableSink(catalogLoader);
+ sink.sinkFrom(new DataStream<>(input.getExecutionEnvironment(), partitioned));
+ }
+
+ private void buildForDynamicBucket(FileStoreTable table, DataStream<CdcRecord> parsed) {
+ new CdcDynamicBucketSink(table).build(parsed, parallelism);
+ }
+
+ private void buildForFixedBucket(FileStoreTable table, DataStream<CdcRecord> parsed) {
+ DataStream<CdcRecord> partitioned =
+ partition(parsed, new CdcRecordChannelComputer(table.schema()), parallelism);
+ new FlinkCdcSink(table).sinkFrom(partitioned);
+ }
+
+ private void buildStaticCdcSink(StreamExecutionEnvironment env) {
SingleOutputStreamOperator<Void> parsed =
input.forward()
.process(new CdcMultiTableParsingProcessFunction<>(parserFactory))
@@ -115,13 +185,18 @@ public class FlinkCdcSyncDatabaseSinkBuilder<T> {
}
}
- private void buildForDynamicBucket(FileStoreTable table, DataStream<CdcRecord> parsed) {
- new CdcDynamicBucketSink(table).build(parsed, parallelism);
+ public FlinkCdcSyncDatabaseSinkBuilder<T> withDatabase(String database) {
+ this.database = database;
+ return this;
}
- private void buildForFixedBucket(FileStoreTable table, DataStream<CdcRecord> parsed) {
- DataStream<CdcRecord> partitioned =
- partition(parsed, new CdcRecordChannelComputer(table.schema()), parallelism);
- new FlinkCdcSink(table).sinkFrom(partitioned);
+ public FlinkCdcSyncDatabaseSinkBuilder<T> withCatalogLoader(Catalog.Loader catalogLoader) {
+ this.catalogLoader = catalogLoader;
+ return this;
+ }
+
+ public FlinkCdcSyncDatabaseSinkBuilder<T> withMode(MySqlDatabaseSyncMode mode) {
+ this.mode = mode;
+ return this;
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java
new file mode 100644
index 000000000..1f0c800d6
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink.cdc;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypeChecks;
+import org.apache.paimon.types.DataTypeRoot;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Preconditions;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * A {@link ProcessFunction} to handle schema changes. New schema is represented by a list of {@link
+ * DataField}s.
+ *
+ * <p>NOTE: To avoid concurrent schema changes, the parallelism of this {@link ProcessFunction} must
+ * be 1.
+ */
+public class MultiTableUpdatedDataFieldsProcessFunction
+ extends ProcessFunction<Tuple2<Identifier, List<DataField>>, Void> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MultiTableUpdatedDataFieldsProcessFunction.class);
+
+ private final Map<Identifier, SchemaManager> schemaManagers = new HashMap<>();
+ private Catalog catalog;
+ private final Catalog.Loader catalogLoader;
+
+ public MultiTableUpdatedDataFieldsProcessFunction(Catalog.Loader catalogLoader) {
+ this.catalogLoader = catalogLoader;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ this.catalog = catalogLoader.load();
+ }
+
+ @Override
+ public void processElement(
+ Tuple2<Identifier, List<DataField>> updatedDataFields,
+ Context context,
+ Collector<Void> collector)
+ throws Exception {
+ Identifier tableId = updatedDataFields.f0;
+ SchemaManager schemaManager =
+ schemaManagers.computeIfAbsent(
+ tableId,
+ id -> {
+ FileStoreTable table;
+ try {
+ table = (FileStoreTable) catalog.getTable(tableId);
+ } catch (Catalog.TableNotExistException e) {
+ return null;
+ }
+ return new SchemaManager(table.fileIO(), table.location());
+ });
+
+ if (Objects.isNull(schemaManager)) {
+ throw new IOException("Failed to get schema manager for table " + tableId);
+ }
+
+ for (SchemaChange schemaChange :
+ extractSchemaChanges(schemaManager, updatedDataFields.f1)) {
+ applySchemaChange(schemaManager, schemaChange);
+ }
+ }
+
+ private List<SchemaChange> extractSchemaChanges(
+ SchemaManager schemaManager, List<DataField> updatedDataFields) {
+ RowType oldRowType = schemaManager.latest().get().logicalRowType();
+ Map<String, DataField> oldFields = new HashMap<>();
+ for (DataField oldField : oldRowType.getFields()) {
+ oldFields.put(oldField.name(), oldField);
+ }
+
+ List<SchemaChange> result = new ArrayList<>();
+ for (DataField newField : updatedDataFields) {
+ if (oldFields.containsKey(newField.name())) {
+ DataField oldField = oldFields.get(newField.name());
+ // we compare by ignoring nullable, because partition keys and primary keys might be
+ // nullable in source database, but they can't be null in Paimon
+ if (oldField.type().equalsIgnoreNullable(newField.type())) {
+ if (!Objects.equals(oldField.description(), newField.description())) {
+ result.add(
+ SchemaChange.updateColumnComment(
+ new String[] {newField.name()}, newField.description()));
+ }
+ } else {
+ result.add(SchemaChange.updateColumnType(newField.name(), newField.type()));
+ if (newField.description() != null) {
+ result.add(
+ SchemaChange.updateColumnComment(
+ new String[] {newField.name()}, newField.description()));
+ }
+ }
+ } else {
+ result.add(
+ SchemaChange.addColumn(
+ newField.name(), newField.type(), newField.description(), null));
+ }
+ }
+ return result;
+ }
+
+ private void applySchemaChange(SchemaManager schemaManager, SchemaChange schemaChange)
+ throws Exception {
+ if (schemaChange instanceof SchemaChange.AddColumn) {
+ try {
+ schemaManager.commitChanges(schemaChange);
+ } catch (Catalog.ColumnAlreadyExistException e) {
+ // This is normal. For example when a table is split into multiple database tables,
+ // all these tables will be added the same column. However schemaManager can't
+ // handle duplicated column adds, so we just catch the exception and log it.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Failed to perform SchemaChange.AddColumn {}, "
+ + "possibly due to duplicated column name",
+ schemaChange,
+ e);
+ }
+ }
+ } else if (schemaChange instanceof SchemaChange.UpdateColumnType) {
+ SchemaChange.UpdateColumnType updateColumnType =
+ (SchemaChange.UpdateColumnType) schemaChange;
+ TableSchema schema =
+ schemaManager
+ .latest()
+ .orElseThrow(
+ () ->
+ new RuntimeException(
+ "Table does not exist. This is unexpected."));
+ int idx = schema.fieldNames().indexOf(updateColumnType.fieldName());
+ Preconditions.checkState(
+ idx >= 0,
+ "Field name "
+ + updateColumnType.fieldName()
+ + " does not exist in table. This is unexpected.");
+ DataType oldType = schema.fields().get(idx).type();
+ DataType newType = updateColumnType.newDataType();
+ switch (canConvert(oldType, newType)) {
+ case CONVERT:
+ schemaManager.commitChanges(schemaChange);
+ break;
+ case EXCEPTION:
+ throw new UnsupportedOperationException(
+ String.format(
+ "Cannot convert field %s from type %s to %s",
+ updateColumnType.fieldName(), oldType, newType));
+ }
+ } else if (schemaChange instanceof SchemaChange.UpdateColumnComment) {
+ schemaManager.commitChanges(schemaChange);
+ } else {
+ throw new UnsupportedOperationException(
+ "Unsupported schema change class "
+ + schemaChange.getClass().getName()
+ + ", content "
+ + schemaChange);
+ }
+ }
+
+ private static final List<DataTypeRoot> STRING_TYPES =
+ Arrays.asList(DataTypeRoot.CHAR, DataTypeRoot.VARCHAR);
+ private static final List<DataTypeRoot> BINARY_TYPES =
+ Arrays.asList(DataTypeRoot.BINARY, DataTypeRoot.VARBINARY);
+ private static final List<DataTypeRoot> INTEGER_TYPES =
+ Arrays.asList(
+ DataTypeRoot.TINYINT,
+ DataTypeRoot.SMALLINT,
+ DataTypeRoot.INTEGER,
+ DataTypeRoot.BIGINT);
+ private static final List<DataTypeRoot> FLOATING_POINT_TYPES =
+ Arrays.asList(DataTypeRoot.FLOAT, DataTypeRoot.DOUBLE);
+
+ public static ConvertAction canConvert(DataType oldType, DataType newType) {
+ if (oldType.equalsIgnoreNullable(newType)) {
+ return ConvertAction.CONVERT;
+ }
+
+ int oldIdx = STRING_TYPES.indexOf(oldType.getTypeRoot());
+ int newIdx = STRING_TYPES.indexOf(newType.getTypeRoot());
+ if (oldIdx >= 0 && newIdx >= 0) {
+ return DataTypeChecks.getLength(oldType) <= DataTypeChecks.getLength(newType)
+ ? ConvertAction.CONVERT
+ : ConvertAction.IGNORE;
+ }
+
+ oldIdx = BINARY_TYPES.indexOf(oldType.getTypeRoot());
+ newIdx = BINARY_TYPES.indexOf(newType.getTypeRoot());
+ if (oldIdx >= 0 && newIdx >= 0) {
+ return DataTypeChecks.getLength(oldType) <= DataTypeChecks.getLength(newType)
+ ? ConvertAction.CONVERT
+ : ConvertAction.IGNORE;
+ }
+
+ oldIdx = INTEGER_TYPES.indexOf(oldType.getTypeRoot());
+ newIdx = INTEGER_TYPES.indexOf(newType.getTypeRoot());
+ if (oldIdx >= 0 && newIdx >= 0) {
+ return oldIdx <= newIdx ? ConvertAction.CONVERT : ConvertAction.IGNORE;
+ }
+
+ oldIdx = FLOATING_POINT_TYPES.indexOf(oldType.getTypeRoot());
+ newIdx = FLOATING_POINT_TYPES.indexOf(newType.getTypeRoot());
+ if (oldIdx >= 0 && newIdx >= 0) {
+ return oldIdx <= newIdx ? ConvertAction.CONVERT : ConvertAction.IGNORE;
+ }
+
+ return ConvertAction.EXCEPTION;
+ }
+
+ /**
+ * Return type of {@link MultiTableUpdatedDataFieldsProcessFunction#canConvert(DataType,
+ * DataType)}. This enum indicates the action to perform.
+ */
+ public enum ConvertAction {
+
+ /** {@code oldType} can be converted to {@code newType}. */
+ CONVERT,
+
+ /**
+ * {@code oldType} and {@code newType} belongs to the same type family, but old type has
+ * higher precision than new type. Ignore this convert request.
+ */
+ IGNORE,
+
+ /**
+ * {@code oldType} and {@code newType} belongs to different type family. Throw an exception
+ * indicating that this convert request cannot be handled.
+ */
+ EXCEPTION
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/resources/META-INF/NOTICE b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/NOTICE
new file mode 100644
index 000000000..c9732f837
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,10 @@
+paimon-flink-common
+Copyright 2023-2023 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt)
+
+- com.alibaba:druid:1.2.15
+
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseActionITCase.java
index 52b14119d..44caea410 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseActionITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncDatabaseActionITCase.java
@@ -371,7 +371,8 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
null,
null,
Collections.emptyMap(),
- tableConfig);
+ tableConfig,
+ MySqlDatabaseSyncMode.STATIC);
action.build(env);
JobClient client = env.executeAsync();
waitJobRunning(client);
@@ -545,7 +546,8 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
includingTables,
excludingTables,
Collections.emptyMap(),
- tableConfig);
+ tableConfig,
+ MySqlDatabaseSyncMode.STATIC);
action.build(env);
JobClient client = env.executeAsync();
waitJobRunning(client);
@@ -555,7 +557,6 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
assertTableNotExists(notExistedTables);
}
- @Test
@Timeout(60)
public void testIgnoreCase() throws Exception {
Map<String, String> mySqlConfig = getBasicMySqlConfig();
@@ -635,16 +636,55 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
@Test
@Timeout(600)
public void testNewlyAddedTables() throws Exception {
- JobClient client = buildSyncDatabaseActionWithNewlyAddedTables();
+ testNewlyAddedTable(1, true, false, "paimon_sync_database_newly_added_tables");
+ }
+
+ @Test
+ @Timeout(600)
+ public void testNewlyAddedTableSingleTable() throws Exception {
+ testNewlyAddedTable(1, false, false, "paimon_sync_database_newly_added_tables_1");
+ }
+
+ @Test
+ @Timeout(600)
+ public void testNewlyAddedTableMultipleTables() throws Exception {
+ testNewlyAddedTable(3, false, false, "paimon_sync_database_newly_added_tables_2");
+ }
+
+ @Test
+ @Timeout(600)
+ public void testNewlyAddedTableSchemaChange() throws Exception {
+ testNewlyAddedTable(1, false, true, "paimon_sync_database_newly_added_tables_3");
+ }
+
+ @Test
+ @Timeout(600)
+ public void testNewlyAddedTableSingleTableWithSavepoint() throws Exception {
+ testNewlyAddedTable(1, true, true, "paimon_sync_database_newly_added_tables_4");
+ }
+
+ public void testNewlyAddedTable(
+ int numOfNewlyAddedTables,
+ boolean testSavepointRecovery,
+ boolean testSchemaChange,
+ String databaseName)
+ throws Exception {
+ JobClient client = buildSyncDatabaseActionWithNewlyAddedTables(databaseName);
waitJobRunning(client);
try (Connection conn =
DriverManager.getConnection(
- MYSQL_CONTAINER.getJdbcUrl("paimon_sync_database_newly_added_tables"),
+ MYSQL_CONTAINER.getJdbcUrl(databaseName),
MYSQL_CONTAINER.getUsername(),
MYSQL_CONTAINER.getPassword())) {
try (Statement statement = conn.createStatement()) {
- testNewlyAddedTableImpl(client, statement, 1, true, false);
+ testNewlyAddedTableImpl(
+ client,
+ statement,
+ numOfNewlyAddedTables,
+ testSavepointRecovery,
+ testSchemaChange,
+ databaseName);
}
}
}
@@ -654,12 +694,13 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
Statement statement,
int newlyAddedTableCount,
boolean testSavepointRecovery,
- boolean testSchemaChange)
+ boolean testSchemaChange,
+ String databaseName)
throws Exception {
FileStoreTable table1 = getFileStoreTable("t1");
FileStoreTable table2 = getFileStoreTable("t2");
- statement.executeUpdate("USE paimon_sync_database_newly_added_tables");
+ statement.executeUpdate("USE " + databaseName);
statement.executeUpdate("INSERT INTO t1 VALUES (1, 'one')");
statement.executeUpdate("INSERT INTO t2 VALUES (2, 'two', 20, 200)");
@@ -699,11 +740,12 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
String newTableName = getNewTableName(newTableCount);
createNewTable(statement, newTableName);
- statement.executeUpdate("INSERT INTO t2 VALUES (8, 'eight', 80, 800)");
+ statement.executeUpdate(
+ String.format("INSERT INTO `%s`.`t2` VALUES (8, 'eight', 80, 800)", databaseName));
List<Tuple2<Integer, String>> newTableRecords = getNewTableRecords(newTableCount);
recordsMap.put(newTableName, newTableRecords);
List<String> newTableExpected = getNewTableExpected(newTableRecords);
- insertRecordsIntoNewTable(statement, newTableName, newTableRecords);
+ insertRecordsIntoNewTable(statement, databaseName, newTableName, newTableRecords);
// suspend the job and restart from savepoint
if (testSavepointRecovery) {
@@ -715,7 +757,7 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
.join();
assertThat(savepoint).isNotBlank();
- client = buildSyncDatabaseActionWithNewlyAddedTables(savepoint);
+ client = buildSyncDatabaseActionWithNewlyAddedTables(savepoint, databaseName);
waitJobRunning(client);
}
@@ -739,7 +781,7 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
// insert records
newTableRecords = getNewTableRecords(newTableCount);
recordsMap.put(newTableName, newTableRecords);
- insertRecordsIntoNewTable(statement, newTableName, newTableRecords);
+ insertRecordsIntoNewTable(statement, databaseName, newTableName, newTableRecords);
newTable = getFileStoreTable(newTableName);
newTableExpected = getNewTableExpected(newTableRecords);
waitForResult(newTableExpected, newTable, newTableRowType, newTablePrimaryKeys);
@@ -752,9 +794,11 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
String tableName = getNewTableName(pick);
List<Tuple2<Integer, String>> records = recordsMap.get(tableName);
records.add(Tuple2.of(80, "eighty"));
- newTable = getFileStoreTable(newTableName);
+ newTable = getFileStoreTable(tableName);
newTableExpected = getNewTableExpected(records);
- statement.executeUpdate(String.format("INSERT INTO %s VALUES (80, 'eighty')", tableName));
+ statement.executeUpdate(
+ String.format(
+ "INSERT INTO `%s`.`%s` VALUES (80, 'eighty')", databaseName, tableName));
waitForResult(newTableExpected, newTable, newTableRowType, newTablePrimaryKeys);
@@ -764,9 +808,13 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
tableName = getNewTableName(pick);
records = recordsMap.get(tableName);
- statement.executeUpdate(String.format("ALTER TABLE %s ADD COLUMN v2 INT", tableName));
statement.executeUpdate(
- String.format("INSERT INTO %s VALUES (100, 'hundred', 10000)", tableName));
+ String.format(
+ "ALTER TABLE `%s`.`%s` ADD COLUMN v2 INT", databaseName, tableName));
+ statement.executeUpdate(
+ String.format(
+ "INSERT INTO `%s`.`%s` VALUES (100, 'hundred', 10000)",
+ databaseName, tableName));
List<String> expectedRecords =
records.stream()
@@ -801,11 +849,15 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
}
private void insertRecordsIntoNewTable(
- Statement statement, String newTableName, List<Tuple2<Integer, String>> newTableRecords)
+ Statement statement,
+ String databaseName,
+ String newTableName,
+ List<Tuple2<Integer, String>> newTableRecords)
throws SQLException {
String sql =
String.format(
- "INSERT INTO %s VALUES %s",
+ "INSERT INTO `%s`.`%s` VALUES %s",
+ databaseName,
newTableName,
newTableRecords.stream()
.map(tuple -> String.format("(%d, '%s')", tuple.f0, tuple.f1))
@@ -823,15 +875,17 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
"CREATE TABLE %s (k INT, v1 VARCHAR(10), PRIMARY KEY (k))", newTableName));
}
- private JobClient buildSyncDatabaseActionWithNewlyAddedTables() throws Exception {
- return buildSyncDatabaseActionWithNewlyAddedTables(null);
+ private JobClient buildSyncDatabaseActionWithNewlyAddedTables(String databaseName)
+ throws Exception {
+ return buildSyncDatabaseActionWithNewlyAddedTables(null, databaseName);
}
- private JobClient buildSyncDatabaseActionWithNewlyAddedTables(String savepointPath)
- throws Exception {
+ private JobClient buildSyncDatabaseActionWithNewlyAddedTables(
+ String savepointPath, String databaseName) throws Exception {
Map<String, String> mySqlConfig = getBasicMySqlConfig();
- mySqlConfig.put("database-name", "paimon_sync_database_newly_added_tables");
+ mySqlConfig.put("database-name", databaseName);
+ mySqlConfig.put("scan.incremental.snapshot.chunk.size", "1");
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(2);
env.enableCheckpointing(1000);
@@ -840,7 +894,7 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
ThreadLocalRandom random = ThreadLocalRandom.current();
Map<String, String> tableConfig = new HashMap<>();
tableConfig.put("bucket", String.valueOf(random.nextInt(3) + 1));
- tableConfig.put("sink.parallelism", String.valueOf(random.nextInt(3) + 1));
+ tableConfig.put("sink.parallelism", String.valueOf(random.nextInt(2) + 2));
MySqlSyncDatabaseAction action =
new MySqlSyncDatabaseAction(
mySqlConfig,
@@ -852,7 +906,8 @@ public class MySqlSyncDatabaseActionITCase extends MySqlActionITCaseBase {
"t.+",
null,
Collections.emptyMap(),
- tableConfig);
+ tableConfig,
+ MySqlDatabaseSyncMode.DYNAMIC);
action.build(env);
if (Objects.nonNull(savepointPath)) {
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java
new file mode 100644
index 000000000..ed54971e1
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/CdcMultiplexRecordChannelComputerTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink.cdc;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowKind;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.TraceableFileIO;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/** Tests for {@link CdcRecordChannelComputer}. */
+public class CdcMultiplexRecordChannelComputerTest {
+
+ @TempDir java.nio.file.Path tempDir;
+ private Catalog.Loader catalogLoader;
+ private Path warehouse;
+ private String databaseName;
+ private Identifier tableWithPartition;
+ private Catalog catalog;
+ private Identifier tableWithoutPartition;
+
+ @BeforeEach
+ public void before() throws Exception {
+ warehouse = new Path(TraceableFileIO.SCHEME + "://" + tempDir.toString());
+
+ databaseName = "test_db";
+ tableWithPartition = Identifier.create(databaseName, "test_table1");
+ tableWithoutPartition = Identifier.create(databaseName, "test_table2");
+
+ Options catalogOptions = new Options();
+ catalogOptions.set(CatalogOptions.WAREHOUSE, warehouse.getPath());
+ catalogOptions.set(CatalogOptions.URI, "");
+ catalogLoader = () -> CatalogFactory.createCatalog(CatalogContext.create(catalogOptions));
+ catalog = catalogLoader.load();
+ catalog.createDatabase(databaseName, true);
+
+ Options conf = new Options();
+ conf.set(CdcRecordStoreWriteOperator.RETRY_SLEEP_TIME, Duration.ofMillis(10));
+
+ RowType rowTypeWithPartition =
+ RowType.of(
+ new DataType[] {DataTypes.INT(), DataTypes.BIGINT(), DataTypes.DOUBLE()},
+ new String[] {"pt", "k", "v"});
+ RowType rowTypeWithoutPartition =
+ RowType.of(
+ new DataType[] {
+ DataTypes.BIGINT(), DataTypes.DOUBLE(),
+ },
+ new String[] {"k", "v"});
+
+ List<Tuple2<Identifier, Schema>> tables =
+ Arrays.asList(
+ Tuple2.of(
+ tableWithPartition,
+ new Schema(
+ rowTypeWithPartition.getFields(),
+ Collections.singletonList("pt"),
+ Arrays.asList("pt", "k"),
+ conf.toMap(),
+ "")),
+ Tuple2.of(
+ tableWithoutPartition,
+ new Schema(
+ rowTypeWithoutPartition.getFields(),
+ Collections.emptyList(),
+ Collections.singletonList("k"),
+ conf.toMap(),
+ "")));
+
+ for (Tuple2<Identifier, Schema> tableAndSchema : tables) {
+ catalog.createTable(tableAndSchema.f0, tableAndSchema.f1, false);
+ }
+ }
+
+ @Test
+ public void testSchemaWithPartition() throws Exception {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ int numInputs = random.nextInt(1000) + 1;
+ List<Map<String, String>> input = new ArrayList<>();
+ for (int i = 0; i < numInputs; i++) {
+ Map<String, String> fields = new HashMap<>();
+ fields.put("pt", String.valueOf(random.nextInt(10) + 1));
+ fields.put("k", String.valueOf(random.nextLong()));
+ fields.put("v", String.valueOf(random.nextDouble()));
+ input.add(fields);
+ }
+
+ testImpl(tableWithPartition, input);
+ }
+
+ @Test
+ public void testSchemaNoPartition() {
+
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ int numInputs = random.nextInt(1000) + 1;
+ List<Map<String, String>> input = new ArrayList<>();
+ for (int i = 0; i < numInputs; i++) {
+ Map<String, String> fields = new HashMap<>();
+ fields.put("k", String.valueOf(random.nextLong()));
+ fields.put("v", String.valueOf(random.nextDouble()));
+ input.add(fields);
+ }
+
+ testImpl(tableWithoutPartition, input);
+ }
+
+ private void testImpl(Identifier tableId, List<Map<String, String>> input) {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+
+ int numChannels = random.nextInt(10) + 1;
+ CdcMultiplexRecordChannelComputer channelComputer =
+ new CdcMultiplexRecordChannelComputer(catalogLoader);
+ channelComputer.setup(numChannels);
+
+ // assert that insert and delete records are routed into same channel
+
+ for (Map<String, String> fields : input) {
+ CdcRecord insertRecord = new CdcRecord(RowKind.INSERT, fields);
+ CdcRecord deleteRecord = new CdcRecord(RowKind.DELETE, fields);
+
+ assertThat(
+ channelComputer.channel(
+ CdcMultiplexRecord.fromCdcRecord(
+ tableId.getDatabaseName(),
+ tableId.getObjectName(),
+ insertRecord)))
+ .isEqualTo(
+ channelComputer.channel(
+ CdcMultiplexRecord.fromCdcRecord(
+ tableId.getDatabaseName(),
+ tableId.getObjectName(),
+ deleteRecord)));
+ }
+
+ // assert that channel >= 0
+ int numTests = random.nextInt(10) + 1;
+ for (int test = 0; test < numTests; test++) {
+ Map<String, String> fields = input.get(random.nextInt(input.size()));
+ CdcRecord record = new CdcRecord(RowKind.INSERT, fields);
+
+ int numBuckets = random.nextInt(numChannels * 4) + 1;
+ for (int i = 0; i < numBuckets; i++) {
+ int channel =
+ channelComputer.channel(
+ CdcMultiplexRecord.fromCdcRecord(
+ tableId.getDatabaseName(),
+ tableId.getObjectName(),
+ record));
+ assertTrue(channel >= 0);
+ }
+ }
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcEventParser.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcEventParser.java
index 204f8536d..7288138bc 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcEventParser.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/cdc/TestCdcEventParser.java
@@ -18,11 +18,13 @@
package org.apache.paimon.flink.sink.cdc;
+import org.apache.paimon.schema.Schema;
import org.apache.paimon.types.DataField;
import org.apache.paimon.utils.ObjectUtils;
import java.util.Collections;
import java.util.List;
+import java.util.Optional;
/** Testing {@link EventParser} for {@link TestCdcEvent}. */
public class TestCdcEventParser implements EventParser<TestCdcEvent> {
@@ -44,8 +46,18 @@ public class TestCdcEventParser implements EventParser<TestCdcEvent> {
return ObjectUtils.coalesce(raw.updatedDataFields(), Collections.emptyList());
}
+ @Override
+ public String parseDatabaseName() {
+ return null;
+ }
+
@Override
public List<CdcRecord> parseRecords() {
return ObjectUtils.coalesce(raw.records(), Collections.emptyList());
}
+
+ @Override
+ public Optional<Schema> parseNewTable(String databaseName) {
+ return Optional.empty();
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/test/resources/mysql/setup.sql b/paimon-flink/paimon-flink-common/src/test/resources/mysql/setup.sql
index 351324bf9..b4c877c40 100644
--- a/paimon-flink/paimon-flink-common/src/test/resources/mysql/setup.sql
+++ b/paimon-flink/paimon-flink-common/src/test/resources/mysql/setup.sql
@@ -459,12 +459,82 @@ CREATE TABLE T (
-- ################################################################################
--- MySqlSyncDatabaseActionITCase#testTableAffix
+-- MySqlSyncDatabaseActionITCase#testNewlyAddedTables
-- ################################################################################
CREATE DATABASE paimon_sync_database_newly_added_tables;
USE paimon_sync_database_newly_added_tables;
+CREATE TABLE t1 (
+ k INT,
+ v1 VARCHAR(10),
+ PRIMARY KEY (k)
+);
+
+CREATE TABLE t2 (
+ k1 INT,
+ k2 VARCHAR(10),
+ v1 INT,
+ v2 BIGINT,
+ PRIMARY KEY (k1, k2)
+);
+
+CREATE DATABASE paimon_sync_database_newly_added_tables_1;
+USE paimon_sync_database_newly_added_tables_1;
+
+CREATE TABLE t1 (
+ k INT,
+ v1 VARCHAR(10),
+ PRIMARY KEY (k)
+);
+
+CREATE TABLE t2 (
+ k1 INT,
+ k2 VARCHAR(10),
+ v1 INT,
+ v2 BIGINT,
+ PRIMARY KEY (k1, k2)
+);
+
+
+CREATE DATABASE paimon_sync_database_newly_added_tables_2;
+USE paimon_sync_database_newly_added_tables_2;
+
+CREATE TABLE t1 (
+ k INT,
+ v1 VARCHAR(10),
+ PRIMARY KEY (k)
+);
+
+CREATE TABLE t2 (
+ k1 INT,
+ k2 VARCHAR(10),
+ v1 INT,
+ v2 BIGINT,
+ PRIMARY KEY (k1, k2)
+);
+
+
+CREATE DATABASE paimon_sync_database_newly_added_tables_3;
+USE paimon_sync_database_newly_added_tables_3;
+
+CREATE TABLE t1 (
+ k INT,
+ v1 VARCHAR(10),
+ PRIMARY KEY (k)
+);
+
+CREATE TABLE t2 (
+ k1 INT,
+ k2 VARCHAR(10),
+ v1 INT,
+ v2 BIGINT,
+ PRIMARY KEY (k1, k2)
+);
+
+
+CREATE DATABASE paimon_sync_database_newly_added_tables_4;
+USE paimon_sync_database_newly_added_tables_4;
CREATE TABLE t1 (
k INT,