You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by le...@apache.org on 2024/12/24 06:11:52 UTC
(flink-cdc) 03/04: [FLINK-36763][cdc-runtime] Introduce distributed schema evolution topology for sources with parallelized metadata
This is an automated email from the ASF dual-hosted git repository.
leonard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
commit 2e03f9ad49f80342a841a235074cdb26955c3743
Author: yuxiqian <34...@users.noreply.github.com>
AuthorDate: Mon Dec 23 18:43:13 2024 +0800
[FLINK-36763][cdc-runtime] Introduce distributed schema evolution topology for sources with parallelized metadata
This closes #3801
---
.../flink/cdc/common/event/AddColumnEvent.java | 5 +
.../cdc/common/event/AlterColumnTypeEvent.java | 5 +
.../flink/cdc/common/event/CreateTableEvent.java | 5 +
.../flink/cdc/common/event/DataChangeEvent.java | 10 +
.../flink/cdc/common/event/DropColumnEvent.java | 5 +
.../flink/cdc/common/event/DropTableEvent.java | 5 +
.../apache/flink/cdc/common/event/FlushEvent.java | 21 +-
.../flink/cdc/common/event/RenameColumnEvent.java | 5 +
.../flink/cdc/common/event/SchemaChangeEvent.java | 3 +
.../org/apache/flink/cdc/common/event/TableId.java | 6 +-
.../flink/cdc/common/event/TruncateTableEvent.java | 5 +
.../apache/flink/cdc/common/route/RouteRule.java | 4 +
.../org/apache/flink/cdc/common/schema/Schema.java | 11 +-
.../apache/flink/cdc/common/source/DataSource.java | 15 +
.../cdc/composer/flink/FlinkPipelineComposer.java | 132 ++-
.../flink/translator/DataSourceTranslator.java | 38 +-
.../flink/translator/PartitioningTranslator.java | 27 +-
.../flink/translator/SchemaOperatorTranslator.java | 61 +-
.../flink/translator/TransformTranslator.java | 16 +-
.../flink/FlinkParallelizedPipelineITCase.java | 1011 ++++++++++++++++++++
.../flink/FlinkPipelineComposerITCase.java | 43 +-
.../flink/FlinkPipelineComposerLenientITCase.java | 8 +-
.../factory/DistributedDataSourceFactory.java | 62 ++
.../testsource/source/DistributedDataSource.java | 56 ++
.../source/DistributedSourceFunction.java | 275 ++++++
.../source/DistributedSourceOptions.java | 37 +
.../org.apache.flink.cdc.common.factories.Factory | 1 +
.../resources/ref-output/distributed-ignore.txt | 172 ++++
.../src/test/resources/ref-output/distributed.txt | 300 ++++++
.../src/test/resources/ref-output/regular.txt | 300 ++++++
.../doris/sink/DorisMetadataApplierITCase.java | 2 +-
.../pom.xml | 74 +-
.../flink-cdc-pipeline-connector-mysql/pom.xml | 14 +
.../connectors/mysql/source/MySqlDataSource.java | 7 +
.../source/MySqlParallelizedPipelineITCase.java | 207 ++++
.../mysql/testutils/MySqSourceTestUtils.java | 17 +
.../sink/v2/bucket/BucketAssignOperator.java | 2 +-
.../v2/bucket/BucketWrapperEventSerializer.java | 5 +-
.../sink/v2/bucket/BucketWrapperFlushEvent.java | 11 +-
.../sink/StarRocksMetadataApplierITCase.java | 2 +-
.../values/sink/ValuesDataSinkHelper.java | 14 +-
.../flink/cdc/pipeline/tests/MysqlE2eITCase.java | 14 +-
.../flink/cdc/pipeline/tests/RouteE2eITCase.java | 125 ++-
.../cdc/pipeline/tests/SchemaEvolveE2eITCase.java | 35 +-
.../tests/SchemaEvolvingTransformE2eITCase.java | 31 +-
.../cdc/pipeline/tests/TransformE2eITCase.java | 10 +-
.../tests/utils/PipelineTestEnvironment.java | 13 +-
.../CoordinationResponseUtils.java | 2 +-
.../common/CoordinatorExecutorThreadFactory.java | 60 ++
.../operators/schema/common/SchemaDerivator.java | 345 +++++++
.../{coordinator => common}/SchemaManager.java | 164 +---
.../operators/schema/common/SchemaRegistry.java | 397 ++++++++
.../operators/schema/common/TableIdRouter.java | 98 ++
.../{ => common}/event/FlushSuccessEvent.java | 39 +-
.../event/GetEvolvedSchemaRequest.java | 4 +-
.../event/GetEvolvedSchemaResponse.java | 4 +-
.../event/GetOriginalSchemaRequest.java | 4 +-
.../event/GetOriginalSchemaResponse.java | 4 +-
.../event/SinkWriterRegisterEvent.java | 4 +-
.../metrics/SchemaOperatorMetrics.java | 4 +-
.../schema/coordinator/SchemaDerivation.java | 361 -------
.../schema/coordinator/SchemaRegistry.java | 444 ---------
.../coordinator/SchemaRegistryRequestHandler.java | 517 ----------
.../schema/distributed/SchemaCoordinator.java | 452 +++++++++
.../SchemaCoordinatorProvider.java} | 60 +-
.../schema/distributed/SchemaOperator.java | 231 +++++
.../schema/distributed/SchemaOperatorFactory.java | 71 ++
.../distributed/event/SchemaChangeRequest.java | 68 ++
.../distributed/event/SchemaChangeResponse.java | 58 ++
.../event/SchemaChangeProcessingResponse.java | 32 -
.../schema/event/SchemaChangeResultRequest.java | 31 -
.../schema/event/SchemaChangeResultResponse.java | 74 --
.../operators/sink/DataSinkFunctionOperator.java | 2 +-
.../operators/sink/DataSinkWriterOperator.java | 2 +-
.../operators/sink/SchemaEvolutionClient.java | 30 +-
.../operators/transform/PostTransformOperator.java | 3 +-
.../operators/transform/PreTransformOperator.java | 46 +-
...r.java => DistributedPrePartitionOperator.java} | 104 +-
.../runtime/partitioning/PartitioningEvent.java | 32 +-
...rator.java => RegularPrePartitionOperator.java} | 15 +-
.../runtime/serializer/event/EventSerializer.java | 7 +-
.../event/PartitioningEventSerializer.java | 12 +-
.../event/SchemaChangeEventSerializer.java | 5 +
.../schema/common/SchemaDerivatorTest.java | 602 ++++++++++++
.../{coordinator => common}/SchemaManagerTest.java | 11 +-
.../operators/schema/common/SchemaTestBase.java | 121 +++
.../operators/schema/common/TableIdRouterTest.java | 84 ++
.../schema/coordinator/SchemaDerivationTest.java | 412 --------
.../schema/distributed/SchemaEvolveTest.java | 381 ++++++++
.../transform/PostTransformOperatorTest.java | 82 +-
.../transform/PreTransformOperatorTest.java | 42 +-
.../TransformOperatorWithSchemaEvolveTest.java | 14 +-
.../transform/UnifiedTransformOperatorTest.java | 14 +-
.../partitioning/PrePartitionOperatorTest.java | 43 +-
.../serializer/event/EventSerializerTest.java | 8 +-
.../event/PartitioningEventSerializerTest.java | 26 +-
...va => DistributedEventOperatorTestHarness.java} | 122 +--
...s.java => RegularEventOperatorTestHarness.java} | 175 ++--
.../schema/TestingSchemaRegistryGateway.java | 2 +-
99 files changed, 6455 insertions(+), 2697 deletions(-)
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AddColumnEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AddColumnEvent.java
index 3966ff46f..bba105c58 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AddColumnEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AddColumnEvent.java
@@ -178,4 +178,9 @@ public final class AddColumnEvent implements SchemaChangeEvent {
public SchemaChangeEventType getType() {
return SchemaChangeEventType.ADD_COLUMN;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new AddColumnEvent(newTableId, addedColumns);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AlterColumnTypeEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AlterColumnTypeEvent.java
index 51acb4319..2257f134d 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AlterColumnTypeEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/AlterColumnTypeEvent.java
@@ -146,4 +146,9 @@ public class AlterColumnTypeEvent implements SchemaChangeEventWithPreSchema, Sch
public SchemaChangeEventType getType() {
return SchemaChangeEventType.ALTER_COLUMN_TYPE;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new AlterColumnTypeEvent(newTableId, typeMapping, oldTypeMapping);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/CreateTableEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/CreateTableEvent.java
index 6d3e54717..36ef4bcc5 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/CreateTableEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/CreateTableEvent.java
@@ -77,4 +77,9 @@ public class CreateTableEvent implements SchemaChangeEvent {
public SchemaChangeEventType getType() {
return SchemaChangeEventType.CREATE_TABLE;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new CreateTableEvent(newTableId, schema);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DataChangeEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DataChangeEvent.java
index da4d454dd..63cade188 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DataChangeEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DataChangeEvent.java
@@ -167,6 +167,16 @@ public class DataChangeEvent implements ChangeEvent, Serializable {
dataChangeEvent.meta);
}
+ /** Updates the {@link TableId} info of current data change event. */
+ public static DataChangeEvent route(DataChangeEvent dataChangeEvent, TableId tableId) {
+ return new DataChangeEvent(
+ tableId,
+ dataChangeEvent.before,
+ dataChangeEvent.after,
+ dataChangeEvent.op,
+ dataChangeEvent.meta);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropColumnEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropColumnEvent.java
index 76cabbde7..0ae5cf2f9 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropColumnEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropColumnEvent.java
@@ -81,4 +81,9 @@ public class DropColumnEvent implements SchemaChangeEvent {
public SchemaChangeEventType getType() {
return SchemaChangeEventType.DROP_COLUMN;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new DropColumnEvent(newTableId, droppedColumnNames);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropTableEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropTableEvent.java
index dd5efdd08..382d9d7b6 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropTableEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/DropTableEvent.java
@@ -68,4 +68,9 @@ public class DropTableEvent implements SchemaChangeEvent {
public SchemaChangeEventType getType() {
return SchemaChangeEventType.DROP_TABLE;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new DropTableEvent(newTableId);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/FlushEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/FlushEvent.java
index 468faa8b5..5fd249b7b 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/FlushEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/FlushEvent.java
@@ -25,15 +25,15 @@ import java.util.Objects;
*/
public class FlushEvent implements Event {
- /** The schema changes from which table. */
- private final TableId tableId;
+ /** Which subTask ID this FlushEvent was initiated from. */
+ private final int sourceSubTaskId;
- public FlushEvent(TableId tableId) {
- this.tableId = tableId;
+ public FlushEvent(int sourceSubTaskId) {
+ this.sourceSubTaskId = sourceSubTaskId;
}
- public TableId getTableId() {
- return tableId;
+ public int getSourceSubTaskId() {
+ return sourceSubTaskId;
}
@Override
@@ -45,11 +45,16 @@ public class FlushEvent implements Event {
return false;
}
FlushEvent that = (FlushEvent) o;
- return Objects.equals(tableId, that.tableId);
+ return sourceSubTaskId == that.sourceSubTaskId;
}
@Override
public int hashCode() {
- return Objects.hash(tableId);
+ return Objects.hash(sourceSubTaskId);
+ }
+
+ @Override
+ public String toString() {
+ return "FlushEvent{" + "sourceSubTaskId=" + sourceSubTaskId + '}';
}
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/RenameColumnEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/RenameColumnEvent.java
index 8bde37872..f38d1fd04 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/RenameColumnEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/RenameColumnEvent.java
@@ -78,4 +78,9 @@ public class RenameColumnEvent implements SchemaChangeEvent {
public SchemaChangeEventType getType() {
return SchemaChangeEventType.RENAME_COLUMN;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new RenameColumnEvent(newTableId, nameMapping);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/SchemaChangeEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/SchemaChangeEvent.java
index d5596e3b3..fdb604141 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/SchemaChangeEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/SchemaChangeEvent.java
@@ -29,4 +29,7 @@ import java.io.Serializable;
public interface SchemaChangeEvent extends ChangeEvent, Serializable {
/** Returns its {@link SchemaChangeEventType}. */
SchemaChangeEventType getType();
+
+ /** Creates a copy of {@link SchemaChangeEvent} with new {@link TableId}. */
+ SchemaChangeEvent copy(TableId newTableId);
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TableId.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TableId.java
index 1c7aff251..12348a0f4 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TableId.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TableId.java
@@ -50,6 +50,7 @@ public class TableId implements Serializable {
@Nullable private final String namespace;
@Nullable private final String schemaName;
private final String tableName;
+ private transient int cachedHashCode;
private TableId(@Nullable String namespace, @Nullable String schemaName, String tableName) {
this.namespace = namespace;
@@ -125,7 +126,10 @@ public class TableId implements Serializable {
@Override
public int hashCode() {
- return Objects.hash(namespace, schemaName, tableName);
+ if (cachedHashCode == 0) {
+ cachedHashCode = Objects.hash(namespace, schemaName, tableName);
+ }
+ return cachedHashCode;
}
@Override
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TruncateTableEvent.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TruncateTableEvent.java
index 2144ff283..29ef6d6b8 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TruncateTableEvent.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/event/TruncateTableEvent.java
@@ -68,4 +68,9 @@ public class TruncateTableEvent implements SchemaChangeEvent {
public SchemaChangeEventType getType() {
return SchemaChangeEventType.TRUNCATE_TABLE;
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return new TruncateTableEvent(newTableId);
+ }
}
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/route/RouteRule.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/route/RouteRule.java
index 4fbfb61b6..2974732fb 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/route/RouteRule.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/route/RouteRule.java
@@ -24,6 +24,10 @@ public class RouteRule implements Serializable {
private static final long serialVersionUID = 1L;
+ public RouteRule(String sourceTable, String sinkTable) {
+ this(sourceTable, sinkTable, null);
+ }
+
public RouteRule(String sourceTable, String sinkTable, String replaceSymbol) {
this.sourceTable = sourceTable;
this.sinkTable = sinkTable;
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/schema/Schema.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/schema/Schema.java
index 2db509e6e..01ca8906b 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/schema/Schema.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/schema/Schema.java
@@ -60,6 +60,12 @@ public class Schema implements Serializable {
// Used to index column by name
private transient volatile Map<String, Column> nameToColumns;
+ /**
+ * Schema might be used as a LoadingCache key frequently, and maintaining a cache of hashCode
+ * would be more efficient.
+ */
+ private transient int cachedHashCode;
+
private Schema(
List<Column> columns,
List<String> primaryKeys,
@@ -186,7 +192,10 @@ public class Schema implements Serializable {
@Override
public int hashCode() {
- return Objects.hash(columns, primaryKeys, partitionKeys, options, comment);
+ if (cachedHashCode == 0) {
+ cachedHashCode = Objects.hash(columns, primaryKeys, partitionKeys, options, comment);
+ }
+ return cachedHashCode;
}
// -----------------------------------------------------------------------------------
diff --git a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/source/DataSource.java b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/source/DataSource.java
index 096648f8b..7766e8207 100644
--- a/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/source/DataSource.java
+++ b/flink-cdc-common/src/main/java/org/apache/flink/cdc/common/source/DataSource.java
@@ -17,6 +17,7 @@
package org.apache.flink.cdc.common.source;
+import org.apache.flink.cdc.common.annotation.Experimental;
import org.apache.flink.cdc.common.annotation.PublicEvolving;
/**
@@ -36,4 +37,18 @@ public interface DataSource {
default SupportedMetadataColumn[] supportedMetadataColumns() {
return new SupportedMetadataColumn[0];
}
+
+ /**
+ * Indicating if this source may generate metadata events (SchemaChangeEvents) in parallel for
+ * each table. If returns {@code false}, you'll get a regular operator topology that is
+ * compatible with single-incremented sources like MySQL. Returns {@code true} for sources that
+ * does not maintain a globally sequential schema change events stream, like MongoDB or Kafka.
+ * <br>
+ * Note that new topology still an experimental feature. Return {@code false} by default to
+ * avoid unexpected behaviors.
+ */
+ @Experimental
+ default boolean isParallelMetadataSource() {
+ return false;
+ }
}
diff --git a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java
index 4578a49fd..533868c56 100644
--- a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java
+++ b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposer.java
@@ -33,6 +33,7 @@ import org.apache.flink.cdc.composer.flink.translator.DataSourceTranslator;
import org.apache.flink.cdc.composer.flink.translator.PartitioningTranslator;
import org.apache.flink.cdc.composer.flink.translator.SchemaOperatorTranslator;
import org.apache.flink.cdc.composer.flink.translator.TransformTranslator;
+import org.apache.flink.cdc.runtime.partitioning.PartitioningEvent;
import org.apache.flink.cdc.runtime.serializer.event.EventSerializer;
import org.apache.flink.configuration.DeploymentOptions;
import org.apache.flink.streaming.api.datastream.DataStream;
@@ -95,79 +96,116 @@ public class FlinkPipelineComposer implements PipelineComposer {
int parallelism = pipelineDefConfig.get(PipelineOptions.PIPELINE_PARALLELISM);
env.getConfig().setParallelism(parallelism);
+ translate(env, pipelineDef);
+
+ // Add framework JARs
+ addFrameworkJars();
+
+ return new FlinkPipelineExecution(
+ env, pipelineDefConfig.get(PipelineOptions.PIPELINE_NAME), isBlocking);
+ }
+
+ private void translate(StreamExecutionEnvironment env, PipelineDef pipelineDef) {
+ Configuration pipelineDefConfig = pipelineDef.getConfig();
+ int parallelism = pipelineDefConfig.get(PipelineOptions.PIPELINE_PARALLELISM);
SchemaChangeBehavior schemaChangeBehavior =
pipelineDefConfig.get(PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR);
- // Build Source Operator
+ // Initialize translators
DataSourceTranslator sourceTranslator = new DataSourceTranslator();
- DataSource dataSource =
- sourceTranslator.createDataSource(pipelineDef.getSource(), env, pipelineDefConfig);
- DataStream<Event> stream =
- sourceTranslator.translate(pipelineDef.getSource(), env, parallelism, dataSource);
-
- // Build PreTransformOperator for processing Schema Event
TransformTranslator transformTranslator = new TransformTranslator();
- stream =
- transformTranslator.translatePreTransform(
- stream,
- pipelineDef.getTransforms(),
- pipelineDef.getUdfs(),
- pipelineDef.getModels(),
- dataSource.supportedMetadataColumns());
-
- // Schema operator
+ PartitioningTranslator partitioningTranslator = new PartitioningTranslator();
SchemaOperatorTranslator schemaOperatorTranslator =
new SchemaOperatorTranslator(
schemaChangeBehavior,
pipelineDefConfig.get(PipelineOptions.PIPELINE_SCHEMA_OPERATOR_UID),
pipelineDefConfig.get(PipelineOptions.PIPELINE_SCHEMA_OPERATOR_RPC_TIMEOUT),
pipelineDefConfig.get(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE));
+ DataSinkTranslator sinkTranslator = new DataSinkTranslator();
+
+ // And required constructors
OperatorIDGenerator schemaOperatorIDGenerator =
new OperatorIDGenerator(schemaOperatorTranslator.getSchemaOperatorUid());
+ DataSource dataSource =
+ sourceTranslator.createDataSource(pipelineDef.getSource(), pipelineDefConfig, env);
+ DataSink dataSink =
+ sinkTranslator.createDataSink(pipelineDef.getSink(), pipelineDefConfig, env);
- // Build PostTransformOperator for processing Data Event
+ boolean isParallelMetadataSource = dataSource.isParallelMetadataSource();
+
+ // O ---> Source
+ DataStream<Event> stream =
+ sourceTranslator.translate(pipelineDef.getSource(), dataSource, env, parallelism);
+
+ // Source ---> PreTransform
stream =
- transformTranslator.translatePostTransform(
+ transformTranslator.translatePreTransform(
stream,
pipelineDef.getTransforms(),
- pipelineDef.getConfig().get(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE),
pipelineDef.getUdfs(),
pipelineDef.getModels(),
- dataSource.supportedMetadataColumns());
-
- // Build DataSink in advance as schema operator requires MetadataApplier
- DataSinkTranslator sinkTranslator = new DataSinkTranslator();
- DataSink dataSink =
- sinkTranslator.createDataSink(pipelineDef.getSink(), pipelineDefConfig, env);
+ dataSource.supportedMetadataColumns(),
+ isParallelMetadataSource);
+ // PreTransform ---> PostTransform
stream =
- schemaOperatorTranslator.translate(
+ transformTranslator.translatePostTransform(
stream,
- parallelism,
- dataSink.getMetadataApplier()
- .setAcceptedSchemaEvolutionTypes(
- pipelineDef.getSink().getIncludedSchemaEvolutionTypes()),
- pipelineDef.getRoute());
+ pipelineDef.getTransforms(),
+ pipelineDef.getConfig().get(PipelineOptions.PIPELINE_LOCAL_TIME_ZONE),
+ pipelineDef.getUdfs(),
+ pipelineDef.getModels(),
+ dataSource.supportedMetadataColumns());
- // Build Partitioner used to shuffle Event
- PartitioningTranslator partitioningTranslator = new PartitioningTranslator();
- stream =
- partitioningTranslator.translate(
- stream,
- parallelism,
- parallelism,
- schemaOperatorIDGenerator.generate(),
- dataSink.getDataChangeEventHashFunctionProvider(parallelism));
+ if (isParallelMetadataSource) {
+ // Translate a distributed topology for sources with distributed tables
+ // PostTransform -> Partitioning
+ DataStream<PartitioningEvent> partitionedStream =
+ partitioningTranslator.translateDistributed(
+ stream,
+ parallelism,
+ parallelism,
+ dataSink.getDataChangeEventHashFunctionProvider(parallelism));
+
+ // Partitioning -> Schema Operator
+ stream =
+ schemaOperatorTranslator.translateDistributed(
+ partitionedStream,
+ parallelism,
+ dataSink.getMetadataApplier()
+ .setAcceptedSchemaEvolutionTypes(
+ pipelineDef
+ .getSink()
+ .getIncludedSchemaEvolutionTypes()),
+ pipelineDef.getRoute());
+
+ } else {
+ // Translate a regular topology for sources without distributed tables
+ // PostTransform ---> Schema Operator
+ stream =
+ schemaOperatorTranslator.translateRegular(
+ stream,
+ parallelism,
+ dataSink.getMetadataApplier()
+ .setAcceptedSchemaEvolutionTypes(
+ pipelineDef
+ .getSink()
+ .getIncludedSchemaEvolutionTypes()),
+ pipelineDef.getRoute());
+
+ // Schema Operator ---(shuffled)---> Partitioning
+ stream =
+ partitioningTranslator.translateRegular(
+ stream,
+ parallelism,
+ parallelism,
+ schemaOperatorIDGenerator.generate(),
+ dataSink.getDataChangeEventHashFunctionProvider(parallelism));
+ }
- // Build Sink Operator
+ // Schema Operator -> Sink -> X
sinkTranslator.translate(
pipelineDef.getSink(), stream, dataSink, schemaOperatorIDGenerator.generate());
-
- // Add framework JARs
- addFrameworkJars();
-
- return new FlinkPipelineExecution(
- env, pipelineDefConfig.get(PipelineOptions.PIPELINE_NAME), isBlocking);
}
private void addFrameworkJars() {
diff --git a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSourceTranslator.java b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSourceTranslator.java
index 4323871ff..62a8128b0 100644
--- a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSourceTranslator.java
+++ b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/DataSourceTranslator.java
@@ -39,29 +39,11 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@Internal
public class DataSourceTranslator {
- public DataSource createDataSource(
- SourceDef sourceDef, StreamExecutionEnvironment env, Configuration pipelineConfig) {
- // Search the data source factory
- DataSourceFactory sourceFactory =
- FactoryDiscoveryUtils.getFactoryByIdentifier(
- sourceDef.getType(), DataSourceFactory.class);
- // Add source JAR to environment
- FactoryDiscoveryUtils.getJarPathByIdentifier(sourceFactory)
- .ifPresent(jar -> FlinkEnvironmentUtils.addJar(env, jar));
- DataSource dataSource =
- sourceFactory.createDataSource(
- new FactoryHelper.DefaultContext(
- sourceDef.getConfig(),
- pipelineConfig,
- Thread.currentThread().getContextClassLoader()));
- return dataSource;
- }
-
public DataStreamSource<Event> translate(
SourceDef sourceDef,
+ DataSource dataSource,
StreamExecutionEnvironment env,
- int sourceParallelism,
- DataSource dataSource) {
+ int sourceParallelism) {
// Get source provider
EventSourceProvider eventSourceProvider = dataSource.getEventSourceProvider();
if (eventSourceProvider instanceof FlinkSourceProvider) {
@@ -93,6 +75,22 @@ public class DataSourceTranslator {
}
}
+ public DataSource createDataSource(
+ SourceDef sourceDef, Configuration pipelineConfig, StreamExecutionEnvironment env) {
+ // Search the data source factory
+ DataSourceFactory sourceFactory =
+ FactoryDiscoveryUtils.getFactoryByIdentifier(
+ sourceDef.getType(), DataSourceFactory.class);
+ // Add source JAR to environment
+ FactoryDiscoveryUtils.getJarPathByIdentifier(sourceFactory)
+ .ifPresent(jar -> FlinkEnvironmentUtils.addJar(env, jar));
+ return sourceFactory.createDataSource(
+ new FactoryHelper.DefaultContext(
+ sourceDef.getConfig(),
+ pipelineConfig,
+ Thread.currentThread().getContextClassLoader()));
+ }
+
private String generateDefaultSourceName(SourceDef sourceDef) {
return String.format("Flink CDC Event Source: %s", sourceDef.getType());
}
diff --git a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/PartitioningTranslator.java b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/PartitioningTranslator.java
index b4e0b34b8..b9e87fd46 100644
--- a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/PartitioningTranslator.java
+++ b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/PartitioningTranslator.java
@@ -21,23 +21,26 @@ import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.event.DataChangeEvent;
import org.apache.flink.cdc.common.event.Event;
import org.apache.flink.cdc.common.function.HashFunctionProvider;
+import org.apache.flink.cdc.runtime.partitioning.DistributedPrePartitionOperator;
import org.apache.flink.cdc.runtime.partitioning.EventPartitioner;
+import org.apache.flink.cdc.runtime.partitioning.PartitioningEvent;
import org.apache.flink.cdc.runtime.partitioning.PartitioningEventKeySelector;
import org.apache.flink.cdc.runtime.partitioning.PostPartitionProcessor;
-import org.apache.flink.cdc.runtime.partitioning.PrePartitionOperator;
+import org.apache.flink.cdc.runtime.partitioning.RegularPrePartitionOperator;
import org.apache.flink.cdc.runtime.typeutils.EventTypeInfo;
import org.apache.flink.cdc.runtime.typeutils.PartitioningEventTypeInfo;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.streaming.api.datastream.DataStream;
/**
- * Translator used to build {@link PrePartitionOperator}, {@link EventPartitioner} and {@link
- * PostPartitionProcessor} which are responsible for events partition.
+ * Translator used to build {@link RegularPrePartitionOperator} or {@link
+ * DistributedPrePartitionOperator}, {@link EventPartitioner} and {@link PostPartitionProcessor}
+ * that are responsible for events partition.
*/
@Internal
public class PartitioningTranslator {
- public DataStream<Event> translate(
+ public DataStream<Event> translateRegular(
DataStream<Event> input,
int upstreamParallelism,
int downstreamParallelism,
@@ -46,11 +49,25 @@ public class PartitioningTranslator {
return input.transform(
"PrePartition",
new PartitioningEventTypeInfo(),
- new PrePartitionOperator(
+ new RegularPrePartitionOperator(
schemaOperatorID, downstreamParallelism, hashFunctionProvider))
.setParallelism(upstreamParallelism)
.partitionCustom(new EventPartitioner(), new PartitioningEventKeySelector())
.map(new PostPartitionProcessor(), new EventTypeInfo())
.name("PostPartition");
}
+
+ public DataStream<PartitioningEvent> translateDistributed(
+ DataStream<Event> input,
+ int upstreamParallelism,
+ int downstreamParallelism,
+ HashFunctionProvider<DataChangeEvent> hashFunctionProvider) {
+ return input.transform(
+ "Partitioning",
+ new PartitioningEventTypeInfo(),
+ new DistributedPrePartitionOperator(
+ downstreamParallelism, hashFunctionProvider))
+ .setParallelism(upstreamParallelism)
+ .partitionCustom(new EventPartitioner(), new PartitioningEventKeySelector());
+ }
}
diff --git a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/SchemaOperatorTranslator.java b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/SchemaOperatorTranslator.java
index c5cadcd1e..47164578d 100644
--- a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/SchemaOperatorTranslator.java
+++ b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/SchemaOperatorTranslator.java
@@ -22,9 +22,11 @@ import org.apache.flink.cdc.common.event.Event;
import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
import org.apache.flink.cdc.common.route.RouteRule;
import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.common.utils.Preconditions;
import org.apache.flink.cdc.composer.definition.RouteDef;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperatorFactory;
+import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
+import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperatorFactory;
+import org.apache.flink.cdc.runtime.partitioning.PartitioningEvent;
import org.apache.flink.cdc.runtime.typeutils.EventTypeInfo;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
@@ -52,12 +54,21 @@ public class SchemaOperatorTranslator {
this.timezone = timezone;
}
- public DataStream<Event> translate(
+ public DataStream<Event> translateRegular(
DataStream<Event> input,
int parallelism,
MetadataApplier metadataApplier,
List<RouteDef> routes) {
- return addSchemaOperator(
+ return addRegularSchemaOperator(
+ input, parallelism, metadataApplier, routes, schemaChangeBehavior, timezone);
+ }
+
+ public DataStream<Event> translateDistributed(
+ DataStream<PartitioningEvent> input,
+ int parallelism,
+ MetadataApplier metadataApplier,
+ List<RouteDef> routes) {
+ return addDistributedSchemaOperator(
input, parallelism, metadataApplier, routes, schemaChangeBehavior, timezone);
}
@@ -65,7 +76,7 @@ public class SchemaOperatorTranslator {
return schemaOperatorUid;
}
- private DataStream<Event> addSchemaOperator(
+ private DataStream<Event> addRegularSchemaOperator(
DataStream<Event> input,
int parallelism,
MetadataApplier metadataApplier,
@@ -93,4 +104,44 @@ public class SchemaOperatorTranslator {
stream.uid(schemaOperatorUid).setParallelism(parallelism);
return stream;
}
+
+ private DataStream<Event> addDistributedSchemaOperator(
+ DataStream<PartitioningEvent> input,
+ int parallelism,
+ MetadataApplier metadataApplier,
+ List<RouteDef> routes,
+ SchemaChangeBehavior schemaChangeBehavior,
+ String timezone) {
+ Preconditions.checkArgument(
+ schemaChangeBehavior == SchemaChangeBehavior.LENIENT
+ || schemaChangeBehavior == SchemaChangeBehavior.IGNORE
+ || schemaChangeBehavior == SchemaChangeBehavior.EXCEPTION,
+ "Schema change behavior %s is not supported because you're trying to compose a "
+ + "pipeline with distributed topology, where data records from different partitions needs to "
+ + "be combined together.\n"
+ + "Use `LENIENT` mode to evolve downstream schema while keep all upstream data fields intact.\n"
+ + "Use `IGNORE` to get a static schema view and ignore any upstream schema changes.\n"
+ + "Use `EXCEPTION` to report error immediately as upstream schema changes are unacceptable.",
+ schemaChangeBehavior);
+ List<RouteRule> routingRules = new ArrayList<>();
+ for (RouteDef route : routes) {
+ routingRules.add(
+ new RouteRule(
+ route.getSourceTable(),
+ route.getSinkTable(),
+ route.getReplaceSymbol().orElse(null)));
+ }
+ return input.transform(
+ "SchemaMapper",
+ new EventTypeInfo(),
+ new org.apache.flink.cdc.runtime.operators.schema.distributed
+ .SchemaOperatorFactory(
+ metadataApplier,
+ routingRules,
+ rpcTimeOut,
+ schemaChangeBehavior,
+ timezone))
+ .uid(schemaOperatorUid)
+ .setParallelism(parallelism);
+ }
}
diff --git a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/TransformTranslator.java b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/TransformTranslator.java
index bd2fadbc2..c7fb15541 100644
--- a/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/TransformTranslator.java
+++ b/flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/flink/translator/TransformTranslator.java
@@ -48,7 +48,8 @@ public class TransformTranslator {
List<TransformDef> transforms,
List<UdfDef> udfFunctions,
List<ModelDef> models,
- SupportedMetadataColumn[] supportedMetadataColumns) {
+ SupportedMetadataColumn[] supportedMetadataColumns,
+ boolean canContainDistributedTables) {
if (transforms.isEmpty()) {
return input;
}
@@ -67,10 +68,15 @@ public class TransformTranslator {
supportedMetadataColumns);
}
- preTransformFunctionBuilder.addUdfFunctions(
- udfFunctions.stream().map(this::udfDefToUDFTuple).collect(Collectors.toList()));
- preTransformFunctionBuilder.addUdfFunctions(
- models.stream().map(this::modelToUDFTuple).collect(Collectors.toList()));
+ preTransformFunctionBuilder
+ .addUdfFunctions(
+ udfFunctions.stream()
+ .map(this::udfDefToUDFTuple)
+ .collect(Collectors.toList()))
+ .addUdfFunctions(
+ models.stream().map(this::modelToUDFTuple).collect(Collectors.toList()))
+ .canContainDistributedTables(canContainDistributedTables);
+
return input.transform(
"Transform:Schema", new EventTypeInfo(), preTransformFunctionBuilder.build());
}
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkParallelizedPipelineITCase.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkParallelizedPipelineITCase.java
new file mode 100644
index 000000000..1920227da
--- /dev/null
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkParallelizedPipelineITCase.java
@@ -0,0 +1,1011 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.flink;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.exceptions.SchemaEvolveException;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.RouteDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.testsource.factory.DistributedDataSourceFactory;
+import org.apache.flink.cdc.composer.testsource.source.DistributedSourceOptions;
+import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSink;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSinkOptions;
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration;
+import org.apache.flink.test.junit5.MiniClusterExtension;
+
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.apache.flink.configuration.CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL;
+
+/** Integration test for {@link FlinkPipelineComposer} in parallelized schema evolution cases. */
+@Timeout(value = 600, unit = java.util.concurrent.TimeUnit.SECONDS)
+class FlinkParallelizedPipelineITCase {
+
+ private static final int MAX_PARALLELISM = 4;
+ private static final int UPSTREAM_TABLE_COUNT = 4;
+ private static final List<RouteDef> ROUTING_RULES;
+
+ static {
+ ROUTING_RULES =
+ IntStream.range(0, UPSTREAM_TABLE_COUNT)
+ .mapToObj(
+ i ->
+ new RouteDef(
+ "default_namespace_subtask_\\d.default_database.table_"
+ + i,
+ "default_namespace.default_database.table_" + i,
+ null,
+ null))
+ .collect(Collectors.toList());
+ }
+
+ private static final List<String> SINGLE_PARALLELISM_REGULAR = loadReferenceFile("regular.txt");
+
+ private static final List<String> SINGLE_PARALLELISM_DISTRIBUTED =
+ loadReferenceFile("distributed.txt");
+
+ private static final List<String> SINGLE_PARALLELISM_DISTRIBUTED_IGNORE =
+ loadReferenceFile("distributed-ignore.txt");
+
+ private static List<String> loadReferenceFile(String name) {
+ final String refFile = String.format("ref-output/%s", name);
+ final URL refFileUrl =
+ FlinkParallelizedPipelineITCase.class.getClassLoader().getResource(refFile);
+ Assertions.assertThat(refFileUrl).isNotNull();
+ try {
+ return Files.readAllLines(Paths.get(refFileUrl.toURI())).stream()
+ .map(String::trim)
+ .collect(Collectors.toList());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ // Always use parent-first classloader for CDC classes.
+ // The reason is that ValuesDatabase uses static field for holding data, we need to make sure
+ // the class is loaded by AppClassloader so that we can verify data in the test case.
+ private static final org.apache.flink.configuration.Configuration MINI_CLUSTER_CONFIG =
+ new org.apache.flink.configuration.Configuration();
+
+ static {
+ MINI_CLUSTER_CONFIG.set(
+ ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
+ Collections.singletonList("org.apache.flink.cdc"));
+ }
+
+ /**
+ * Use {@link MiniClusterExtension} to reduce the overhead of restarting the MiniCluster for
+ * every test case.
+ */
+ @RegisterExtension
+ static final MiniClusterExtension MINI_CLUSTER_RESOURCE =
+ new MiniClusterExtension(
+ new MiniClusterResourceConfiguration.Builder()
+ .setNumberTaskManagers(1)
+ .setNumberSlotsPerTaskManager(MAX_PARALLELISM)
+ .setConfiguration(MINI_CLUSTER_CONFIG)
+ .build());
+
+ private final PrintStream standardOut = System.out;
+ private final ByteArrayOutputStream outCaptor = new ByteArrayOutputStream();
+
+ @BeforeEach
+ void init() {
+ // Take over STDOUT as we need to check the output of values sink
+ System.setOut(new PrintStream(outCaptor));
+ // Initialize in-memory database
+ ValuesDatabase.clear();
+ }
+
+ @AfterEach
+ void cleanup() {
+ System.setOut(standardOut);
+ System.out.println(
+ "NOTICE: This is a semi-fuzzy test. Please also check if value sink prints expected events:");
+ System.out.println("================================");
+ System.out.print(outCaptor);
+ System.out.println("================================");
+ outCaptor.reset();
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testDistributedTablesSourceInSingleParallelismWithLenientBehavior(
+ ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runPipelineJob(sinkApi, 1, SourceTraits.DISTRIBUTED, SchemaChangeBehavior.LENIENT);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace", "default_database", "table_" + idx));
+
+ // The order of result is determined.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactly(
+ Column.physicalColumn("id", DataTypes.STRING()),
+ Column.physicalColumn("col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_0_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_0_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_0_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn("col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_0_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_0_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_0_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_0_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_0_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn("col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_0_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn("col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_0_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn("col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_0_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_0_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "col_localzonedtimestamptype", DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)));
+ }
+
+ String outputStr = outCaptor.toString();
+
+ // In single-parallelism mode, orderliness of output is determined.
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ Assertions.assertThat(dataLines).containsExactlyElementsOf(SINGLE_PARALLELISM_DISTRIBUTED);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testDistributedTablesSourceInMultipleParallelismWithLenientBehavior(
+ ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runPipelineJob(
+ sinkApi, MAX_PARALLELISM, SourceTraits.DISTRIBUTED, SchemaChangeBehavior.LENIENT);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace", "default_database", "table_" + idx));
+
+ // The order of result schema is uncertain.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactlyInAnyOrder(
+ Column.physicalColumn("id", DataTypes.STRING()),
+ Column.physicalColumn("col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_0_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_1_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_2_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_3_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_0_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_1_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_2_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_3_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_0_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_1_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_2_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_3_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn("col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_0_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_1_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_2_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_3_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_0_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_1_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_2_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_3_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_0_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_1_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_2_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_3_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn("col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_0_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_1_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_2_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_3_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_0_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_1_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_2_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_3_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_0_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_1_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_2_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_3_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_1_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_2_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_3_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn("col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_0_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_1_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_2_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_3_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_1_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_2_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_3_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn("col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_0_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_1_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_2_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_3_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_0_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_1_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_2_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_3_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_1_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_2_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_3_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "col_localzonedtimestamptype", DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_1_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_2_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_3_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)));
+ }
+
+ String outputStr = outCaptor.toString();
+ IntStream.range(0, MAX_PARALLELISM)
+ .forEach(
+ subTaskId ->
+ IntStream.range(0, 168)
+ .forEach(
+ seqNum ->
+ Assertions.assertThat(outputStr)
+ .contains(
+ String.format(
+ "__$%d$%d$__",
+ subTaskId,
+ seqNum))));
+
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ String[] expectedTokens = {
+ "true",
+ "17",
+ "34",
+ "68",
+ "136",
+ "272.0",
+ "544.0",
+ "1088.00000000000",
+ "Alice",
+ "Bob",
+ "Q2ljYWRh",
+ "RGVycmlkYQ==",
+ "64800000",
+ "2019-12-31T18:00",
+ "2020-07-17T18:00",
+ "1970-01-05T05:20:00.000123456+08:00"
+ };
+
+ Stream.of(expectedTokens)
+ .forEach(
+ token ->
+ Assertions.assertThat(
+ Stream.of(dataLines)
+ .filter(line -> line.contains(token))
+ .count())
+ .as("Checking presence of %s", token)
+ .isGreaterThanOrEqualTo(
+ UPSTREAM_TABLE_COUNT * MAX_PARALLELISM));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testDistributedTablesSourceInSingleParallelismWithIgnoreBehavior(
+ ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runPipelineJob(sinkApi, 1, SourceTraits.DISTRIBUTED, SchemaChangeBehavior.IGNORE);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace", "default_database", "table_" + idx));
+
+ // The order of result is determined.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactly(Column.physicalColumn("id", DataTypes.STRING()));
+ }
+
+ String outputStr = outCaptor.toString();
+
+ // We will have initial `id` column only since no others will be
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ Assertions.assertThat(dataLines)
+ .containsExactlyElementsOf(SINGLE_PARALLELISM_DISTRIBUTED_IGNORE);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testDistributedTablesSourceInMultipleParallelismWithIgnoreBehavior(
+ ValuesDataSink.SinkApi sinkApi) throws Exception {
+ runPipelineJob(
+ sinkApi, MAX_PARALLELISM, SourceTraits.DISTRIBUTED, SchemaChangeBehavior.IGNORE);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace", "default_database", "table_" + idx));
+
+ // The order of result schema is uncertain.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactly(Column.physicalColumn("id", DataTypes.STRING()));
+ }
+
+ String outputStr = outCaptor.toString();
+ IntStream.range(0, MAX_PARALLELISM)
+ .forEach(
+ subTaskId ->
+ IntStream.range(0, 168)
+ .forEach(
+ seqNum ->
+ Assertions.assertThat(outputStr)
+ .contains(
+ String.format(
+ "__$%d$%d$__",
+ subTaskId,
+ seqNum))));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testDistributedTablesSourceInSingleParallelismWithExceptionBehavior(
+ ValuesDataSink.SinkApi sinkApi) {
+ Assertions.assertThatThrownBy(
+ () ->
+ runPipelineJob(
+ sinkApi,
+ 1,
+ SourceTraits.DISTRIBUTED,
+ SchemaChangeBehavior.EXCEPTION))
+ .rootCause()
+ .isExactlyInstanceOf(SchemaEvolveException.class)
+ .extracting("exceptionMessage")
+ .isEqualTo(
+ "Unexpected schema change events occurred in EXCEPTION mode. Job will fail now.");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testDistributedTablesSourceInMultipleParallelismWithExceptionBehavior(
+ ValuesDataSink.SinkApi sinkApi) {
+ Assertions.assertThatThrownBy(
+ () ->
+ runPipelineJob(
+ sinkApi,
+ MAX_PARALLELISM,
+ SourceTraits.DISTRIBUTED,
+ SchemaChangeBehavior.EXCEPTION))
+ .rootCause()
+ .isExactlyInstanceOf(SchemaEvolveException.class)
+ .extracting("exceptionMessage")
+ .isEqualTo(
+ "Unexpected schema change events occurred in EXCEPTION mode. Job will fail now.");
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRegularTablesSourceInSingleParallelism(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ runPipelineJob(sinkApi, 1, SourceTraits.REGULAR, SchemaChangeBehavior.LENIENT);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace_subtask_0",
+ "default_database",
+ "table_" + idx));
+
+ // The order of result is determined.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactly(
+ Column.physicalColumn("id", DataTypes.STRING()),
+ Column.physicalColumn("col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_0_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_0_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_0_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn("col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_0_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_0_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_0_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_0_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_0_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn("col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_0_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn("col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_0_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn("col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_0_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_0_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "col_localzonedtimestamptype", DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)));
+ }
+
+ String outputStr = outCaptor.toString();
+
+ // In single-parallelism mode, orderliness of output is determined.
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ Assertions.assertThat(dataLines).containsExactlyElementsOf(SINGLE_PARALLELISM_REGULAR);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRegularTablesSourceInMultipleParallelism(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ runPipelineJob(
+ sinkApi, MAX_PARALLELISM, SourceTraits.REGULAR, SchemaChangeBehavior.LENIENT);
+
+ // Validate generated downstream schema
+
+ for (int taskIdx = 0; taskIdx < MAX_PARALLELISM; taskIdx++) {
+ for (int tableIdx = 0; tableIdx < UPSTREAM_TABLE_COUNT; tableIdx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace_subtask_" + taskIdx,
+ "default_database",
+ "table_" + tableIdx));
+
+ // The order of result schema is uncertain.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactlyInAnyOrder(
+ Column.physicalColumn("id", DataTypes.STRING()),
+ Column.physicalColumn("col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_booleantype",
+ DataTypes.BOOLEAN()),
+ Column.physicalColumn("col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_tinyinttype",
+ DataTypes.TINYINT()),
+ Column.physicalColumn("col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_smallinttype",
+ DataTypes.SMALLINT()),
+ Column.physicalColumn("col_inttype", DataTypes.INT()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_biginttype",
+ DataTypes.BIGINT()),
+ Column.physicalColumn("col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_decimaltype",
+ DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn("col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_doubletype",
+ DataTypes.DOUBLE()),
+ Column.physicalColumn("col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_varchartype",
+ DataTypes.VARCHAR(17)),
+ Column.physicalColumn("col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_binarytype",
+ DataTypes.BINARY(17)),
+ Column.physicalColumn("col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_varbinarytype",
+ DataTypes.VARBINARY(17)),
+ Column.physicalColumn("col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_timestamptype",
+ DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_zonedtimestamptype",
+ DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "col_localzonedtimestamptype", DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_" + taskIdx + "_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)));
+ }
+ }
+
+ String outputStr = outCaptor.toString();
+ IntStream.range(0, MAX_PARALLELISM)
+ .forEach(
+ subTaskId ->
+ IntStream.range(0, 168)
+ .forEach(
+ seqNum ->
+ Assertions.assertThat(outputStr)
+ .contains(
+ String.format(
+ "__$%d$%d$__",
+ subTaskId,
+ seqNum))));
+
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ String[] expectedTokens = {
+ "true",
+ "17",
+ "34",
+ "68",
+ "136",
+ "272.0",
+ "544.0",
+ "1088.00000000000",
+ "Alice",
+ "Bob",
+ "Q2ljYWRh",
+ "RGVycmlkYQ==",
+ "64800000",
+ "2019-12-31T18:00",
+ "2020-07-17T18:00",
+ "1970-01-05T05:20:00.000123456+08:00"
+ };
+
+ Stream.of(expectedTokens)
+ .forEach(
+ token ->
+ Assertions.assertThat(
+ Stream.of(dataLines)
+ .filter(line -> line.contains(token))
+ .count())
+ .as("Checking presence of %s", token)
+ .isGreaterThanOrEqualTo(
+ UPSTREAM_TABLE_COUNT * MAX_PARALLELISM));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRegularTablesSourceMergedInSingleParallelism(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ runPipelineJob(sinkApi, 1, SourceTraits.MERGING, SchemaChangeBehavior.LENIENT);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace", "default_database", "table_" + idx));
+
+ // The order of result is determined.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactly(
+ Column.physicalColumn("id", DataTypes.STRING()),
+ Column.physicalColumn("col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_0_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_0_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_0_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn("col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_0_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_0_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_0_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_0_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_0_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn("col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_0_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn("col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_0_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn("col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_0_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_0_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "col_localzonedtimestamptype", DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)));
+ }
+
+ String outputStr = outCaptor.toString();
+
+ // In single-parallelism mode, orderliness of output is determined.
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ Assertions.assertThat(dataLines).containsExactlyElementsOf(SINGLE_PARALLELISM_DISTRIBUTED);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ @Disabled(
+ "Currently, it's not safe to merge tables with hetero schema from different partitions.")
+ void testRegularTablesSourceMergedInMultipleParallelism(ValuesDataSink.SinkApi sinkApi)
+ throws Exception {
+ runPipelineJob(
+ sinkApi, MAX_PARALLELISM, SourceTraits.MERGING, SchemaChangeBehavior.LENIENT);
+
+ // Validate generated downstream schema
+ for (int idx = 0; idx < UPSTREAM_TABLE_COUNT; idx++) {
+ Schema schema =
+ ValuesDatabase.getTableSchema(
+ TableId.tableId(
+ "default_namespace", "default_database", "table_" + idx));
+
+ // The order of result schema is uncertain.
+ Assertions.assertThat(schema.getColumns())
+ .containsExactlyInAnyOrder(
+ Column.physicalColumn("id", DataTypes.STRING()),
+ Column.physicalColumn("col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_0_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_1_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_2_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("subtask_3_col_booleantype", DataTypes.BOOLEAN()),
+ Column.physicalColumn("col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_0_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_1_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_2_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("subtask_3_col_tinyinttype", DataTypes.TINYINT()),
+ Column.physicalColumn("col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_0_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_1_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_2_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn(
+ "subtask_3_col_smallinttype", DataTypes.SMALLINT()),
+ Column.physicalColumn("col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_0_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_1_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_2_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("subtask_3_col_inttype", DataTypes.INT()),
+ Column.physicalColumn("col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_0_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_1_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_2_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("subtask_3_col_biginttype", DataTypes.BIGINT()),
+ Column.physicalColumn("col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_0_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_1_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_2_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn(
+ "subtask_3_col_decimaltype", DataTypes.DECIMAL(17, 11)),
+ Column.physicalColumn("col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_0_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_1_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_2_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("subtask_3_col_floattype", DataTypes.FLOAT()),
+ Column.physicalColumn("col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_0_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_1_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_2_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("subtask_3_col_doubletype", DataTypes.DOUBLE()),
+ Column.physicalColumn("col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_0_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_1_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_2_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("subtask_3_col_chartype", DataTypes.CHAR(17)),
+ Column.physicalColumn("col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_1_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_2_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn(
+ "subtask_3_col_varchartype", DataTypes.VARCHAR(17)),
+ Column.physicalColumn("col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_0_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_1_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_2_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("subtask_3_col_binarytype", DataTypes.BINARY(17)),
+ Column.physicalColumn("col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_0_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_1_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_2_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn(
+ "subtask_3_col_varbinarytype", DataTypes.VARBINARY(17)),
+ Column.physicalColumn("col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_0_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_1_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_2_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("subtask_3_col_timetype", DataTypes.TIME(9)),
+ Column.physicalColumn("col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_0_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_1_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_2_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "subtask_3_col_timestamptype", DataTypes.TIMESTAMP(9)),
+ Column.physicalColumn(
+ "col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_1_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_2_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "subtask_3_col_zonedtimestamptype", DataTypes.TIMESTAMP_TZ(9)),
+ Column.physicalColumn(
+ "col_localzonedtimestamptype", DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_0_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_1_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_2_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)),
+ Column.physicalColumn(
+ "subtask_3_col_localzonedtimestamptype",
+ DataTypes.TIMESTAMP_LTZ(9)));
+ }
+
+ String outputStr = outCaptor.toString();
+ IntStream.range(0, MAX_PARALLELISM)
+ .forEach(
+ subTaskId ->
+ IntStream.range(0, 168)
+ .forEach(
+ seqNum ->
+ Assertions.assertThat(outputStr)
+ .contains(
+ String.format(
+ "__$%d$%d$__",
+ subTaskId,
+ seqNum))));
+
+ String[] dataLines = outputStr.split(System.lineSeparator());
+ String[] expectedTokens = {
+ "true",
+ "17",
+ "34",
+ "68",
+ "136",
+ "272.0",
+ "544.0",
+ "1088.00000000000",
+ "Alice",
+ "Bob",
+ "Q2ljYWRh",
+ "RGVycmlkYQ==",
+ "64800000",
+ "2019-12-31T18:00",
+ "2020-07-17T18:00",
+ "1970-01-05T05:20:00.000123456+08:00"
+ };
+
+ Stream.of(expectedTokens)
+ .forEach(
+ token ->
+ Assertions.assertThat(
+ Stream.of(dataLines)
+ .filter(line -> line.contains(token))
+ .count())
+ .as("Checking presence of %s", token)
+ .isGreaterThanOrEqualTo(
+ UPSTREAM_TABLE_COUNT * MAX_PARALLELISM));
+ }
+
+ private void runPipelineJob(
+ ValuesDataSink.SinkApi sinkApi,
+ int parallelism,
+ SourceTraits traits,
+ SchemaChangeBehavior exception)
+ throws Exception {
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup value source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(
+ DistributedSourceOptions.DISTRIBUTED_TABLES, traits == SourceTraits.DISTRIBUTED);
+ sourceConfig.set(DistributedSourceOptions.TABLE_COUNT, UPSTREAM_TABLE_COUNT);
+ SourceDef sourceDef =
+ new SourceDef(
+ DistributedDataSourceFactory.IDENTIFIER,
+ "Distributed Source",
+ sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ sinkConfig.set(ValuesDataSinkOptions.SINK_API, sinkApi);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, parallelism);
+ pipelineConfig.set(PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, exception);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ traits == SourceTraits.MERGING ? ROUTING_RULES : Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+
+ execution.execute();
+ }
+
+ /**
+ *
+ *
+ * <ul>
+ * <li>DISTRIBUTED means data from one table might evolve independently in multiple
+ * partitions.
+ * <li>REGULAR means data from each table will not be presented in multiple partitions.
+ * <li>MERGING means data from multiple tables are forcefully merged into one partition.
+ * </ul>
+ */
+ private enum SourceTraits {
+ DISTRIBUTED,
+ REGULAR,
+ MERGING
+ }
+}
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java
index c8877ef6e..380baaa5d 100644
--- a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerITCase.java
@@ -817,7 +817,7 @@ class FlinkPipelineComposerITCase {
Schema.newBuilder()
.physicalColumn("id", DataTypes.BIGINT())
.physicalColumn("name", DataTypes.STRING())
- .physicalColumn("age", DataTypes.BIGINT())
+ .physicalColumn("age", DataTypes.INT())
.physicalColumn("description", DataTypes.STRING())
.physicalColumn("last_name", DataTypes.STRING())
.physicalColumn("gender", DataTypes.STRING())
@@ -830,13 +830,13 @@ class FlinkPipelineComposerITCase {
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
- "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=LAST, existedColumnName=null}]}",
- "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={age=BIGINT, id=BIGINT}, oldTypeMapping={age=INT, id=INT}}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=AFTER, existedColumnName=age}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT}, oldTypeMapping={id=INT}}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, student], after=[], op=DELETE, meta=()}",
- "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`last_name` STRING, position=LAST, existedColumnName=null}]}",
- "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`last_name` STRING, position=AFTER, existedColumnName=description}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=AFTER, existedColumnName=last_name}]}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, null, 24, null, Eliza, null], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, student, null, male], op=INSERT, meta=()}");
}
@@ -1026,7 +1026,7 @@ class FlinkPipelineComposerITCase {
Schema.newBuilder()
.physicalColumn("id", DataTypes.BIGINT())
.physicalColumn("name", DataTypes.STRING())
- .physicalColumn("age", DataTypes.BIGINT())
+ .physicalColumn("age", DataTypes.INT())
.physicalColumn("last_name", DataTypes.STRING())
.physicalColumn("description", DataTypes.STRING())
.physicalColumn("gender", DataTypes.STRING())
@@ -1039,12 +1039,12 @@ class FlinkPipelineComposerITCase {
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[1, Alice, 18, last_name], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, last_name], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20, last_name], after=[2, Bob, 30, last_name], op=UPDATE, meta=()}",
- "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=LAST, existedColumnName=null}]}",
- "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={age=BIGINT, id=BIGINT}, oldTypeMapping={age=INT, id=INT}}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=AFTER, existedColumnName=last_name}]}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT}, oldTypeMapping={id=INT}}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, last_name, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, last_name, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, last_name, student], after=[], op=DELETE, meta=()}",
- "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`gender` STRING, position=AFTER, existedColumnName=description}]}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[5, Eliza, 24, last_name, null, null], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[6, Frank, 30, last_name, student, male], op=INSERT, meta=()}");
}
@@ -1207,16 +1207,17 @@ class FlinkPipelineComposerITCase {
"DataChangeEvent{tableId={}_everything_merged, before=[], after=[1, Alice, 17, 2020-01-01T14:28:57], op=INSERT, meta=()}",
"AlterColumnTypeEvent{tableId={}_everything_merged, typeMapping={birthday=TIMESTAMP(9)}, oldTypeMapping={birthday=TIMESTAMP(0)}}",
"DataChangeEvent{tableId={}_everything_merged, before=[], after=[2, Alice, 17, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[3, Alice, 17, 2020-01-01T09:28:57], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[4, Alice, 17, 2020-01-01T09:28:57.123456789], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[5, Alice, 17, 2020-01-01T09:28:57], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[6, Alice, 17, 2020-01-01T09:28:57.123456789], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[101, Zen, 19, 2020-01-01T14:28:57], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[102, Zen, 19, 2020-01-01T14:28:57.123456789], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[103, Zen, 19, 2020-01-01T09:28:57], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[104, Zen, 19, 2020-01-01T09:28:57.123456789], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[105, Zen, 19, 2020-01-01T09:28:57], op=INSERT, meta=()}",
- "DataChangeEvent{tableId={}_everything_merged, before=[], after=[106, Zen, 19, 2020-01-01T09:28:57.123456789], op=INSERT, meta=()}")
+ "AlterColumnTypeEvent{tableId={}_everything_merged, typeMapping={birthday=TIMESTAMP(9) WITH TIME ZONE}, oldTypeMapping={birthday=TIMESTAMP(9)}}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[3, Alice, 17, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[4, Alice, 17, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[5, Alice, 17, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[6, Alice, 17, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[101, Zen, 19, 2020-01-01T09:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[102, Zen, 19, 2020-01-01T09:28:57.123456789-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[103, Zen, 19, 2020-01-01T14:28:57Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[104, Zen, 19, 2020-01-01T14:28:57.123456789Z], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[105, Zen, 19, 2020-01-01T04:28:57-05:00], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId={}_everything_merged, before=[], after=[106, Zen, 19, 2020-01-01T04:28:57.123456789-05:00], op=INSERT, meta=()}")
.map(s -> s.replace("{}", "default_namespace.default_schema.default"))
.toArray(String[]::new);
@@ -1276,9 +1277,11 @@ class FlinkPipelineComposerITCase {
Stream.of(
"CreateTableEvent{tableId={}, schema=columns={`id` INT,`name` STRING,`age` INT,`fav_num` TINYINT}, primaryKeys=id, options=()}",
"DataChangeEvent{tableId={}, before=[], after=[1, Alice, 17, 1], op=INSERT, meta=()}",
- "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=BIGINT}, oldTypeMapping={fav_num=TINYINT}}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=SMALLINT}, oldTypeMapping={fav_num=TINYINT}}",
"DataChangeEvent{tableId={}, before=[], after=[2, Alice, 17, 22], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=INT}, oldTypeMapping={fav_num=SMALLINT}}",
"DataChangeEvent{tableId={}, before=[], after=[3, Alice, 17, 3333], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=BIGINT}, oldTypeMapping={fav_num=INT}}",
"DataChangeEvent{tableId={}, before=[], after=[4, Alice, 17, 44444444], op=INSERT, meta=()}",
"AlterColumnTypeEvent{tableId={}, typeMapping={fav_num=DECIMAL(19, 0)}, oldTypeMapping={fav_num=BIGINT}}",
"DataChangeEvent{tableId={}, before=[], after=[5, Alice, 17, 555555555555555], op=INSERT, meta=()}",
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java
index da33d83d5..358094aa6 100644
--- a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineComposerLenientITCase.java
@@ -851,7 +851,7 @@ class FlinkPipelineComposerLenientITCase {
Schema.newBuilder()
.physicalColumn("id", DataTypes.BIGINT())
.physicalColumn("name", DataTypes.STRING())
- .physicalColumn("age", DataTypes.BIGINT())
+ .physicalColumn("age", DataTypes.INT())
.physicalColumn("description", DataTypes.STRING())
.physicalColumn("last_name", DataTypes.STRING())
.physicalColumn("gender", DataTypes.STRING())
@@ -865,7 +865,7 @@ class FlinkPipelineComposerLenientITCase {
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20], after=[2, Bob, 30], op=UPDATE, meta=()}",
"AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=LAST, existedColumnName=null}]}",
- "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={age=BIGINT, id=BIGINT}, oldTypeMapping={age=INT, id=INT}}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT}, oldTypeMapping={id=INT}}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, student], after=[], op=DELETE, meta=()}",
@@ -1059,7 +1059,7 @@ class FlinkPipelineComposerLenientITCase {
Schema.newBuilder()
.physicalColumn("id", DataTypes.BIGINT())
.physicalColumn("name", DataTypes.STRING())
- .physicalColumn("age", DataTypes.BIGINT())
+ .physicalColumn("age", DataTypes.INT())
.physicalColumn("last_name", DataTypes.STRING())
.physicalColumn("description", DataTypes.STRING())
.physicalColumn("gender", DataTypes.STRING())
@@ -1073,7 +1073,7 @@ class FlinkPipelineComposerLenientITCase {
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[2, Bob, 20, last_name], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[2, Bob, 20, last_name], after=[2, Bob, 30, last_name], op=UPDATE, meta=()}",
"AddColumnEvent{tableId=default_namespace.default_schema.merged, addedColumns=[ColumnWithPosition{column=`description` STRING, position=LAST, existedColumnName=null}]}",
- "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={age=BIGINT, id=BIGINT}, oldTypeMapping={age=INT, id=INT}}",
+ "AlterColumnTypeEvent{tableId=default_namespace.default_schema.merged, typeMapping={id=BIGINT}, oldTypeMapping={id=INT}}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[3, Charlie, 15, last_name, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[], after=[4, Donald, 25, last_name, student], op=INSERT, meta=()}",
"DataChangeEvent{tableId=default_namespace.default_schema.merged, before=[4, Donald, 25, last_name, student], after=[], op=DELETE, meta=()}",
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/factory/DistributedDataSourceFactory.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/factory/DistributedDataSourceFactory.java
new file mode 100644
index 000000000..868fa8aa9
--- /dev/null
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/factory/DistributedDataSourceFactory.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.testsource.factory;
+
+import org.apache.flink.cdc.common.annotation.Internal;
+import org.apache.flink.cdc.common.configuration.ConfigOption;
+import org.apache.flink.cdc.common.factories.DataSourceFactory;
+import org.apache.flink.cdc.common.factories.Factory;
+import org.apache.flink.cdc.common.source.DataSource;
+import org.apache.flink.cdc.composer.testsource.source.DistributedDataSource;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.ImmutableSet;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.flink.cdc.composer.testsource.source.DistributedSourceOptions.DISTRIBUTED_TABLES;
+import static org.apache.flink.cdc.composer.testsource.source.DistributedSourceOptions.TABLE_COUNT;
+
+/** A source {@link Factory} to create {@link DistributedDataSource}. */
+@Internal
+public class DistributedDataSourceFactory implements DataSourceFactory {
+
+ public static final String IDENTIFIER = "dds";
+
+ @Override
+ public DataSource createDataSource(Context context) {
+ int tableCount = context.getFactoryConfiguration().get(TABLE_COUNT);
+ boolean distributedTables = context.getFactoryConfiguration().get(DISTRIBUTED_TABLES);
+ return new DistributedDataSource(tableCount, distributedTables);
+ }
+
+ @Override
+ public String identifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ return new HashSet<>();
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ return ImmutableSet.of(TABLE_COUNT, DISTRIBUTED_TABLES);
+ }
+}
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedDataSource.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedDataSource.java
new file mode 100644
index 000000000..dd3952039
--- /dev/null
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedDataSource.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.testsource.source;
+
+import org.apache.flink.cdc.common.annotation.Internal;
+import org.apache.flink.cdc.common.source.DataSource;
+import org.apache.flink.cdc.common.source.EventSourceProvider;
+import org.apache.flink.cdc.common.source.FlinkSourceFunctionProvider;
+import org.apache.flink.cdc.common.source.MetadataAccessor;
+
+/** A {@link DataSource} that emits single-table-in-multi-partition data records. */
+@Internal
+public class DistributedDataSource implements DataSource {
+
+ private final int tableCount;
+ private final boolean distributedTables;
+
+ public DistributedDataSource(int tableCount, boolean distributedTables) {
+ this.tableCount = tableCount;
+ this.distributedTables = distributedTables;
+ }
+
+ @Override
+ public EventSourceProvider getEventSourceProvider() {
+ return FlinkSourceFunctionProvider.of(
+ new DistributedSourceFunction(tableCount, distributedTables));
+ }
+
+ @Override
+ public MetadataAccessor getMetadataAccessor() {
+ throw new UnsupportedOperationException(
+ "'Distributed' data source does not provide a metadata accessor.");
+ }
+
+ @Override
+ public boolean isParallelMetadataSource() {
+ // Since we're simulating a data source with distributed tables, metadata events might be
+ // emitted in parallel.
+ return distributedTables;
+ }
+}
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java
new file mode 100644
index 000000000..b7b7787f1
--- /dev/null
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.testsource.source;
+
+import org.apache.flink.cdc.common.data.DecimalData;
+import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
+import org.apache.flink.cdc.common.data.TimestampData;
+import org.apache.flink.cdc.common.data.ZonedTimestampData;
+import org.apache.flink.cdc.common.data.binary.BinaryRecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/** Distributed source function for testing purposes only. */
+public class DistributedSourceFunction extends RichParallelSourceFunction<Event> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DistributedSourceFunction.class);
+
+ private int subTaskId;
+ private int parallelism;
+
+ private final int numOfTables;
+ private final boolean distributedTables;
+ private transient Map<DataType, Object> dummyDataTypes;
+ private List<TableId> tables;
+ private transient int iotaCounter;
+
+ public DistributedSourceFunction(int numOfTables, boolean distributedTables) {
+ this.numOfTables = numOfTables;
+ this.distributedTables = distributedTables;
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ iotaCounter = 0;
+ subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
+ parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks();
+ if (distributedTables) {
+ tables =
+ IntStream.range(0, numOfTables)
+ .mapToObj(
+ idx ->
+ TableId.tableId(
+ "default_namespace",
+ "default_database",
+ "table_" + idx))
+ .collect(Collectors.toList());
+ } else {
+ tables =
+ IntStream.range(0, numOfTables)
+ .mapToObj(
+ idx ->
+ TableId.tableId(
+ "default_namespace_subtask_" + subTaskId,
+ "default_database",
+ "table_" + idx))
+ .collect(Collectors.toList());
+ }
+ dummyDataTypes = new LinkedHashMap<>();
+ dummyDataTypes.put(DataTypes.BOOLEAN(), true);
+ dummyDataTypes.put(DataTypes.TINYINT(), (byte) 17);
+ dummyDataTypes.put(DataTypes.SMALLINT(), (short) 34);
+ dummyDataTypes.put(DataTypes.INT(), (int) 68);
+ dummyDataTypes.put(DataTypes.BIGINT(), (long) 136);
+ dummyDataTypes.put(DataTypes.FLOAT(), (float) 272.0);
+ dummyDataTypes.put(DataTypes.DOUBLE(), (double) 544.0);
+ dummyDataTypes.put(
+ DataTypes.DECIMAL(17, 11),
+ DecimalData.fromBigDecimal(new BigDecimal("1088.000"), 17, 11));
+ dummyDataTypes.put(DataTypes.CHAR(17), BinaryStringData.fromString("Alice"));
+ dummyDataTypes.put(DataTypes.VARCHAR(17), BinaryStringData.fromString("Bob"));
+ dummyDataTypes.put(DataTypes.BINARY(17), "Cicada".getBytes());
+ dummyDataTypes.put(DataTypes.VARBINARY(17), "Derrida".getBytes());
+ dummyDataTypes.put(DataTypes.TIME(9), 64800000);
+ dummyDataTypes.put(
+ DataTypes.TIMESTAMP(9),
+ TimestampData.fromTimestamp(Timestamp.valueOf("2020-07-17 18:00:00")));
+ dummyDataTypes.put(
+ DataTypes.TIMESTAMP_TZ(9),
+ ZonedTimestampData.of(364800000, 123456, "Asia/Shanghai"));
+ dummyDataTypes.put(
+ DataTypes.TIMESTAMP_LTZ(9),
+ LocalZonedTimestampData.fromInstant(toInstant("2019-12-31 18:00:00")));
+ }
+
+ // Generates statically incrementing data, could be used for data integrity verification.
+ private BinaryStringData iota() {
+ return BinaryStringData.fromString(String.format("__$%d$%d$__", subTaskId, iotaCounter++));
+ }
+
+ private void sendFromTables(Consumer<TableId> tableIdConsumer) {
+ if (parallelism > 1) {
+ // Inject a little randomness in multi-parallelism mode
+ Collections.shuffle(tables);
+ }
+ tables.forEach(tableIdConsumer);
+ }
+
+ @Override
+ public void run(SourceContext<Event> context) throws InterruptedException {
+ Schema initialSchema =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.STRING())
+ .primaryKey("id")
+ .partitionKey("id")
+ .build();
+
+ Map<TableId, Schema> headSchemaMap = new HashMap<>();
+
+ sendFromTables(
+ tableId -> {
+ CreateTableEvent createTableEvent =
+ new CreateTableEvent(tableId, initialSchema);
+ headSchemaMap.compute(
+ tableId,
+ (tbl, schema) ->
+ SchemaUtils.applySchemaChangeEvent(schema, createTableEvent));
+ collect(context, createTableEvent);
+ for (int i = 0; i < 10; i++) {
+ collect(
+ context,
+ DataChangeEvent.insertEvent(
+ tableId, generateBinRec(headSchemaMap.get(tableId))));
+ }
+ });
+
+ List<DataType> fullTypes = new ArrayList<>(dummyDataTypes.keySet());
+ if (parallelism > 1) {
+ // Inject randomness in multi-partition mode
+ Collections.shuffle(fullTypes);
+ }
+ fullTypes.forEach(
+ colType -> {
+ sendFromTables(
+ tableId -> {
+ AddColumnEvent addColumnEvent =
+ new AddColumnEvent(
+ tableId,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "col_"
+ + colType.getClass()
+ .getSimpleName()
+ .toLowerCase(),
+ colType))));
+
+ headSchemaMap.compute(
+ tableId,
+ (tbl, schema) ->
+ SchemaUtils.applySchemaChangeEvent(
+ schema, addColumnEvent));
+ collect(context, addColumnEvent);
+ collect(
+ context,
+ DataChangeEvent.insertEvent(
+ tableId,
+ generateBinRec(headSchemaMap.get(tableId))));
+ });
+
+ sendFromTables(
+ tableId -> {
+ AddColumnEvent addColumnEvent =
+ new AddColumnEvent(
+ tableId,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "subtask_"
+ + subTaskId
+ + "_col_"
+ + colType.getClass()
+ .getSimpleName()
+ .toLowerCase(),
+ colType))));
+
+ headSchemaMap.compute(
+ tableId,
+ (tbl, schema) ->
+ SchemaUtils.applySchemaChangeEvent(
+ schema, addColumnEvent));
+ collect(context, addColumnEvent);
+ collect(
+ context,
+ DataChangeEvent.insertEvent(
+ tableId,
+ generateBinRec(headSchemaMap.get(tableId))));
+ });
+ });
+
+ if (parallelism > 1) {
+ // To allow test running correctly, we need to wait for downstream schema evolutions
+ // to finish before closing any subTask.
+ Thread.sleep(10000);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ // Do nothing
+ }
+
+ private BinaryRecordData generateBinRec(Schema schema) {
+ BinaryRecordDataGenerator generator =
+ new BinaryRecordDataGenerator(schema.getColumnDataTypes().toArray(new DataType[0]));
+
+ int arity = schema.getColumnDataTypes().size();
+ List<DataType> rowTypes = schema.getColumnDataTypes();
+ Object[] rowObjects = new Object[arity];
+
+ for (int i = 0; i < arity; i++) {
+ DataType type = rowTypes.get(i);
+ if (Objects.equals(type, DataTypes.STRING())) {
+ rowObjects[i] = iota();
+ } else {
+ rowObjects[i] = dummyDataTypes.get(type);
+ }
+ }
+ return generator.generate(rowObjects);
+ }
+
+ private void collect(SourceContext<Event> sourceContext, Event event) {
+ LOG.info("{}> Emitting event {}", subTaskId, event);
+ sourceContext.collect(event);
+ }
+
+ private Instant toInstant(String ts) {
+ return Timestamp.valueOf(ts).toLocalDateTime().atZone(ZoneId.of("UTC")).toInstant();
+ }
+}
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceOptions.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceOptions.java
new file mode 100644
index 000000000..70fbb0501
--- /dev/null
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceOptions.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.composer.testsource.source;
+
+import org.apache.flink.cdc.common.configuration.ConfigOption;
+import org.apache.flink.cdc.common.configuration.ConfigOptions;
+
+/** Configurations for {@link DistributedDataSource}. */
+public class DistributedSourceOptions {
+ public static final ConfigOption<Integer> TABLE_COUNT =
+ ConfigOptions.key("table-count")
+ .intType()
+ .defaultValue(4)
+ .withDescription("Number of parallelized tables in one single parallelism.");
+
+ public static final ConfigOption<Boolean> DISTRIBUTED_TABLES =
+ ConfigOptions.key("distributed-tables")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Whether this source should emit distributed tables that might present in multiple partitions. Defaults to false.");
+}
diff --git a/flink-cdc-composer/src/test/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory b/flink-cdc-composer/src/test/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory
index 6cea454e0..97da54827 100644
--- a/flink-cdc-composer/src/test/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory
+++ b/flink-cdc-composer/src/test/resources/META-INF/services/org.apache.flink.cdc.common.factories.Factory
@@ -16,3 +16,4 @@ org.apache.flink.cdc.composer.utils.factory.DataSinkFactory1
org.apache.flink.cdc.composer.utils.factory.DataSinkFactory2
org.apache.flink.cdc.composer.utils.factory.DataSourceFactory1
org.apache.flink.cdc.composer.utils.factory.DataSourceFactory2
+org.apache.flink.cdc.composer.testsource.factory.DistributedDataSourceFactory
\ No newline at end of file
diff --git a/flink-cdc-composer/src/test/resources/ref-output/distributed-ignore.txt b/flink-cdc-composer/src/test/resources/ref-output/distributed-ignore.txt
new file mode 100644
index 000000000..39fd0ec50
--- /dev/null
+++ b/flink-cdc-composer/src/test/resources/ref-output/distributed-ignore.txt
@@ -0,0 +1,172 @@
+CreateTableEvent{tableId=default_namespace.default_database.table_0, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$0$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$1$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$2$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$3$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$4$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$5$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$6$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$7$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$8$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$9$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace.default_database.table_1, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$10$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$11$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$12$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$13$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$14$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$15$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$16$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$17$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$18$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$19$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace.default_database.table_2, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$20$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$21$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$22$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$23$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$24$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$25$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$26$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$27$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$28$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$29$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace.default_database.table_3, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$30$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$31$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$32$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$33$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$34$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$35$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$36$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$37$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$38$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$39$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$40$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$41$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$42$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$43$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$44$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$45$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$46$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$47$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$48$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$49$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$50$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$51$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$52$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$53$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$54$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$55$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$56$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$57$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$58$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$59$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$60$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$61$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$62$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$63$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$64$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$65$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$66$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$67$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$68$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$69$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$70$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$71$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$72$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$73$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$74$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$75$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$76$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$77$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$78$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$79$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$80$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$81$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$82$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$83$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$84$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$85$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$86$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$87$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$88$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$89$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$90$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$91$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$92$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$93$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$94$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$95$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$96$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$97$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$98$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$99$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$100$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$101$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$102$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$103$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$104$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$105$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$106$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$107$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$108$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$109$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$110$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$111$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$112$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$113$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$114$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$115$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$116$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$117$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$118$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$119$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$120$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$121$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$122$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$123$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$124$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$125$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$126$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$127$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$128$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$129$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$130$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$131$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$132$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$133$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$134$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$135$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$136$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$137$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$138$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$139$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$140$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$141$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$142$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$143$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$144$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$145$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$146$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$147$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$148$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$149$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$150$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$151$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$152$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$153$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$154$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$155$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$156$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$157$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$158$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$159$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$160$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$161$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$162$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$163$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$164$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$165$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$166$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$167$__], op=INSERT, meta=()}
diff --git a/flink-cdc-composer/src/test/resources/ref-output/distributed.txt b/flink-cdc-composer/src/test/resources/ref-output/distributed.txt
new file mode 100644
index 000000000..5cecea29c
--- /dev/null
+++ b/flink-cdc-composer/src/test/resources/ref-output/distributed.txt
@@ -0,0 +1,300 @@
+CreateTableEvent{tableId=default_namespace.default_database.table_0, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$0$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$1$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$2$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$3$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$4$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$5$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$6$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$7$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$8$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$9$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace.default_database.table_1, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$10$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$11$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$12$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$13$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$14$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$15$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$16$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$17$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$18$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$19$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace.default_database.table_2, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$20$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$21$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$22$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$23$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$24$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$25$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$26$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$27$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$28$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$29$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace.default_database.table_3, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$30$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$31$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$32$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$33$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$34$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$35$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$36$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$37$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$38$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$39$__], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$40$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$41$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$42$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$43$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$44$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$45$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$46$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$47$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$48$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$49$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$50$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$51$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$52$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$53$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$54$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$55$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$56$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$57$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$58$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$59$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$60$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$61$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$62$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$63$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$64$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$65$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$66$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$67$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$68$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$69$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$70$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$71$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$72$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$73$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$74$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$75$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$76$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$77$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$78$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$79$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$80$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$81$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$82$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$83$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$84$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$85$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$86$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$87$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$88$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$89$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$90$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$91$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$92$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$93$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$94$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$95$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$96$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$97$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$98$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$99$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$100$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$101$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$102$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$103$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$104$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$105$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$106$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$107$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$108$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$109$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$110$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$111$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$112$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$113$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$114$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$115$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$116$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$117$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$118$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$119$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$120$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$121$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$122$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$123$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$124$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$125$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$126$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$127$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$128$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$129$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$130$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$131$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$132$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$133$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$134$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$135$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$136$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$137$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$138$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$139$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$140$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$141$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$142$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$143$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$144$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$145$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$146$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$147$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$148$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$149$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$150$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$151$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$152$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$153$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$154$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$155$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$156$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$157$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$158$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$159$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$160$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$161$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$162$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$163$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_0, before=[], after=[__$0$164$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_1, before=[], after=[__$0$165$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_2, before=[], after=[__$0$166$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace.default_database.table_3, before=[], after=[__$0$167$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
diff --git a/flink-cdc-composer/src/test/resources/ref-output/regular.txt b/flink-cdc-composer/src/test/resources/ref-output/regular.txt
new file mode 100644
index 000000000..2781e76e7
--- /dev/null
+++ b/flink-cdc-composer/src/test/resources/ref-output/regular.txt
@@ -0,0 +1,300 @@
+CreateTableEvent{tableId=default_namespace_subtask_0.default_database.table_0, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$0$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$1$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$2$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$3$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$4$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$5$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$6$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$7$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$8$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$9$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace_subtask_0.default_database.table_1, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$10$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$11$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$12$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$13$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$14$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$15$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$16$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$17$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$18$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$19$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace_subtask_0.default_database.table_2, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$20$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$21$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$22$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$23$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$24$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$25$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$26$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$27$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$28$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$29$__], op=INSERT, meta=()}
+CreateTableEvent{tableId=default_namespace_subtask_0.default_database.table_3, schema=columns={`id` STRING}, primaryKeys=id, partitionKeys=id, options=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$30$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$31$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$32$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$33$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$34$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$35$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$36$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$37$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$38$__], op=INSERT, meta=()}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$39$__], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$40$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$41$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$42$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$43$__, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$44$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$45$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$46$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_booleantype` BOOLEAN, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$47$__, true, true], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$48$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$49$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$50$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$51$__, true, true, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$52$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$53$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$54$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_tinyinttype` TINYINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$55$__, true, true, 17, 17], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$56$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$57$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$58$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$59$__, true, true, 17, 17, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$60$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$61$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$62$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_smallinttype` SMALLINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$63$__, true, true, 17, 17, 34, 34], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$64$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$65$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$66$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$67$__, true, true, 17, 17, 34, 34, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$68$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$69$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$70$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_inttype` INT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$71$__, true, true, 17, 17, 34, 34, 68, 68], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$72$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$73$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$74$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$75$__, true, true, 17, 17, 34, 34, 68, 68, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$76$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$77$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$78$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_biginttype` BIGINT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$79$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$80$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$81$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$82$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$83$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$84$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$85$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$86$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_floattype` FLOAT, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$87$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$88$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$89$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$90$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$91$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$92$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$93$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$94$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_doubletype` DOUBLE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$95$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$96$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$97$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$98$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$99$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$100$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$101$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$102$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_decimaltype` DECIMAL(17, 11), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$103$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$104$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$105$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$106$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$107$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$108$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$109$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$110$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_chartype` CHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$111$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$112$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$113$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$114$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$115$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$116$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$117$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$118$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varchartype` VARCHAR(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$119$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$120$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$121$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$122$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$123$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$124$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$125$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$126$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_binarytype` BINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$127$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$128$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$129$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$130$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$131$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$132$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$133$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$134$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_varbinarytype` VARBINARY(17), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$135$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$136$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$137$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$138$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$139$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$140$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$141$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$142$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timetype` TIME(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$143$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$144$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$145$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$146$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$147$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$148$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$149$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$150$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_timestamptype` TIMESTAMP(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$151$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$152$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$153$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$154$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$155$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$156$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$157$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$158$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_zonedtimestamptype` TIMESTAMP(9) WITH TIME ZONE, position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$159$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$160$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$161$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$162$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$163$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_0, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_0, before=[], after=[__$0$164$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_1, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_1, before=[], after=[__$0$165$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_2, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_2, before=[], after=[__$0$166$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
+AddColumnEvent{tableId=default_namespace_subtask_0.default_database.table_3, addedColumns=[ColumnWithPosition{column=`subtask_0_col_localzonedtimestamptype` TIMESTAMP_LTZ(9), position=LAST, existedColumnName=null}]}
+DataChangeEvent{tableId=default_namespace_subtask_0.default_database.table_3, before=[], after=[__$0$167$__, true, true, 17, 17, 34, 34, 68, 68, 136, 136, 272.0, 272.0, 544.0, 544.0, 1088.00000000000, 1088.00000000000, Alice, Alice, Bob, Bob, Q2ljYWRh, Q2ljYWRh, RGVycmlkYQ==, RGVycmlkYQ==, 64800000, 64800000, 2020-07-17T18:00, 2020-07-17T18:00, 1970-01-05T05:20:00.000123456+08:00, 1970-01-05T05:20:00.000123456+08:00, 2019-12-31T18:00, 2019-12-31T18:00], op=INSERT, meta=()}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris/src/test/java/org/apache/flink/cdc/connectors/doris/sink/DorisMetadataApplierITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris/src/test/java/org/apache/flink/cdc/connectors/doris/sink/DorisMetadataApplierITCase.java
index 4d28b8f77..8aa0c5422 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris/src/test/java/org/apache/flink/cdc/connectors/doris/sink/DorisMetadataApplierITCase.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris/src/test/java/org/apache/flink/cdc/connectors/doris/sink/DorisMetadataApplierITCase.java
@@ -458,7 +458,7 @@ public class DorisMetadataApplierITCase extends DorisSinkTestBase {
new OperatorIDGenerator(schemaOperatorTranslator.getSchemaOperatorUid());
stream =
- schemaOperatorTranslator.translate(
+ schemaOperatorTranslator.translateRegular(
stream,
DEFAULT_PARALLELISM,
dorisSink
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch/pom.xml b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch/pom.xml
index 9512884f2..516eb1cb7 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch/pom.xml
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch/pom.xml
@@ -34,41 +34,13 @@ limitations under the License.
<url>http://maven.apache.org</url>
<properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<elasticsearch.version>8.12.1</elasticsearch.version>
- <flink.version>1.19.1</flink.version>
- <scala.binary.version>4.0</scala.binary.version>
- <jackson.version>2.13.2</jackson.version>
- <surefire.module.config>--add-opens=java.base/java.util=ALL-UNNAMED</surefire.module.config>
- <testcontainers.version>1.16.0</testcontainers.version>
<flink.connector.elasticsearch.version>3.0.1-1.17</flink.connector.elasticsearch.version>
<httpclient.version>4.5.13</httpclient.version>
- <junit.jupiter.version>5.7.1</junit.jupiter.version>
- <assertj.version>3.18.1</assertj.version>
- <junit.version>4.13.2</junit.version>
- <slf4j.version>1.7.32</slf4j.version>
- <junit.platform.version>1.10.2</junit.platform.version>
<jakarta.json.version>2.0.2</jakarta.json.version>
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-streaming-java</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-api-java-bridge</artifactId>
- <version>${flink.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-base</artifactId>
- <version>${flink.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-json</artifactId>
@@ -142,39 +114,6 @@ limitations under the License.
<version>${elasticsearch.version}</version>
</dependency>
- <!-- SLF4J API -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
- <!-- Testing -->
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-api</artifactId>
- <version>${junit.jupiter.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- <version>${junit.jupiter.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.assertj</groupId>
- <artifactId>assertj-core</artifactId>
- <version>${assertj.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- <scope>test</scope>
- </dependency>
-
<!-- Architecture Tests -->
<dependency>
<groupId>org.apache.flink</groupId>
@@ -188,12 +127,6 @@ limitations under the License.
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.junit.platform</groupId>
- <artifactId>junit-platform-launcher</artifactId>
- <version>${junit.platform.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-cdc-composer</artifactId>
@@ -222,7 +155,6 @@ limitations under the License.
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <version>${maven.shade.plugin.version}</version>
<executions>
<execution>
<phase>package</phase>
@@ -245,6 +177,10 @@ limitations under the License.
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ </plugin>
</plugins>
</build>
-</project>
+</project>
\ No newline at end of file
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml
index 475f3b62b..371b1dacf 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/pom.xml
@@ -46,6 +46,20 @@ limitations under the License.
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-composer</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-pipeline-connector-values</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-test-util</artifactId>
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlDataSource.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlDataSource.java
index 17cb5cd5a..ae676b37a 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlDataSource.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlDataSource.java
@@ -88,4 +88,11 @@ public class MySqlDataSource implements DataSource {
public SupportedMetadataColumn[] supportedMetadataColumns() {
return new SupportedMetadataColumn[] {new OpTsMetadataColumn()};
}
+
+ @Override
+ public boolean isParallelMetadataSource() {
+ // During incremental stage, MySQL never emits schema change events on different partitions
+ // (since it has one Binlog stream only.)
+ return false;
+ }
}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlParallelizedPipelineITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlParallelizedPipelineITCase.java
new file mode 100644
index 000000000..4bd9806a5
--- /dev/null
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/source/MySqlParallelizedPipelineITCase.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.connectors.mysql.source;
+
+import org.apache.flink.cdc.common.configuration.Configuration;
+import org.apache.flink.cdc.common.pipeline.PipelineOptions;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.composer.PipelineExecution;
+import org.apache.flink.cdc.composer.definition.PipelineDef;
+import org.apache.flink.cdc.composer.definition.SinkDef;
+import org.apache.flink.cdc.composer.definition.SourceDef;
+import org.apache.flink.cdc.composer.flink.FlinkPipelineComposer;
+import org.apache.flink.cdc.connectors.mysql.factory.MySqlDataSourceFactory;
+import org.apache.flink.cdc.connectors.mysql.testutils.UniqueDatabase;
+import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import org.apache.flink.cdc.connectors.values.factory.ValuesDataFactory;
+import org.apache.flink.cdc.connectors.values.sink.ValuesDataSinkOptions;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import static org.apache.flink.cdc.connectors.mysql.testutils.MySqSourceTestUtils.TEST_PASSWORD;
+import static org.apache.flink.cdc.connectors.mysql.testutils.MySqSourceTestUtils.TEST_USER;
+import static org.apache.flink.cdc.connectors.mysql.testutils.MySqSourceTestUtils.getServerId;
+import static org.apache.flink.cdc.connectors.mysql.testutils.MySqSourceTestUtils.loopCheck;
+import static org.apache.flink.configuration.CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Parallelized Integration test for MySQL connector. */
+public class MySqlParallelizedPipelineITCase extends MySqlSourceTestBase {
+
+ private static final int PARALLELISM = 4;
+ private static final int TEST_TABLE_NUMBER = 100;
+
+ // Always use parent-first classloader for CDC classes.
+ // The reason is that ValuesDatabase uses static field for holding data, we need to make sure
+ // the class is loaded by AppClassloader so that we can verify data in the test case.
+ private static final org.apache.flink.configuration.Configuration MINI_CLUSTER_CONFIG =
+ new org.apache.flink.configuration.Configuration();
+
+ static {
+ MINI_CLUSTER_CONFIG.set(
+ ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
+ Collections.singletonList("org.apache.flink.cdc"));
+ }
+
+ private final PrintStream standardOut = System.out;
+ private final ByteArrayOutputStream outCaptor = new ByteArrayOutputStream();
+
+ private final UniqueDatabase parallelismDatabase =
+ new UniqueDatabase(
+ MYSQL_CONTAINER, "extreme_parallelism_test_database", TEST_USER, TEST_PASSWORD);
+
+ @Before
+ public void init() {
+ // Take over STDOUT as we need to check the output of values sink
+ System.setOut(new PrintStream(outCaptor));
+ // Initialize in-memory database
+ ValuesDatabase.clear();
+ }
+
+ @After
+ public void cleanup() {
+ System.setOut(standardOut);
+ }
+
+ @Test
+ public void testExtremeParallelizedSchemaChange() throws Exception {
+ final String databaseName = parallelismDatabase.getDatabaseName();
+ try (Connection conn =
+ DriverManager.getConnection(
+ MYSQL_CONTAINER.getJdbcUrl(), TEST_USER, TEST_PASSWORD);
+ Statement stat = conn.createStatement()) {
+ stat.execute(String.format("CREATE DATABASE %s;", databaseName));
+ stat.execute(String.format("USE %s;", databaseName));
+ for (int i = 1; i <= TEST_TABLE_NUMBER; i++) {
+ stat.execute(String.format("DROP TABLE IF EXISTS TABLE%d;", i));
+ stat.execute(
+ String.format(
+ "CREATE TABLE TABLE%d (ID INT NOT NULL PRIMARY KEY,VERSION VARCHAR(17));",
+ i));
+ stat.execute(String.format("INSERT INTO TABLE%d VALUES (%d, 'No.%d');", i, i, i));
+ }
+ } catch (SQLException e) {
+ LOG.error("Initialize table failed.", e);
+ throw e;
+ }
+ LOG.info("Table initialized successfully.");
+
+ FlinkPipelineComposer composer = FlinkPipelineComposer.ofMiniCluster();
+
+ // Setup MySQL source
+ Configuration sourceConfig = new Configuration();
+ sourceConfig.set(MySqlDataSourceOptions.HOSTNAME, MYSQL_CONTAINER.getHost());
+ sourceConfig.set(MySqlDataSourceOptions.PORT, MYSQL_CONTAINER.getDatabasePort());
+ sourceConfig.set(MySqlDataSourceOptions.USERNAME, TEST_USER);
+ sourceConfig.set(MySqlDataSourceOptions.PASSWORD, TEST_PASSWORD);
+ sourceConfig.set(MySqlDataSourceOptions.SERVER_TIME_ZONE, "UTC");
+ sourceConfig.set(MySqlDataSourceOptions.TABLES, "\\.*.\\.*");
+ sourceConfig.set(MySqlDataSourceOptions.SERVER_ID, getServerId(PARALLELISM));
+
+ SourceDef sourceDef =
+ new SourceDef(MySqlDataSourceFactory.IDENTIFIER, "MySQL Source", sourceConfig);
+
+ // Setup value sink
+ Configuration sinkConfig = new Configuration();
+ sinkConfig.set(ValuesDataSinkOptions.MATERIALIZED_IN_MEMORY, true);
+ SinkDef sinkDef = new SinkDef(ValuesDataFactory.IDENTIFIER, "Value Sink", sinkConfig);
+
+ // Setup pipeline
+ Configuration pipelineConfig = new Configuration();
+ pipelineConfig.set(PipelineOptions.PIPELINE_PARALLELISM, PARALLELISM);
+ pipelineConfig.set(
+ PipelineOptions.PIPELINE_SCHEMA_CHANGE_BEHAVIOR, SchemaChangeBehavior.EVOLVE);
+ PipelineDef pipelineDef =
+ new PipelineDef(
+ sourceDef,
+ sinkDef,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ pipelineConfig);
+
+ // Execute the pipeline
+ PipelineExecution execution = composer.compose(pipelineDef);
+ Thread executeThread =
+ new Thread(
+ () -> {
+ try {
+ execution.execute();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ executeThread.start();
+
+ try {
+ loopCheck(
+ () ->
+ outCaptor.toString().trim().split("\n").length
+ >= TEST_TABLE_NUMBER * (PARALLELISM + 1),
+ "collect enough rows",
+ Duration.ofSeconds(120),
+ Duration.ofSeconds(1));
+ } finally {
+ executeThread.interrupt();
+ }
+
+ // Check the order and content of all received events
+ String outputEvents = outCaptor.toString();
+ assertThat(outputEvents)
+ .contains(
+ IntStream.rangeClosed(1, TEST_TABLE_NUMBER)
+ .boxed()
+ .flatMap(
+ i ->
+ Stream.concat(
+ IntStream.range(0, PARALLELISM)
+ .boxed()
+ .map(
+ subTaskId ->
+ String.format(
+ "%d> CreateTableEvent{tableId=%s.TABLE%d, schema=columns={`ID` INT NOT NULL,`VERSION` VARCHAR(17)}, primaryKeys=ID, options=()}",
+ subTaskId,
+ parallelismDatabase
+ .getDatabaseName(),
+ i)),
+ Stream.of(
+ String.format(
+ "> DataChangeEvent{tableId=%s.TABLE%d, before=[], after=[%d, No.%d], op=INSERT, meta=()}",
+ parallelismDatabase
+ .getDatabaseName(),
+ i,
+ i,
+ i))))
+ .collect(Collectors.toList()));
+ }
+}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/testutils/MySqSourceTestUtils.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/testutils/MySqSourceTestUtils.java
index a76838d66..d193341ea 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/testutils/MySqSourceTestUtils.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/java/org/apache/flink/cdc/connectors/mysql/testutils/MySqSourceTestUtils.java
@@ -20,10 +20,13 @@ package org.apache.flink.cdc.connectors.mysql.testutils;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.cdc.common.event.CreateTableEvent;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
/** Test utilities for MySQL event source. */
public class MySqSourceTestUtils {
@@ -63,5 +66,19 @@ public class MySqSourceTestUtils {
return serverId + "-" + (serverId + parallelism);
}
+ public static void loopCheck(
+ Supplier<Boolean> runnable, String description, Duration timeout, Duration interval)
+ throws Exception {
+ long deadline = System.currentTimeMillis() + timeout.toMillis();
+ while (System.currentTimeMillis() < deadline) {
+ if (runnable.get()) {
+ return;
+ }
+ Thread.sleep(interval.toMillis());
+ }
+ throw new TimeoutException(
+ "Ran out of time when waiting for " + description + " to success.");
+ }
+
private MySqSourceTestUtils() {}
}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketAssignOperator.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketAssignOperator.java
index 358d7c36c..9b3b3afb9 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketAssignOperator.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketAssignOperator.java
@@ -124,7 +124,7 @@ public class BucketAssignOperator extends AbstractStreamOperator<Event>
output.collect(
new StreamRecord<>(
new BucketWrapperFlushEvent(
- currentTaskNumber, ((FlushEvent) event).getTableId())));
+ currentTaskNumber, ((FlushEvent) event).getSourceSubTaskId())));
return;
}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperEventSerializer.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperEventSerializer.java
index f37d9952f..95e0f6d3f 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperEventSerializer.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperEventSerializer.java
@@ -81,7 +81,7 @@ public class BucketWrapperEventSerializer extends TypeSerializerSingleton<Event>
enumSerializer.serialize(EventClass.BUCKET_WRAPPER_FLUSH_EVENT, dataOutputView);
BucketWrapperFlushEvent bucketWrapperFlushEvent = (BucketWrapperFlushEvent) event;
dataOutputView.writeInt(bucketWrapperFlushEvent.getBucket());
- tableIdSerializer.serialize(bucketWrapperFlushEvent.getTableId(), dataOutputView);
+ dataOutputView.writeInt(bucketWrapperFlushEvent.getSourceSubTaskId());
}
}
@@ -89,8 +89,7 @@ public class BucketWrapperEventSerializer extends TypeSerializerSingleton<Event>
public Event deserialize(DataInputView source) throws IOException {
EventClass eventClass = enumSerializer.deserialize(source);
if (eventClass.equals(EventClass.BUCKET_WRAPPER_FLUSH_EVENT)) {
- return new BucketWrapperFlushEvent(
- source.readInt(), tableIdSerializer.deserialize(source));
+ return new BucketWrapperFlushEvent(source.readInt(), source.readInt());
} else {
return new BucketWrapperChangeEvent(
source.readInt(), (ChangeEvent) eventSerializer.deserialize(source));
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperFlushEvent.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperFlushEvent.java
index 046a22a31..065871676 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperFlushEvent.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon/src/main/java/org/apache/flink/cdc/connectors/paimon/sink/v2/bucket/BucketWrapperFlushEvent.java
@@ -18,7 +18,6 @@
package org.apache.flink.cdc.connectors.paimon.sink.v2.bucket;
import org.apache.flink.cdc.common.event.FlushEvent;
-import org.apache.flink.cdc.common.event.TableId;
import java.util.Objects;
@@ -27,8 +26,8 @@ public class BucketWrapperFlushEvent extends FlushEvent implements BucketWrapper
private final int bucket;
- public BucketWrapperFlushEvent(int bucket, TableId tableId) {
- super(tableId);
+ public BucketWrapperFlushEvent(int bucket, int subTaskId) {
+ super(subTaskId);
this.bucket = bucket;
}
@@ -59,6 +58,10 @@ public class BucketWrapperFlushEvent extends FlushEvent implements BucketWrapper
@Override
public String toString() {
- return "BucketWrapperFlushEvent{tableId=" + getTableId() + ", bucket=" + bucket + '}';
+ return "BucketWrapperFlushEvent{subTaskId="
+ + getSourceSubTaskId()
+ + ", bucket="
+ + bucket
+ + '}';
}
}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierITCase.java
index 8b433ae4f..338af86d5 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierITCase.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks/src/test/java/org/apache/flink/cdc/connectors/starrocks/sink/StarRocksMetadataApplierITCase.java
@@ -373,7 +373,7 @@ public class StarRocksMetadataApplierITCase extends StarRocksSinkTestBase {
new OperatorIDGenerator(schemaOperatorTranslator.getSchemaOperatorUid());
stream =
- schemaOperatorTranslator.translate(
+ schemaOperatorTranslator.translateRegular(
stream,
DEFAULT_PARALLELISM,
starRocksSink
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSinkHelper.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSinkHelper.java
index ba52c1fd8..1d93899f9 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSinkHelper.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSinkHelper.java
@@ -22,8 +22,11 @@ import org.apache.flink.cdc.common.event.DataChangeEvent;
import org.apache.flink.cdc.common.event.Event;
import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.shaded.guava31.com.google.common.io.BaseEncoding;
+
import java.util.ArrayList;
import java.util.List;
+import java.util.stream.Collectors;
/** A helper class for {@link ValuesDataSink} to process {@link Event}. */
public class ValuesDataSinkHelper {
@@ -61,6 +64,15 @@ public class ValuesDataSinkHelper {
for (RecordData.FieldGetter fieldGetter : fieldGetters) {
fields.add(fieldGetter.getFieldOrNull(recordData));
}
- return fields;
+ return fields.stream()
+ .map(
+ o -> {
+ if (o instanceof byte[]) {
+ return BaseEncoding.base64().encode((byte[]) o);
+ } else {
+ return o;
+ }
+ })
+ .collect(Collectors.toList());
}
}
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java
index 8bd740794..976522b60 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java
@@ -545,17 +545,9 @@ public class MysqlE2eITCase extends PipelineTestEnvironment {
Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
submitPipelineJob(pipelineJob, mysqlCdcJar, valuesCdcJar, mysqlDriverJar);
waitUntilJobRunning(Duration.ofSeconds(30));
- LOG.info("Pipeline job is running");
- waitUntilSpecificEvent(
- String.format(
- "Table %s.live_fast received SchemaChangeEvent DropTableEvent{tableId=%s.live_fast} and start to be blocked.",
- mysqlInventoryDatabase.getDatabaseName(),
- mysqlInventoryDatabase.getDatabaseName()));
-
- waitUntilSpecificEvent(
- String.format(
- "Schema change event DropTableEvent{tableId=%s.live_fast} has been handled in another subTask already.",
- mysqlInventoryDatabase.getDatabaseName()));
+ validateResult(
+ "CreateTableEvent{tableId=%s.customers, schema=columns={`id` INT NOT NULL,`name` VARCHAR(255) NOT NULL 'flink',`address` VARCHAR(1024),`phone_number` VARCHAR(512)}, primaryKeys=id, options=()}",
+ "CreateTableEvent{tableId=%s.products, schema=columns={`id` INT NOT NULL,`name` VARCHAR(255) NOT NULL 'flink',`description` VARCHAR(512),`weight` FLOAT,`enum_c` STRING 'red',`json_c` STRING,`point_c` STRING}, primaryKeys=id, options=()}");
}
private void validateResult(String... expectedEvents) throws Exception {
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java
index c8072cf7d..cd342ef46 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java
@@ -24,7 +24,6 @@ import org.apache.flink.cdc.connectors.mysql.testutils.UniqueDatabase;
import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
@@ -41,6 +40,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
import java.util.concurrent.TimeoutException;
+import java.util.stream.IntStream;
/** E2e tests for routing features. */
@RunWith(Parameterized.class)
@@ -55,6 +55,7 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
protected static final String MYSQL_DRIVER_CLASS = "com.mysql.cj.jdbc.Driver";
protected static final String INTER_CONTAINER_MYSQL_ALIAS = "mysql";
protected static final long EVENT_DEFAULT_TIMEOUT = 60000L;
+ protected static final int TEST_TABLE_NUMBER = 100;
@ClassRule
public static final MySqlContainer MYSQL =
@@ -73,6 +74,9 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
protected final UniqueDatabase routeTestDatabase =
new UniqueDatabase(MYSQL, "route_test", MYSQL_TEST_USER, MYSQL_TEST_PASSWORD);
+ protected final UniqueDatabase extremeRouteTestDatabase =
+ new UniqueDatabase(MYSQL, "extreme_route_test", MYSQL_TEST_USER, MYSQL_TEST_PASSWORD);
+
@Before
public void before() throws Exception {
super.before();
@@ -303,9 +307,9 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
generateSchemaChanges();
validateResult(
- "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=AFTER, existedColumnName=VERSION}]}",
"DataChangeEvent{tableId=%s.ALL, before=[], after=[10001, 12, Derrida], op=INSERT, meta=()}",
- "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=AFTER, existedColumnName=NAME}]}",
"DataChangeEvent{tableId=%s.ALL, before=[], after=[10002, null, null, 15], op=INSERT, meta=()}",
"AlterColumnTypeEvent{tableId=%s.ALL, typeMapping={VERSION=STRING}, oldTypeMapping={VERSION=VARCHAR(17)}}",
"DataChangeEvent{tableId=%s.ALL, before=[], after=[10003, null, null, Fluorite], op=INSERT, meta=()}",
@@ -397,9 +401,9 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
generateSchemaChanges();
validateResult(
- "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=AFTER, existedColumnName=VERSION}]}",
"DataChangeEvent{tableId=NEW_%s.ALPHABET, before=[], after=[10001, 12, Derrida], op=INSERT, meta=()}",
- "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=AFTER, existedColumnName=NAME}]}",
"DataChangeEvent{tableId=NEW_%s.ALPHABET, before=[], after=[10002, null, null, 15], op=INSERT, meta=()}",
"AlterColumnTypeEvent{tableId=%s.TABLEGAMMA, typeMapping={VERSION=VARCHAR(19)}, oldTypeMapping={VERSION=VARCHAR(17)}}",
"RenameColumnEvent{tableId=%s.TABLEGAMMA, nameMapping={VERSION=VERSION_EX}}",
@@ -502,10 +506,10 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
generateSchemaChanges();
validateResult(
- "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=AFTER, existedColumnName=VERSION}]}",
"DataChangeEvent{tableId=NEW_%s.ALPHABET, before=[], after=[10001, 12, Derrida], op=INSERT, meta=()}",
- "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=LAST, existedColumnName=null}]}",
- "AddColumnEvent{tableId=NEW_%s.BETAGAMM, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=NEW_%s.ALPHABET, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=AFTER, existedColumnName=NAME}]}",
+ "AddColumnEvent{tableId=NEW_%s.BETAGAMM, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=AFTER, existedColumnName=VERSION}]}",
"DataChangeEvent{tableId=NEW_%s.ALPHABET, before=[], after=[10002, null, null, 15], op=INSERT, meta=()}",
"DataChangeEvent{tableId=NEW_%s.BETAGAMM, before=[], after=[10002, null, 15], op=INSERT, meta=()}",
"AlterColumnTypeEvent{tableId=NEW_%s.BETAGAMM, typeMapping={VERSION=STRING}, oldTypeMapping={VERSION=VARCHAR(17)}}",
@@ -708,9 +712,9 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
generateSchemaChanges();
validateResult(
- "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`NAME` VARCHAR(17), position=AFTER, existedColumnName=EXTRAS}]}",
"DataChangeEvent{tableId=%s.ALL, before=[], after=[10001, 12, extras, Derrida], op=INSERT, meta=()}",
- "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=LAST, existedColumnName=null}]}",
+ "AddColumnEvent{tableId=%s.ALL, addedColumns=[ColumnWithPosition{column=`VERSION_EX` VARCHAR(17), position=AFTER, existedColumnName=NAME}]}",
"DataChangeEvent{tableId=%s.ALL, before=[], after=[10002, null, extras, null, 15], op=INSERT, meta=()}",
"AlterColumnTypeEvent{tableId=%s.ALL, typeMapping={VERSION=STRING}, oldTypeMapping={VERSION=VARCHAR(17)}}",
"DataChangeEvent{tableId=%s.ALL, before=[], after=[10003, null, extras, null, Fluorite], op=INSERT, meta=()}",
@@ -814,15 +818,108 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
"DataChangeEvent{tableId=NEW_%s.NEW_TABLEDELTA, before=[], after=[10004], op=INSERT, meta=()}");
}
+ @Test
+ public void testExtremeMergeTableRoute() throws Exception {
+ final String databaseName = extremeRouteTestDatabase.getDatabaseName();
+ try (Connection conn =
+ DriverManager.getConnection(
+ MYSQL.getJdbcUrl(), MYSQL_TEST_USER, MYSQL_TEST_PASSWORD);
+ Statement stat = conn.createStatement()) {
+ stat.execute(String.format("CREATE DATABASE %s;", databaseName));
+ stat.execute(String.format("USE %s;", databaseName));
+ for (int i = 1; i <= TEST_TABLE_NUMBER; i++) {
+ stat.execute(String.format("DROP TABLE IF EXISTS TABLE%d;", i));
+ stat.execute(
+ String.format(
+ "CREATE TABLE TABLE%d (ID INT NOT NULL PRIMARY KEY,VERSION VARCHAR(17));",
+ i));
+ stat.execute(String.format("INSERT INTO TABLE%d VALUES (%d, 'No.%d');", i, i, i));
+ }
+ } catch (SQLException e) {
+ LOG.error("Initialize table failed.", e);
+ throw e;
+ }
+ LOG.info("Table initialized successfully.");
+
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: mysql\n"
+ + " hostname: %s\n"
+ + " port: 3306\n"
+ + " username: %s\n"
+ + " password: %s\n"
+ + " tables: %s.\\.*\n"
+ + " server-id: 5400-5404\n"
+ + " server-time-zone: UTC\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d",
+ INTER_CONTAINER_MYSQL_ALIAS,
+ MYSQL_TEST_USER,
+ MYSQL_TEST_PASSWORD,
+ databaseName,
+ parallelism);
+ Path mysqlCdcJar = TestUtils.getResource("mysql-cdc-pipeline-connector.jar");
+ Path valuesCdcJar = TestUtils.getResource("values-cdc-pipeline-connector.jar");
+ Path mysqlDriverJar = TestUtils.getResource("mysql-driver.jar");
+ submitPipelineJob(pipelineJob, mysqlCdcJar, valuesCdcJar, mysqlDriverJar);
+ waitUntilJobRunning(Duration.ofSeconds(30));
+
+ // In single parallelism mode, sink will not print out the "subTaskId>" prefix.
+ String prefix = parallelism > 1 ? "> " : "";
+
+ LOG.info("Verifying CreateTableEvents...");
+ validateResult(
+ 180_000L,
+ IntStream.rangeClosed(1, TEST_TABLE_NUMBER)
+ .mapToObj(
+ i ->
+ String.format(
+ prefix
+ + "CreateTableEvent{tableId=%s.TABLE%d, schema=columns={`ID` INT NOT NULL,`VERSION` VARCHAR(17)}, primaryKeys=ID, options=()}",
+ databaseName,
+ i))
+ .toArray(String[]::new));
+
+ LOG.info("Verifying DataChangeEvents...");
+ validateResult(
+ 180_000L,
+ IntStream.rangeClosed(1, TEST_TABLE_NUMBER)
+ .mapToObj(
+ i ->
+ String.format(
+ prefix
+ + "DataChangeEvent{tableId=%s.TABLE%d, before=[], after=[%d, No.%d], op=INSERT, meta=()}",
+ databaseName,
+ i,
+ i,
+ i))
+ .toArray(String[]::new));
+ extremeRouteTestDatabase.dropDatabase();
+ }
+
private void validateResult(String... expectedEvents) throws Exception {
+ validateResult(EVENT_DEFAULT_TIMEOUT, expectedEvents);
+ }
+
+ private void validateResult(long timeout, String... expectedEvents) throws Exception {
for (String event : expectedEvents) {
- waitUntilSpecificEvent(String.format(event, routeTestDatabase.getDatabaseName()));
+ waitUntilSpecificEvent(
+ timeout, String.format(event, routeTestDatabase.getDatabaseName()));
}
}
private void waitUntilSpecificEvent(String event) throws Exception {
+ waitUntilSpecificEvent(EVENT_DEFAULT_TIMEOUT, event);
+ }
+
+ private void waitUntilSpecificEvent(long timeout, String event) throws Exception {
boolean result = false;
- long endTimeout = System.currentTimeMillis() + EVENT_DEFAULT_TIMEOUT;
+ long endTimeout = System.currentTimeMillis() + timeout;
while (System.currentTimeMillis() < endTimeout) {
String stdout = taskManagerConsumer.toUtf8String();
if (stdout.contains(event + "\n")) {
@@ -839,8 +936,4 @@ public class RouteE2eITCase extends PipelineTestEnvironment {
+ taskManagerConsumer.toUtf8String());
}
}
-
- private void assertNotExists(String event) {
- Assert.assertFalse(taskManagerConsumer.toUtf8String().contains(event));
- }
}
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java
index 0b514c3b9..98e4deba9 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java
@@ -113,10 +113,14 @@ public class SchemaEvolveE2eITCase extends PipelineTestEnvironment {
true,
false,
false,
- Collections.emptyList(),
Arrays.asList(
- "java.lang.IllegalStateException: Incompatible types found for column `age`: \"INT\" and \"DOUBLE\"",
- "org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy"));
+ "AddColumnEvent{tableId=%s.merged, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}",
+ "DataChangeEvent{tableId=%s.merged, before=[], after=[1012, Eve, 17, 0], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId=%s.merged, typeMapping={age=DOUBLE}, oldTypeMapping={age=INT}}",
+ "AddColumnEvent{tableId=%s.merged, addedColumns=[ColumnWithPosition{column=`precise_age` DOUBLE, position=AFTER, existedColumnName=gender}]}",
+ "AddColumnEvent{tableId=%s.merged, addedColumns=[ColumnWithPosition{column=`biological_sex` TINYINT, position=AFTER, existedColumnName=precise_age}]}",
+ "DataChangeEvent{tableId=%s.merged, before=[], after=[1013, Fiona, null, null, 16.0, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=%s.merged, before=[], after=[1014, Gem, null, null, 17.0, null], op=INSERT, meta=()}"));
}
@Test
@@ -128,8 +132,8 @@ public class SchemaEvolveE2eITCase extends PipelineTestEnvironment {
false,
Collections.emptyList(),
Arrays.asList(
- "Failed to apply schema change AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]} to table %s.members. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabl [...]
- "UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Caused by: org.apache.flink.util.FlinkRuntimeException: Failed to apply schema change event.",
+ "Caused by: org.apache.flink.cdc.common.exceptions.UnsupportedSchemaChangeEventException",
"org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy"));
}
@@ -143,10 +147,16 @@ public class SchemaEvolveE2eITCase extends PipelineTestEnvironment {
Arrays.asList(
// Add column never succeeded, so age column will not appear.
"DataChangeEvent{tableId=%s.members, before=[], after=[1012, Eve, 17], op=INSERT, meta=()}",
- "DataChangeEvent{tableId=%s.members, before=[], after=[1013, Fiona, null], op=INSERT, meta=()}"),
+ "DataChangeEvent{tableId=%s.members, before=[], after=[1013, Fiona, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=%s.members, before=[], after=[1014, Gem, null], op=INSERT, meta=()}"),
Arrays.asList(
- "Failed to apply schema change AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]} to table %s.members. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabl [...]
- "UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}"));
+ "Failed to apply schema change AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema [...]
+ "Failed to apply schema change AlterColumnTypeEvent{tableId=%s.members, typeMapping={age=DOUBLE}, oldTypeMapping={age=INT}}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AlterColumnTypeEvent{tableId=%s.members, typeMapping={age=DOUBLE}, oldTypeMapping={age=INT}}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change RenameColumnEvent{tableId=%s.members, nameMapping={age=precise_age}}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=RenameColumnEvent{tableId=%s.members, nameMapping={age=precise_age}}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change RenameColumnEvent{tableId=%s.members, nameMapping={gender=biological_sex}}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=RenameColumnEvent{tableId=%s.members, nameMapping={gender=biological_sex}}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change TruncateTableEvent{tableId=%s.members}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=TruncateTableEvent{tableId=%s.members}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change DropTableEvent{tableId=%s.members}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=DropTableEvent{tableId=%s.members}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}"));
}
@Test
@@ -171,8 +181,9 @@ public class SchemaEvolveE2eITCase extends PipelineTestEnvironment {
false,
false,
Collections.emptyList(),
- Collections.singletonList(
- "java.lang.RuntimeException: Refused to apply schema change event AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]} in EXCEPTION mode."));
+ Arrays.asList(
+ "An exception was triggered from Schema change applying task. Job will fail now.",
+ "org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy"));
}
@Test
@@ -216,8 +227,8 @@ public class SchemaEvolveE2eITCase extends PipelineTestEnvironment {
"TruncateTableEvent{tableId=%s.members}",
"DataChangeEvent{tableId=%s.members, before=[], after=[1014, Gem, 17.0, null], op=INSERT, meta=()}"),
Arrays.asList(
- "Ignored schema change DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]} to table %s.members.",
- "Ignored schema change DropTableEvent{tableId=%s.members} to table %s.members."));
+ "Ignored schema change DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]}.",
+ "Ignored schema change DropTableEvent{tableId=%s.members}."));
}
@Test
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java
index 4f40d8972..7523387cf 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java
@@ -113,10 +113,13 @@ public class SchemaEvolvingTransformE2eITCase extends PipelineTestEnvironment {
true,
false,
false,
- Collections.emptyList(),
Arrays.asList(
- "java.lang.IllegalStateException: Incompatible types found for column `age`: \"INT\" and \"DOUBLE\"",
- "org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy"));
+ "AddColumnEvent{tableId=%s.merged, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=tag}]}",
+ "DataChangeEvent{tableId=%s.merged, before=[], after=[1012 -> Eve, 1012, Eve, 17, 1024144, age < 20, 0], op=INSERT, meta=()}",
+ "AlterColumnTypeEvent{tableId=%s.merged, typeMapping={age=DOUBLE}, oldTypeMapping={age=INT}}",
+ "AddColumnEvent{tableId=%s.merged, addedColumns=[ColumnWithPosition{column=`biological_sex` TINYINT, position=AFTER, existedColumnName=gender}]}",
+ "DataChangeEvent{tableId=%s.merged, before=[], after=[1013 -> Fiona, 1013, Fiona, 16.0, 1026169, age < 20, null, null], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=%s.merged, before=[], after=[1014 -> Gem, 1014, Gem, 17.0, 1028196, age < 20, null, null], op=INSERT, meta=()}"));
}
@Test
@@ -128,8 +131,9 @@ public class SchemaEvolvingTransformE2eITCase extends PipelineTestEnvironment {
false,
Collections.emptyList(),
Arrays.asList(
- "Failed to apply schema change AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]} to table %s.members. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabl [...]
- "UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "An exception was triggered from Schema change applying task. Job will fail now.",
+ "org.apache.flink.util.FlinkRuntimeException: Failed to apply schema change event.",
+ "Caused by: org.apache.flink.cdc.common.exceptions.UnsupportedSchemaChangeEventException",
"org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy"));
}
@@ -146,8 +150,12 @@ public class SchemaEvolvingTransformE2eITCase extends PipelineTestEnvironment {
"DataChangeEvent{tableId=%s.members, before=[], after=[1013 -> Fiona, 1013, Fiona, null, 1026169, age < 20], op=INSERT, meta=()}",
"DataChangeEvent{tableId=%s.members, before=[], after=[1014 -> Gem, 1014, Gem, null, 1028196, age < 20], op=INSERT, meta=()}"),
Arrays.asList(
- "Failed to apply schema change AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]} to table %s.members. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabl [...]
- "UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}"));
+ "Failed to apply schema change AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]}, exceptionMessage='Rejected schema change event since error.on.schema [...]
+ "Failed to apply schema change AlterColumnTypeEvent{tableId=%s.members, typeMapping={age=DOUBLE}, oldTypeMapping={age=INT}}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=AlterColumnTypeEvent{tableId=%s.members, typeMapping={age=DOUBLE}, oldTypeMapping={age=INT}}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change RenameColumnEvent{tableId=%s.members, nameMapping={gender=biological_sex}}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=RenameColumnEvent{tableId=%s.members, nameMapping={gender=biological_sex}}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change TruncateTableEvent{tableId=%s.members}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=TruncateTableEvent{tableId=%s.members}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}",
+ "Failed to apply schema change DropTableEvent{tableId=%s.members}, but keeps running in tolerant mode. Caused by: UnsupportedSchemaChangeEventException{applyingEvent=DropTableEvent{tableId=%s.members}, exceptionMessage='Rejected schema change event since error.on.schema.change is enabled.', cause='null'}"));
}
@Test
@@ -172,8 +180,9 @@ public class SchemaEvolvingTransformE2eITCase extends PipelineTestEnvironment {
false,
false,
Collections.emptyList(),
- Collections.singletonList(
- "java.lang.RuntimeException: Refused to apply schema change event AddColumnEvent{tableId=%s.members, addedColumns=[ColumnWithPosition{column=`gender` TINYINT, position=AFTER, existedColumnName=age}]} in EXCEPTION mode."));
+ Arrays.asList(
+ "An exception was triggered from Schema change applying task. Job will fail now.",
+ "org.apache.flink.util.FlinkRuntimeException: Failed to apply schema change event."));
}
@Test
@@ -209,8 +218,8 @@ public class SchemaEvolvingTransformE2eITCase extends PipelineTestEnvironment {
"TruncateTableEvent{tableId=%s.members}",
"DataChangeEvent{tableId=%s.members, before=[], after=[1014 -> Gem, 1014, Gem, 17.0, null, 1028196, age < 20], op=INSERT, meta=()}"),
Arrays.asList(
- "Ignored schema change DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]} to table %s.members.",
- "Ignored schema change DropTableEvent{tableId=%s.members} to table %s.members."));
+ "Ignored schema change DropColumnEvent{tableId=%s.members, droppedColumnNames=[biological_sex]}.",
+ "Ignored schema change DropTableEvent{tableId=%s.members}."));
}
@Test
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java
index c125de59b..8b292ff85 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java
@@ -1215,8 +1215,14 @@ public class TransformE2eITCase extends PipelineTestEnvironment {
boolean extractDataLines(String line) {
// In multiple parallelism mode, a prefix with subTaskId (like '1> ') will be appended.
// Should trim it before extracting data fields.
- if (!line.startsWith("DataChangeEvent{", 3)) {
- return false;
+ if (parallelism > 1) {
+ if (!line.startsWith("DataChangeEvent{", 3)) {
+ return false;
+ }
+ } else {
+ if (!line.startsWith("DataChangeEvent{")) {
+ return false;
+ }
}
Stream.of("before", "after")
.forEach(
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
index a1c3c482b..eb7798e0c 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
@@ -68,7 +68,18 @@ public abstract class PipelineTestEnvironment extends TestLogger {
@Parameterized.Parameter public String flinkVersion;
- public Integer parallelism = 4;
+ public Integer parallelism = getParallelism();
+
+ private int getParallelism() {
+ try {
+ return Integer.parseInt(System.getProperty("specifiedParallelism"));
+ } catch (NumberFormatException ex) {
+ LOG.warn(
+ "Unable to parse specified parallelism configuration ({} provided). Use 4 by default.",
+ System.getProperty("specifiedParallelism"));
+ return 4;
+ }
+ }
// ------------------------------------------------------------------------------------------
// Flink Variables
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/CoordinationResponseUtils.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/CoordinationResponseUtils.java
similarity index 99%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/CoordinationResponseUtils.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/CoordinationResponseUtils.java
index 4f13e928e..2bedb62a7 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/CoordinationResponseUtils.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/CoordinationResponseUtils.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common;
import org.apache.flink.api.common.typeutils.SimpleTypeSerializerSnapshot;
import org.apache.flink.api.common.typeutils.TypeSerializer;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/CoordinatorExecutorThreadFactory.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/CoordinatorExecutorThreadFactory.java
new file mode 100644
index 000000000..579495667
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/CoordinatorExecutorThreadFactory.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.util.FatalExitExceptionHandler;
+
+import java.util.concurrent.ThreadFactory;
+
+/** A thread factory class that provides some helper methods. */
+public class CoordinatorExecutorThreadFactory implements ThreadFactory {
+
+ private final String coordinatorThreadName;
+ private final ClassLoader cl;
+ private final Thread.UncaughtExceptionHandler errorHandler;
+
+ private Thread t;
+
+ public CoordinatorExecutorThreadFactory(
+ final String coordinatorThreadName, final ClassLoader contextClassLoader) {
+ this(coordinatorThreadName, contextClassLoader, FatalExitExceptionHandler.INSTANCE);
+ }
+
+ public CoordinatorExecutorThreadFactory(
+ final String coordinatorThreadName,
+ final ClassLoader contextClassLoader,
+ final Thread.UncaughtExceptionHandler errorHandler) {
+ this.coordinatorThreadName = coordinatorThreadName;
+ this.cl = contextClassLoader;
+ this.errorHandler = errorHandler;
+ }
+
+ @Override
+ public synchronized Thread newThread(Runnable r) {
+ if (t != null) {
+ throw new Error(
+ "This indicates that a fatal error has happened and caused the "
+ + "coordinator executor thread to exit. Check the earlier logs"
+ + "to see the root cause of the problem.");
+ }
+ t = new Thread(r, coordinatorThreadName);
+ t.setContextClassLoader(cl);
+ t.setUncaughtExceptionHandler(errorHandler);
+ return t;
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaDerivator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaDerivator.java
new file mode 100755
index 000000000..445387b02
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaDerivator.java
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.cdc.common.data.RecordData;
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.DropColumnEvent;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEventWithPreSchema;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.utils.SchemaMergingUtils;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+
+import org.apache.flink.shaded.guava31.com.google.common.cache.CacheBuilder;
+import org.apache.flink.shaded.guava31.com.google.common.cache.CacheLoader;
+import org.apache.flink.shaded.guava31.com.google.common.cache.LoadingCache;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Table;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** A utility class to derive how evolved schemas should change. */
+public class SchemaDerivator {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaDerivator.class);
+
+ private static final Duration CACHE_EXPIRE_DURATION = Duration.ofDays(1);
+ private final LoadingCache<Schema, List<RecordData.FieldGetter>> upstreamRecordGetterCache;
+ private final LoadingCache<Schema, BinaryRecordDataGenerator> evolvedRecordWriterCache;
+
+ public SchemaDerivator() {
+ upstreamRecordGetterCache =
+ CacheBuilder.newBuilder()
+ .expireAfterAccess(CACHE_EXPIRE_DURATION)
+ .build(
+ new CacheLoader<Schema, List<RecordData.FieldGetter>>() {
+ @Override
+ public @Nonnull List<RecordData.FieldGetter> load(
+ @Nonnull Schema schema) {
+ return SchemaUtils.createFieldGetters(schema);
+ }
+ });
+
+ evolvedRecordWriterCache =
+ CacheBuilder.newBuilder()
+ .expireAfterAccess(CACHE_EXPIRE_DURATION)
+ .build(
+ new CacheLoader<Schema, BinaryRecordDataGenerator>() {
+ @Override
+ public @Nonnull BinaryRecordDataGenerator load(
+ @Nonnull Schema schema) {
+ return new BinaryRecordDataGenerator(
+ schema.getColumnDataTypes()
+ .toArray(new DataType[0]));
+ }
+ });
+ }
+
+ /** Get affected evolved table IDs based on changed upstream tables. */
+ public static Set<TableId> getAffectedEvolvedTables(
+ final TableIdRouter tableIdRouter, final Set<TableId> changedUpstreamTables) {
+ return changedUpstreamTables.stream()
+ .flatMap(cut -> tableIdRouter.route(cut).stream())
+ .collect(Collectors.toSet());
+ }
+
+ /** For an evolved table ID, reverse lookup all upstream tables that it depends on. */
+ public static Set<TableId> reverseLookupDependingUpstreamTables(
+ final TableIdRouter tableIdRouter,
+ final TableId evolvedTableId,
+ final Set<TableId> upstreamSchemaTables) {
+ return upstreamSchemaTables.stream()
+ .filter(kut -> tableIdRouter.route(kut).contains(evolvedTableId))
+ .collect(Collectors.toSet());
+ }
+
+ /** For an evolved table ID, reverse lookup all upstream tables that it depends on. */
+ public static Set<TableId> reverseLookupDependingUpstreamTables(
+ final TableIdRouter tableIdRouter,
+ final TableId evolvedTableId,
+ final Table<TableId, Integer, Schema> upstreamSchemaTable) {
+ return upstreamSchemaTable.rowKeySet().stream()
+ .filter(kut -> tableIdRouter.route(kut).contains(evolvedTableId))
+ .collect(Collectors.toSet());
+ }
+
+ /** For an evolved table ID, reverse lookup all upstream schemas that needs to be fit in. */
+ public static Set<Schema> reverseLookupDependingUpstreamSchemas(
+ final TableIdRouter tableIdRouter,
+ final TableId evolvedTableId,
+ final SchemaManager schemaManager) {
+ return reverseLookupDependingUpstreamTables(
+ tableIdRouter, evolvedTableId, schemaManager.getAllOriginalTables())
+ .stream()
+ .map(utid -> schemaManager.getLatestOriginalSchema(utid).get())
+ .collect(Collectors.toSet());
+ }
+
+ /** For an evolved table ID, reverse lookup all upstream schemas that needs to be fit in. */
+ public static Set<Schema> reverseLookupDependingUpstreamSchemas(
+ final TableIdRouter tableIdRouter,
+ final TableId evolvedTableId,
+ final Table<TableId, Integer, Schema> upstreamSchemaTable) {
+ return reverseLookupDependingUpstreamTables(
+ tableIdRouter, evolvedTableId, upstreamSchemaTable)
+ .stream()
+ .flatMap(utid -> upstreamSchemaTable.row(utid).values().stream())
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Rewrite {@link SchemaChangeEvent}s by current {@link SchemaChangeBehavior} and include /
+ * exclude them by fine-grained schema change event configurations.
+ */
+ public static List<SchemaChangeEvent> normalizeSchemaChangeEvents(
+ Schema oldSchema,
+ List<SchemaChangeEvent> schemaChangeEvents,
+ SchemaChangeBehavior schemaChangeBehavior,
+ MetadataApplier metadataApplier) {
+ List<SchemaChangeEvent> rewrittenSchemaChangeEvents =
+ rewriteSchemaChangeEvents(oldSchema, schemaChangeEvents, schemaChangeBehavior);
+ rewrittenSchemaChangeEvents.forEach(
+ evt -> {
+ if (evt instanceof SchemaChangeEventWithPreSchema) {
+ SchemaChangeEventWithPreSchema eventNeedsPreSchema =
+ (SchemaChangeEventWithPreSchema) evt;
+ if (!eventNeedsPreSchema.hasPreSchema()) {
+ eventNeedsPreSchema.fillPreSchema(oldSchema);
+ }
+ }
+ });
+
+ List<SchemaChangeEvent> finalSchemaChangeEvents = new ArrayList<>();
+ for (SchemaChangeEvent schemaChangeEvent : rewrittenSchemaChangeEvents) {
+ if (metadataApplier.acceptsSchemaEvolutionType(schemaChangeEvent.getType())) {
+ finalSchemaChangeEvents.add(schemaChangeEvent);
+ } else {
+ LOG.info("Ignored schema change {}.", schemaChangeEvent);
+ }
+ }
+ return finalSchemaChangeEvents;
+ }
+
+ private static List<SchemaChangeEvent> rewriteSchemaChangeEvents(
+ Schema oldSchema,
+ List<SchemaChangeEvent> schemaChangeEvents,
+ SchemaChangeBehavior schemaChangeBehavior) {
+ switch (schemaChangeBehavior) {
+ case EVOLVE:
+ case TRY_EVOLVE:
+ case EXCEPTION:
+ return schemaChangeEvents;
+ case LENIENT:
+ return schemaChangeEvents.stream()
+ .flatMap(evt -> lenientizeSchemaChangeEvent(oldSchema, evt))
+ .collect(Collectors.toList());
+ case IGNORE:
+ return schemaChangeEvents.stream()
+ .filter(e -> e instanceof CreateTableEvent)
+ .collect(Collectors.toList());
+ default:
+ throw new IllegalArgumentException(
+ "Unexpected schema change behavior: " + schemaChangeBehavior);
+ }
+ }
+
+ private static Stream<SchemaChangeEvent> lenientizeSchemaChangeEvent(
+ Schema oldSchema, SchemaChangeEvent schemaChangeEvent) {
+ TableId tableId = schemaChangeEvent.tableId();
+ switch (schemaChangeEvent.getType()) {
+ case ADD_COLUMN:
+ return lenientizeAddColumnEvent((AddColumnEvent) schemaChangeEvent, tableId);
+ case DROP_COLUMN:
+ return lenientizeDropColumnEvent(
+ oldSchema, (DropColumnEvent) schemaChangeEvent, tableId);
+ case RENAME_COLUMN:
+ return lenientizeRenameColumnEvent(
+ oldSchema, (RenameColumnEvent) schemaChangeEvent, tableId);
+ default:
+ return Stream.of(schemaChangeEvent);
+ }
+ }
+
+ private static Stream<SchemaChangeEvent> lenientizeRenameColumnEvent(
+ Schema oldSchema, RenameColumnEvent schemaChangeEvent, TableId tableId) {
+ List<AddColumnEvent.ColumnWithPosition> appendColumns = new ArrayList<>();
+ Map<String, DataType> convertNullableColumns = new HashMap<>();
+ schemaChangeEvent
+ .getNameMapping()
+ .forEach(
+ (oldColName, newColName) -> {
+ Column column =
+ oldSchema
+ .getColumn(oldColName)
+ .orElseThrow(
+ () ->
+ new IllegalArgumentException(
+ "Non-existed column "
+ + oldColName
+ + " in evolved schema."));
+ if (!column.getType().isNullable()) {
+ // It's a not-nullable column, we need to cast it to
+ // nullable first
+ convertNullableColumns.put(oldColName, column.getType().nullable());
+ }
+ appendColumns.add(
+ new AddColumnEvent.ColumnWithPosition(
+ column.copy(newColName)
+ .copy(column.getType().nullable())));
+ });
+
+ List<SchemaChangeEvent> events = new ArrayList<>();
+ events.add(new AddColumnEvent(tableId, appendColumns));
+ if (!convertNullableColumns.isEmpty()) {
+ events.add(new AlterColumnTypeEvent(tableId, convertNullableColumns));
+ }
+ return events.stream();
+ }
+
+ private static Stream<SchemaChangeEvent> lenientizeDropColumnEvent(
+ Schema oldSchema, DropColumnEvent schemaChangeEvent, TableId tableId) {
+ Map<String, DataType> convertNullableColumns =
+ schemaChangeEvent.getDroppedColumnNames().stream()
+ .map(oldSchema::getColumn)
+ .flatMap(e -> e.map(Stream::of).orElse(Stream.empty()))
+ .filter(col -> !col.getType().isNullable())
+ .collect(
+ Collectors.toMap(
+ Column::getName, column -> column.getType().nullable()));
+
+ if (convertNullableColumns.isEmpty()) {
+ return Stream.empty();
+ } else {
+ return Stream.of(new AlterColumnTypeEvent(tableId, convertNullableColumns));
+ }
+ }
+
+ private static Stream<SchemaChangeEvent> lenientizeAddColumnEvent(
+ AddColumnEvent schemaChangeEvent, TableId tableId) {
+ return Stream.of(
+ new AddColumnEvent(
+ tableId,
+ schemaChangeEvent.getAddedColumns().stream()
+ .map(
+ col ->
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ col.getAddColumn().getName(),
+ col.getAddColumn()
+ .getType()
+ .nullable(),
+ col.getAddColumn().getComment(),
+ col.getAddColumn()
+ .getDefaultValueExpression())))
+ .collect(Collectors.toList())));
+ }
+
+ /** Coerce a {@link DataChangeEvent} from upstream to expected downstream schema. */
+ public Optional<DataChangeEvent> coerceDataRecord(
+ String timezone,
+ DataChangeEvent dataChangeEvent,
+ Schema upstreamSchema,
+ @Nullable Schema evolvedSchema) {
+ if (evolvedSchema == null) {
+ // Sink does not recognize this tableId, might have been dropped, just ignore it.
+ return Optional.empty();
+ }
+
+ if (upstreamSchema.equals(evolvedSchema)) {
+ // If there's no schema difference, just return the original event.
+ return Optional.of(dataChangeEvent);
+ }
+
+ List<RecordData.FieldGetter> upstreamSchemaReader =
+ upstreamRecordGetterCache.getUnchecked(upstreamSchema);
+ SchemaUtils.createFieldGetters(upstreamSchema);
+ BinaryRecordDataGenerator evolvedSchemaWriter =
+ evolvedRecordWriterCache.getUnchecked(evolvedSchema);
+
+ // Coerce binary data records
+ if (dataChangeEvent.before() != null) {
+ List<Object> upstreamFields =
+ SchemaUtils.restoreOriginalData(dataChangeEvent.before(), upstreamSchemaReader);
+ Object[] coercedRow =
+ SchemaMergingUtils.coerceRow(
+ timezone, evolvedSchema, upstreamSchema, upstreamFields);
+
+ dataChangeEvent =
+ DataChangeEvent.projectBefore(
+ dataChangeEvent, evolvedSchemaWriter.generate(coercedRow));
+ }
+
+ if (dataChangeEvent.after() != null) {
+ List<Object> upstreamFields =
+ SchemaUtils.restoreOriginalData(dataChangeEvent.after(), upstreamSchemaReader);
+ Object[] coercedRow =
+ SchemaMergingUtils.coerceRow(
+ timezone, evolvedSchema, upstreamSchema, upstreamFields);
+
+ dataChangeEvent =
+ DataChangeEvent.projectAfter(
+ dataChangeEvent, evolvedSchemaWriter.generate(coercedRow));
+ }
+
+ return Optional.of(dataChangeEvent);
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaManager.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaManager.java
similarity index 69%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaManager.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaManager.java
index 60e70b7ed..1df4279c9 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaManager.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaManager.java
@@ -15,18 +15,15 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
+package org.apache.flink.cdc.runtime.operators.schema.common;
import org.apache.flink.cdc.common.annotation.Internal;
-import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.annotation.VisibleForTesting;
import org.apache.flink.cdc.common.event.CreateTableEvent;
import org.apache.flink.cdc.common.event.SchemaChangeEvent;
import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.common.event.visitor.SchemaChangeEventVisitor;
import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
-import org.apache.flink.cdc.common.schema.Column;
import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.common.types.DataType;
import org.apache.flink.cdc.common.utils.SchemaUtils;
import org.apache.flink.cdc.runtime.serializer.TableIdSerializer;
import org.apache.flink.cdc.runtime.serializer.schema.SchemaSerializer;
@@ -34,22 +31,20 @@ import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Objects;
import java.util.Optional;
+import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import static org.apache.flink.cdc.common.utils.Preconditions.checkArgument;
@@ -59,7 +54,6 @@ import static org.apache.flink.cdc.common.utils.Preconditions.checkArgument;
*/
@Internal
public class SchemaManager {
- private static final Logger LOG = LoggerFactory.getLogger(SchemaManager.class);
private static final int INITIAL_SCHEMA_VERSION = 0;
private static final int VERSIONS_TO_KEEP = 3;
private final SchemaChangeBehavior behavior;
@@ -74,23 +68,17 @@ public class SchemaManager {
private final Map<TableId, SortedMap<Integer, Schema>> evolvedSchemas;
public SchemaManager() {
- evolvedSchemas = new HashMap<>();
- originalSchemas = new HashMap<>();
+ evolvedSchemas = new ConcurrentHashMap<>();
+ originalSchemas = new ConcurrentHashMap<>();
behavior = SchemaChangeBehavior.EVOLVE;
}
- public SchemaManager(SchemaChangeBehavior behavior) {
- evolvedSchemas = new HashMap<>();
- originalSchemas = new HashMap<>();
- this.behavior = behavior;
- }
-
public SchemaManager(
Map<TableId, SortedMap<Integer, Schema>> originalSchemas,
Map<TableId, SortedMap<Integer, Schema>> evolvedSchemas,
SchemaChangeBehavior behavior) {
- this.evolvedSchemas = evolvedSchemas;
- this.originalSchemas = originalSchemas;
+ this.evolvedSchemas = new ConcurrentHashMap<>(evolvedSchemas);
+ this.originalSchemas = new ConcurrentHashMap<>(originalSchemas);
this.behavior = behavior;
}
@@ -98,97 +86,6 @@ public class SchemaManager {
return behavior;
}
- /**
- * This function checks if the given schema change event has been applied already. If so, it
- * will be ignored to avoid sending duplicate evolved schema change events to sink metadata
- * applier.
- */
- public final boolean isOriginalSchemaChangeEventRedundant(SchemaChangeEvent event) {
- TableId tableId = event.tableId();
- Optional<Schema> latestSchema = getLatestOriginalSchema(tableId);
- return Boolean.TRUE.equals(
- SchemaChangeEventVisitor.visit(
- event,
- addColumnEvent -> {
- // It has not been applied if schema does not even exist
- if (!latestSchema.isPresent()) {
- return false;
- }
- List<Column> existedColumns = latestSchema.get().getColumns();
-
- // It has been applied only if all columns are present in existedColumns
- for (AddColumnEvent.ColumnWithPosition column :
- addColumnEvent.getAddedColumns()) {
- if (!existedColumns.contains(column.getAddColumn())) {
- return false;
- }
- }
- return true;
- },
- alterColumnTypeEvent -> {
- // It has not been applied if schema does not even exist
- if (!latestSchema.isPresent()) {
- return false;
- }
- Schema schema = latestSchema.get();
-
- // It has been applied only if all column types are set as expected
- for (Map.Entry<String, DataType> entry :
- alterColumnTypeEvent.getTypeMapping().entrySet()) {
- if (!schema.getColumn(entry.getKey()).isPresent()
- || !schema.getColumn(entry.getKey())
- .get()
- .getType()
- .equals(entry.getValue())) {
- return false;
- }
- }
- return true;
- },
- createTableEvent -> {
- // It has been applied if such table already exists
- return latestSchema.isPresent();
- },
- dropColumnEvent -> {
- // It has not been applied if schema does not even exist
- if (!latestSchema.isPresent()) {
- return false;
- }
- List<String> existedColumnNames = latestSchema.get().getColumnNames();
-
- // It has been applied only if corresponding column types do not exist
- return dropColumnEvent.getDroppedColumnNames().stream()
- .noneMatch(existedColumnNames::contains);
- },
- dropTableEvent -> {
- // It has been applied if such table does not exist
- return !latestSchema.isPresent();
- },
- renameColumnEvent -> {
- // It has been applied if such table already exists
- if (!latestSchema.isPresent()) {
- return false;
- }
- List<String> existedColumnNames = latestSchema.get().getColumnNames();
-
- // It has been applied only if all previous names do not exist, and all
- // new names already exist
- for (Map.Entry<String, String> entry :
- renameColumnEvent.getNameMapping().entrySet()) {
- if (existedColumnNames.contains(entry.getKey())
- || !existedColumnNames.contains(entry.getValue())) {
- return false;
- }
- }
- return true;
- },
- truncateTableEvent -> {
- // We have no way to ensure if a TruncateTableEvent has been applied
- // before. Just assume it's not.
- return false;
- }));
- }
-
public final boolean schemaExists(
Map<TableId, SortedMap<Integer, Schema>> schemaMap, TableId tableId) {
return schemaMap.containsKey(tableId) && !schemaMap.get(tableId).isEmpty();
@@ -202,6 +99,10 @@ public class SchemaManager {
return schemaExists(evolvedSchemas, tableId);
}
+ public final Set<TableId> getAllOriginalTables() {
+ return originalSchemas.keySet();
+ }
+
/** Get the latest evolved schema of the specified table. */
public Optional<Schema> getLatestEvolvedSchema(TableId tableId) {
return getLatestSchemaVersion(evolvedSchemas, tableId)
@@ -254,8 +155,6 @@ public class SchemaManager {
optionalSchema.isPresent(),
"Unable to apply SchemaChangeEvent for table \"%s\" without existing schema",
schemaChangeEvent.tableId());
-
- LOG.info("Handling original schema change event: {}", schemaChangeEvent);
registerNewSchema(
originalSchemas,
schemaChangeEvent.tableId(),
@@ -273,8 +172,6 @@ public class SchemaManager {
optionalSchema.isPresent(),
"Unable to apply SchemaChangeEvent for table \"%s\" without existing schema",
schemaChangeEvent.tableId());
-
- LOG.info("Handling evolved schema change event: {}", schemaChangeEvent);
registerNewSchema(
evolvedSchemas,
schemaChangeEvent.tableId(),
@@ -316,11 +213,9 @@ public class SchemaManager {
private void handleCreateTableEvent(
final Map<TableId, SortedMap<Integer, Schema>> schemaMap, CreateTableEvent event) {
- checkArgument(
- !schemaExists(schemaMap, event.tableId()),
- "Unable to apply CreateTableEvent to an existing schema for table \"%s\"",
- event.tableId());
- LOG.info("Handling schema change event: {}", event);
+ // Upstream schemas are transient, and might be sent differently after restarting from
+ // state. Thus, we should allow upstream CreateTableEvents to emplace existing ones, instead
+ // of throwing an exception.
registerNewSchema(schemaMap, event.tableId(), event.getSchema());
}
@@ -342,6 +237,16 @@ public class SchemaManager {
}
}
+ @VisibleForTesting
+ public void registerNewOriginalSchema(TableId tableId, Schema newSchema) {
+ registerNewSchema(originalSchemas, tableId, newSchema);
+ }
+
+ @VisibleForTesting
+ public void registerNewEvolvedSchema(TableId tableId, Schema newSchema) {
+ registerNewSchema(evolvedSchemas, tableId, newSchema);
+ }
+
/** Serializer for {@link SchemaManager}. */
public static class Serializer implements SimpleVersionedSerializer<SchemaManager> {
@@ -449,4 +354,23 @@ public class SchemaManager {
return tableSchemas;
}
}
+
+ @Override
+ public String toString() {
+ return String.format(
+ "Schema Manager %s: \n"
+ + "\toriginal schema map:\n%s\n"
+ + "\tevolved schema map:\n%s",
+ hashCode(), schemaMapToString(originalSchemas), schemaMapToString(evolvedSchemas));
+ }
+
+ private static String schemaMapToString(Map<TableId, SortedMap<Integer, Schema>> schemaMap) {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<TableId, SortedMap<Integer, Schema>> entry : schemaMap.entrySet()) {
+ TableId tableId = entry.getKey();
+ SortedMap<Integer, Schema> versionedSchemas = entry.getValue();
+ sb.append(String.format("\t\t- table %s: %s\n", tableId, schemaMap));
+ }
+ return sb.toString();
+ }
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaRegistry.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaRegistry.java
new file mode 100644
index 000000000..566dffa93
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaRegistry.java
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.cdc.common.annotation.Internal;
+import org.apache.flink.cdc.common.annotation.VisibleForTesting;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.route.RouteRule;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.FlushSuccessEvent;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaRequest;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaResponse;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetOriginalSchemaRequest;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetOriginalSchemaResponse;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.SinkWriterRegisterEvent;
+import org.apache.flink.cdc.runtime.operators.sink.SchemaEvolutionClient;
+import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
+import org.apache.flink.runtime.operators.coordination.CoordinationRequestHandler;
+import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
+import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
+import org.apache.flink.runtime.operators.coordination.OperatorEvent;
+import org.apache.flink.util.ExceptionUtils;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.function.ThrowingRunnable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeoutException;
+import java.util.function.BooleanSupplier;
+
+import static org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils.wrap;
+
+/**
+ * An abstract centralized schema registry that accepts requests from {@link SchemaEvolutionClient}.
+ * A legit schema registry should be able to:
+ *
+ * <ul>
+ * <li>Handle schema retrieval requests from {@link SchemaEvolutionClient}s
+ * <li>Accept and trace sink writers' registering events
+ * <li>Snapshot & Restore its state during checkpoints
+ * </ul>
+ *
+ * <br>
+ * These abilities are done by overriding abstract methods of {@link SchemaRegistry}. All methods
+ * will run in given {@link ExecutorService} asynchronously except {@code SchemaRegistry#restore}.
+ */
+@Internal
+public abstract class SchemaRegistry implements OperatorCoordinator, CoordinationRequestHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaRegistry.class);
+
+ // -------------------------
+ // Static fields that got bind after construction
+ // -------------------------
+ protected final OperatorCoordinator.Context context;
+ protected final String operatorName;
+ protected final ExecutorService coordinatorExecutor;
+ protected final MetadataApplier metadataApplier;
+ protected final Duration rpcTimeout;
+ protected final List<RouteRule> routingRules;
+ protected final SchemaChangeBehavior behavior;
+
+ // -------------------------
+ // Dynamically initialized transient fields (after coordinator starts)
+ // -------------------------
+ protected transient int currentParallelism;
+ protected transient Set<Integer> activeSinkWriters;
+ protected transient Map<Integer, Throwable> failedReasons;
+ protected transient SchemaManager schemaManager;
+ protected transient TableIdRouter router;
+
+ protected SchemaRegistry(
+ OperatorCoordinator.Context context,
+ String operatorName,
+ ExecutorService coordinatorExecutor,
+ MetadataApplier metadataApplier,
+ List<RouteRule> routingRules,
+ SchemaChangeBehavior schemaChangeBehavior,
+ Duration rpcTimeout) {
+ this.context = context;
+ this.operatorName = operatorName;
+ this.coordinatorExecutor = coordinatorExecutor;
+ this.metadataApplier = metadataApplier;
+ this.routingRules = routingRules;
+ this.rpcTimeout = rpcTimeout;
+ this.behavior = schemaChangeBehavior;
+ }
+
+ // ---------------
+ // Lifecycle hooks
+ // ---------------
+ @Override
+ public void start() throws Exception {
+ LOG.info("Starting SchemaRegistry - {}.", operatorName);
+ this.currentParallelism = context.currentParallelism();
+ this.activeSinkWriters = ConcurrentHashMap.newKeySet();
+ this.failedReasons = new ConcurrentHashMap<>();
+ this.schemaManager = new SchemaManager();
+ this.router = new TableIdRouter(routingRules);
+ }
+
+ @Override
+ public void close() throws Exception {
+ LOG.info("Closing SchemaRegistry - {}.", operatorName);
+ coordinatorExecutor.shutdown();
+ }
+
+ // ------------------------------
+ // Overridable checkpoint methods
+ // ------------------------------
+ /** Snapshot current schema registry state in byte array form. */
+ protected abstract void snapshot(CompletableFuture<byte[]> resultFuture) throws Exception;
+
+ /** Restore schema registry state from byte array. */
+ protected abstract void restore(byte[] checkpointData) throws Exception;
+
+ // ------------------------------------
+ // Overridable event & request handlers
+ // ------------------------------------
+
+ /** Overridable handler for {@link SinkWriterRegisterEvent}s. */
+ protected void handleSinkWriterRegisterEvent(SinkWriterRegisterEvent event) throws Exception {
+ LOG.info("Sink subtask {} already registered.", event.getSubtask());
+ activeSinkWriters.add(event.getSubtask());
+ }
+
+ /** Overridable handler for {@link FlushSuccessEvent}s. */
+ protected abstract void handleFlushSuccessEvent(FlushSuccessEvent event) throws Exception;
+
+ /** Overridable handler for {@link GetEvolvedSchemaRequest}s. */
+ protected void handleGetEvolvedSchemaRequest(
+ GetEvolvedSchemaRequest request, CompletableFuture<CoordinationResponse> responseFuture)
+ throws Exception {
+ LOG.info("Handling evolved schema request: {}", request);
+ int schemaVersion = request.getSchemaVersion();
+ TableId tableId = request.getTableId();
+ if (schemaVersion == GetEvolvedSchemaRequest.LATEST_SCHEMA_VERSION) {
+ responseFuture.complete(
+ wrap(
+ new GetEvolvedSchemaResponse(
+ schemaManager.getLatestEvolvedSchema(tableId).orElse(null))));
+ } else {
+ try {
+ responseFuture.complete(
+ wrap(
+ new GetEvolvedSchemaResponse(
+ schemaManager.getEvolvedSchema(tableId, schemaVersion))));
+ } catch (IllegalArgumentException iae) {
+ LOG.warn(
+ "Some client is requesting an non-existed evolved schema for table {} with version {}",
+ tableId,
+ schemaVersion);
+ responseFuture.complete(wrap(new GetEvolvedSchemaResponse(null)));
+ }
+ }
+ }
+
+ /** Overridable handler for {@link GetOriginalSchemaRequest}s. */
+ protected void handleGetOriginalSchemaRequest(
+ GetOriginalSchemaRequest request,
+ CompletableFuture<CoordinationResponse> responseFuture)
+ throws Exception {
+ LOG.info("Handling original schema request: {}", request);
+ int schemaVersion = request.getSchemaVersion();
+ TableId tableId = request.getTableId();
+ if (schemaVersion == GetOriginalSchemaRequest.LATEST_SCHEMA_VERSION) {
+ responseFuture.complete(
+ wrap(
+ new GetOriginalSchemaResponse(
+ schemaManager.getLatestOriginalSchema(tableId).orElse(null))));
+ } else {
+ try {
+ responseFuture.complete(
+ wrap(
+ new GetOriginalSchemaResponse(
+ schemaManager.getOriginalSchema(tableId, schemaVersion))));
+ } catch (IllegalArgumentException iae) {
+ LOG.warn(
+ "Some client is requesting an non-existed original schema for table {} with version {}",
+ tableId,
+ schemaVersion);
+ responseFuture.complete(wrap(new GetOriginalSchemaResponse(null)));
+ }
+ }
+ }
+
+ /** Coordination handler for customized {@link CoordinationRequest}s. */
+ protected abstract void handleCustomCoordinationRequest(
+ CoordinationRequest request, CompletableFuture<CoordinationResponse> responseFuture)
+ throws Exception;
+
+ /** Last chance to execute codes before job fails globally. */
+ protected void handleUnrecoverableError(String taskDescription, Throwable t) {
+ LOG.error(
+ "Uncaught exception in the Schema Registry ({}) event loop for {}.",
+ operatorName,
+ taskDescription,
+ t);
+ LOG.error("\tCurrent schema manager state: {}", schemaManager);
+ }
+
+ // ---------------------------------
+ // Event & Request Dispatching Stuff
+ // ---------------------------------
+
+ @Override
+ public final CompletableFuture<CoordinationResponse> handleCoordinationRequest(
+ CoordinationRequest request) {
+ CompletableFuture<CoordinationResponse> future = new CompletableFuture<>();
+ runInEventLoop(
+ () -> {
+ if (request instanceof GetEvolvedSchemaRequest) {
+ handleGetEvolvedSchemaRequest((GetEvolvedSchemaRequest) request, future);
+ } else if (request instanceof GetOriginalSchemaRequest) {
+ handleGetOriginalSchemaRequest((GetOriginalSchemaRequest) request, future);
+ } else {
+ handleCustomCoordinationRequest(request, future);
+ }
+ },
+ "Handling request - %s",
+ request);
+ return future;
+ }
+
+ @Override
+ public final void handleEventFromOperator(
+ int subTaskId, int attemptNumber, OperatorEvent event) {
+ runInEventLoop(
+ () -> {
+ if (event instanceof FlushSuccessEvent) {
+ handleFlushSuccessEvent((FlushSuccessEvent) event);
+ } else if (event instanceof SinkWriterRegisterEvent) {
+ handleSinkWriterRegisterEvent((SinkWriterRegisterEvent) event);
+ } else {
+ throw new FlinkRuntimeException("Unrecognized Operator Event: " + event);
+ }
+ },
+ "Handling event - %s (from subTask %d)",
+ event,
+ subTaskId);
+ }
+
+ // --------------------------
+ // Gateway registration stuff
+ // --------------------------
+
+ @Override
+ public final void subtaskReset(int subTaskId, long checkpointId) {
+ Throwable rootCause = failedReasons.get(subTaskId);
+ LOG.error("Subtask {} reset at checkpoint {}.", subTaskId, checkpointId, rootCause);
+ }
+
+ @Override
+ public final void executionAttemptFailed(
+ int subTaskId, int attemptNumber, @Nullable Throwable reason) {
+ failedReasons.put(subTaskId, reason);
+ }
+
+ @Override
+ public final void executionAttemptReady(
+ int subTaskId, int attemptNumber, SubtaskGateway gateway) {
+ // Needless to do anything. SchemaRegistry does not post message to the coordinator
+ // spontaneously.
+ }
+
+ // ---------------------------
+ // Checkpointing related stuff
+ // ---------------------------
+ @Override
+ public final void checkpointCoordinator(
+ long checkpointId, CompletableFuture<byte[]> completableFuture) throws Exception {
+ LOG.info("Going to start checkpoint No.{}", checkpointId);
+ runInEventLoop(() -> snapshot(completableFuture), "Taking checkpoint - %d", checkpointId);
+ }
+
+ @Override
+ public final void notifyCheckpointComplete(long checkpointId) {
+ LOG.info("Successfully completed checkpoint No.{}", checkpointId);
+ }
+
+ @Override
+ public final void resetToCheckpoint(long checkpointId, @Nullable byte[] checkpointData)
+ throws Exception {
+ LOG.info("Going to restore from checkpoint No.{}", checkpointId);
+ if (checkpointData == null) {
+ return;
+ }
+ restore(checkpointData);
+ }
+
+ // ---------------------------
+ // Utility functions
+ // ---------------------------
+ /**
+ * Run a time-consuming task in given {@link ExecutorService}. All overridable functions have
+ * been wrapped inside already, so there's no need to call this method again. However, if you're
+ * overriding methods from {@link OperatorCoordinator} or {@link CoordinationRequestHandler}
+ * directly, make sure you're running heavy logics inside, or the entire job might hang!
+ */
+ protected void runInEventLoop(
+ final ThrowingRunnable<Throwable> action,
+ final String actionName,
+ final Object... actionNameFormatParameters) {
+ coordinatorExecutor.execute(
+ () -> {
+ try {
+ action.run();
+ } catch (Throwable t) {
+ // if we have a JVM critical error, promote it immediately, there is a good
+ // chance the logging or job failing will not succeed anymore
+ ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
+ handleUnrecoverableError(
+ String.format(actionName, actionNameFormatParameters), t);
+ context.failJob(t);
+ }
+ });
+ }
+
+ /**
+ * Keeps checking if {@code conditionChecker} is satisfied. If not, emit a message and retry.
+ */
+ protected void loopUntil(
+ BooleanSupplier conditionChecker, Runnable message, Duration timeout, Duration interval)
+ throws TimeoutException {
+ loopWhen(() -> !conditionChecker.getAsBoolean(), message, timeout, interval);
+ }
+
+ /**
+ * Keeps checking if {@code conditionChecker} is satisfied. Otherwise, emit a message and retry.
+ */
+ protected void loopWhen(
+ BooleanSupplier conditionChecker, Runnable message, Duration timeout, Duration interval)
+ throws TimeoutException {
+ long deadline = System.currentTimeMillis() + timeout.toMillis();
+ long intervalMs = interval.toMillis();
+ while (conditionChecker.getAsBoolean()) {
+ message.run();
+ if (System.currentTimeMillis() > deadline) {
+ throw new TimeoutException("Loop checking time limit has exceeded.");
+ }
+ try {
+ Thread.sleep(intervalMs);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ protected <T extends Throwable> void failJob(String taskDescription, T t) {
+ ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
+ LOG.error("An exception was triggered from {}. Job will fail now.", taskDescription, t);
+ handleUnrecoverableError(taskDescription, t);
+ context.failJob(t);
+ }
+
+ // ------------------------
+ // Visible just for testing
+ // ------------------------
+
+ @VisibleForTesting
+ public void emplaceOriginalSchema(TableId tableId, Schema schema) {
+ schemaManager.registerNewOriginalSchema(tableId, schema);
+ }
+
+ @VisibleForTesting
+ public void emplaceEvolvedSchema(TableId tableId, Schema schema) {
+ schemaManager.registerNewEvolvedSchema(tableId, schema);
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/TableIdRouter.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/TableIdRouter.java
new file mode 100755
index 000000000..6b18404e1
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/TableIdRouter.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.route.RouteRule;
+import org.apache.flink.cdc.common.schema.Selectors;
+
+import org.apache.flink.shaded.guava31.com.google.common.cache.CacheBuilder;
+import org.apache.flink.shaded.guava31.com.google.common.cache.CacheLoader;
+import org.apache.flink.shaded.guava31.com.google.common.cache.LoadingCache;
+
+import javax.annotation.Nonnull;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
+
+/**
+ * Calculates how upstream data change events should be dispatched to downstream tables. Returns one
+ * or many destination Table IDs based on provided routing rules.
+ */
+public class TableIdRouter {
+
+ private final List<Tuple3<Selectors, String, String>> routes;
+ private final LoadingCache<TableId, List<TableId>> routingCache;
+ private static final Duration CACHE_EXPIRE_DURATION = Duration.ofDays(1);
+
+ public TableIdRouter(List<RouteRule> routingRules) {
+ this.routes = new ArrayList<>();
+ for (RouteRule rule : routingRules) {
+ try {
+ String tableInclusions = rule.sourceTable;
+ Selectors selectors =
+ new Selectors.SelectorsBuilder().includeTables(tableInclusions).build();
+ routes.add(new Tuple3<>(selectors, rule.sinkTable, rule.replaceSymbol));
+ } catch (PatternSyntaxException e) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Failed to parse regular expression in routing rule %s. Notice that `.` is used to separate Table ID components. To use it as a regex token, put a `\\` before to escape it.",
+ rule),
+ e);
+ }
+ }
+ this.routingCache =
+ CacheBuilder.newBuilder()
+ .expireAfterAccess(CACHE_EXPIRE_DURATION)
+ .build(
+ new CacheLoader<TableId, List<TableId>>() {
+ @Override
+ public @Nonnull List<TableId> load(@Nonnull TableId key) {
+ return calculateRoute(key);
+ }
+ });
+ }
+
+ public List<TableId> route(TableId sourceTableId) {
+ return routingCache.getUnchecked(sourceTableId);
+ }
+
+ private List<TableId> calculateRoute(TableId sourceTableId) {
+ List<TableId> routedTableIds =
+ routes.stream()
+ .filter(route -> route.f0.isMatch(sourceTableId))
+ .map(route -> resolveReplacement(sourceTableId, route))
+ .collect(Collectors.toList());
+ if (routedTableIds.isEmpty()) {
+ routedTableIds.add(sourceTableId);
+ }
+ return routedTableIds;
+ }
+
+ private TableId resolveReplacement(
+ TableId originalTable, Tuple3<Selectors, String, String> route) {
+ if (route.f2 != null) {
+ return TableId.parse(route.f1.replace(route.f2, originalTable.getTableName()));
+ }
+ return TableId.parse(route.f1);
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/FlushSuccessEvent.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/FlushSuccessEvent.java
similarity index 60%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/FlushSuccessEvent.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/FlushSuccessEvent.java
index 729514464..9777b6d66 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/FlushSuccessEvent.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/FlushSuccessEvent.java
@@ -15,10 +15,9 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common.event;
-import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;
import java.util.Objects;
@@ -31,22 +30,22 @@ public class FlushSuccessEvent implements OperatorEvent {
private static final long serialVersionUID = 1L;
/** The sink subtask finished flushing. */
- private final int subtask;
+ private final int sinkSubTaskId;
- /** The schema changes from which table is executing it. */
- private final TableId tableId;
+ /** The schema changes initiated from which source partition. */
+ private final int sourceSubTaskId;
- public FlushSuccessEvent(int subtask, TableId tableId) {
- this.subtask = subtask;
- this.tableId = tableId;
+ public FlushSuccessEvent(int sinkSubTaskId, int sourceSubTaskId) {
+ this.sinkSubTaskId = sinkSubTaskId;
+ this.sourceSubTaskId = sourceSubTaskId;
}
- public int getSubtask() {
- return subtask;
+ public int getSinkSubTaskId() {
+ return sinkSubTaskId;
}
- public TableId getTableId() {
- return tableId;
+ public int getSourceSubTaskId() {
+ return sourceSubTaskId;
}
@Override
@@ -58,11 +57,21 @@ public class FlushSuccessEvent implements OperatorEvent {
return false;
}
FlushSuccessEvent that = (FlushSuccessEvent) o;
- return subtask == that.subtask && Objects.equals(tableId, that.tableId);
+ return sinkSubTaskId == that.sinkSubTaskId && sourceSubTaskId == that.sourceSubTaskId;
}
@Override
public int hashCode() {
- return Objects.hash(subtask, tableId);
+ return Objects.hash(sinkSubTaskId, sourceSubTaskId);
+ }
+
+ @Override
+ public String toString() {
+ return "FlushSuccessEvent{"
+ + "subtask="
+ + sinkSubTaskId
+ + ", sourceSubTaskId="
+ + sourceSubTaskId
+ + '}';
}
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetEvolvedSchemaRequest.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetEvolvedSchemaRequest.java
similarity index 93%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetEvolvedSchemaRequest.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetEvolvedSchemaRequest.java
index f5b7a5860..a08beeeef 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetEvolvedSchemaRequest.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetEvolvedSchemaRequest.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common.event;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
/** Request to {@link SchemaRegistry} for getting schema of a table. */
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetEvolvedSchemaResponse.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetEvolvedSchemaResponse.java
similarity index 91%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetEvolvedSchemaResponse.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetEvolvedSchemaResponse.java
index 81fef92d7..6cbcdb988 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetEvolvedSchemaResponse.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetEvolvedSchemaResponse.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common.event;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
import javax.annotation.Nullable;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetOriginalSchemaRequest.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetOriginalSchemaRequest.java
similarity index 93%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetOriginalSchemaRequest.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetOriginalSchemaRequest.java
index 36544da77..f581f737b 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetOriginalSchemaRequest.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetOriginalSchemaRequest.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common.event;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
/** Request to {@link SchemaRegistry} for getting original schema of a table. */
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetOriginalSchemaResponse.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetOriginalSchemaResponse.java
similarity index 91%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetOriginalSchemaResponse.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetOriginalSchemaResponse.java
index 2c5343349..f93aa38c0 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/GetOriginalSchemaResponse.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/GetOriginalSchemaResponse.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common.event;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
import javax.annotation.Nullable;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SinkWriterRegisterEvent.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/SinkWriterRegisterEvent.java
similarity index 92%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SinkWriterRegisterEvent.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/SinkWriterRegisterEvent.java
index 190dbafc2..1757341f4 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SinkWriterRegisterEvent.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/event/SinkWriterRegisterEvent.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.event;
+package org.apache.flink.cdc.runtime.operators.schema.common.event;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;
import java.util.Objects;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/metrics/SchemaOperatorMetrics.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/metrics/SchemaOperatorMetrics.java
similarity index 96%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/metrics/SchemaOperatorMetrics.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/metrics/SchemaOperatorMetrics.java
index 4ba8e30c7..d40ffdafe 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/metrics/SchemaOperatorMetrics.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/common/metrics/SchemaOperatorMetrics.java
@@ -15,10 +15,10 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.metrics;
+package org.apache.flink.cdc.runtime.operators.schema.common.metrics;
import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
+import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.MetricGroup;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaDerivation.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaDerivation.java
deleted file mode 100644
index 39ebc50ba..000000000
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaDerivation.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
-
-import org.apache.flink.api.java.tuple.Tuple3;
-import org.apache.flink.cdc.common.event.AddColumnEvent;
-import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
-import org.apache.flink.cdc.common.event.CreateTableEvent;
-import org.apache.flink.cdc.common.event.RenameColumnEvent;
-import org.apache.flink.cdc.common.event.SchemaChangeEvent;
-import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.common.event.visitor.SchemaChangeEventVisitor;
-import org.apache.flink.cdc.common.route.RouteRule;
-import org.apache.flink.cdc.common.schema.Column;
-import org.apache.flink.cdc.common.schema.PhysicalColumn;
-import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.common.schema.Selectors;
-import org.apache.flink.cdc.common.types.DataType;
-import org.apache.flink.cdc.common.utils.ChangeEventUtils;
-import org.apache.flink.cdc.common.utils.SchemaUtils;
-import org.apache.flink.cdc.runtime.serializer.TableIdSerializer;
-import org.apache.flink.core.memory.DataInputViewStreamWrapper;
-import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/** Derive schema changes based on the routing rules. */
-public class SchemaDerivation {
- private final SchemaManager schemaManager;
- private final Map<TableId, Set<TableId>> derivationMapping;
-
- /**
- * Storing route source table selector, sink table name (before symbol replacement), and replace
- * symbol in a tuple.
- */
- private transient List<Tuple3<Selectors, String, String>> routes;
-
- public SchemaDerivation(
- SchemaManager schemaManager,
- List<RouteRule> routeRules,
- Map<TableId, Set<TableId>> derivationMapping) {
- this.schemaManager = schemaManager;
- this.routes =
- routeRules.stream()
- .map(
- rule -> {
- String tableInclusions = rule.sourceTable;
- Selectors selectors =
- new Selectors.SelectorsBuilder()
- .includeTables(tableInclusions)
- .build();
- return new Tuple3<>(
- selectors, rule.sinkTable, rule.replaceSymbol);
- })
- .collect(Collectors.toList());
- this.derivationMapping = derivationMapping;
- }
-
- public List<SchemaChangeEvent> applySchemaChange(SchemaChangeEvent schemaChangeEvent) {
- List<SchemaChangeEvent> events = new ArrayList<>();
- TableId originalTable = schemaChangeEvent.tableId();
- boolean noRouteMatched = true;
-
- for (Tuple3<Selectors, String, String> route : routes) {
- // Check routing table
- if (!route.f0.isMatch(originalTable)) {
- continue;
- }
-
- noRouteMatched = false;
-
- // Matched a routing rule
- TableId derivedTable = resolveReplacement(originalTable, route);
- Set<TableId> originalTables =
- derivationMapping.computeIfAbsent(derivedTable, t -> new HashSet<>());
- originalTables.add(originalTable);
-
- if (originalTables.size() == 1) {
- // single source mapping, replace the table ID directly
- SchemaChangeEvent derivedSchemaChangeEvent =
- ChangeEventUtils.recreateSchemaChangeEvent(schemaChangeEvent, derivedTable);
- events.add(derivedSchemaChangeEvent);
- } else {
- // multiple source mapping (merging tables)
- Schema derivedTableSchema =
- schemaManager.getLatestEvolvedSchema(derivedTable).get();
- events.addAll(
- Objects.requireNonNull(
- SchemaChangeEventVisitor.visit(
- schemaChangeEvent,
- addColumnEvent ->
- handleAddColumnEvent(
- addColumnEvent,
- derivedTableSchema,
- derivedTable),
- alterColumnTypeEvent ->
- handleAlterColumnTypeEvent(
- alterColumnTypeEvent,
- derivedTableSchema,
- derivedTable),
- createTableEvent ->
- handleCreateTableEvent(
- createTableEvent,
- derivedTableSchema,
- derivedTable),
- dropColumnEvent ->
- Collections.emptyList(), // Column drop shouldn't be
- // spread to route
- // destination.
- dropTableEvent ->
- Collections.emptyList(), // Table drop shouldn't be
- // spread to route
- // destination.
- renameColumnEvent ->
- handleRenameColumnEvent(
- renameColumnEvent,
- derivedTableSchema,
- derivedTable),
- truncateTableEvent ->
- Collections.emptyList() // // Table truncation
- // shouldn't be spread to route
- // destination.
- )));
- }
- }
-
- if (noRouteMatched) {
- // No routes are matched, leave it as-is
- return Collections.singletonList(schemaChangeEvent);
- } else {
- return events;
- }
- }
-
- private TableId resolveReplacement(
- TableId originalTable, Tuple3<Selectors, String, String> route) {
- if (route.f2 != null) {
- return TableId.parse(route.f1.replace(route.f2, originalTable.getTableName()));
- }
- return TableId.parse(route.f1);
- }
-
- public Map<TableId, Set<TableId>> getDerivationMapping() {
- return derivationMapping;
- }
-
- public static void serializeDerivationMapping(
- SchemaDerivation schemaDerivation, DataOutputStream out) throws IOException {
- TableIdSerializer tableIdSerializer = TableIdSerializer.INSTANCE;
- // Serialize derivation mapping in SchemaDerivation
- Map<TableId, Set<TableId>> derivationMapping = schemaDerivation.getDerivationMapping();
- out.writeInt(derivationMapping.size());
- for (Map.Entry<TableId, Set<TableId>> entry : derivationMapping.entrySet()) {
- // Routed table ID
- TableId routedTableId = entry.getKey();
- tableIdSerializer.serialize(routedTableId, new DataOutputViewStreamWrapper(out));
- // Original table IDs
- Set<TableId> originalTableIds = entry.getValue();
- out.writeInt(originalTableIds.size());
- for (TableId originalTableId : originalTableIds) {
- tableIdSerializer.serialize(originalTableId, new DataOutputViewStreamWrapper(out));
- }
- }
- }
-
- public static Map<TableId, Set<TableId>> deserializerDerivationMapping(DataInputStream in)
- throws IOException {
- TableIdSerializer tableIdSerializer = TableIdSerializer.INSTANCE;
- int derivationMappingSize = in.readInt();
- Map<TableId, Set<TableId>> derivationMapping = new HashMap<>(derivationMappingSize);
- for (int i = 0; i < derivationMappingSize; i++) {
- // Routed table ID
- TableId routedTableId =
- tableIdSerializer.deserialize(new DataInputViewStreamWrapper(in));
- // Original table IDs
- int numOriginalTables = in.readInt();
- Set<TableId> originalTableIds = new HashSet<>(numOriginalTables);
- for (int j = 0; j < numOriginalTables; j++) {
- TableId originalTableId =
- tableIdSerializer.deserialize(new DataInputViewStreamWrapper(in));
- originalTableIds.add(originalTableId);
- }
- derivationMapping.put(routedTableId, originalTableIds);
- }
- return derivationMapping;
- }
-
- private List<SchemaChangeEvent> handleRenameColumnEvent(
- RenameColumnEvent renameColumnEvent, Schema derivedTableSchema, TableId derivedTable) {
- List<AddColumnEvent.ColumnWithPosition> newColumns = new ArrayList<>();
- renameColumnEvent
- .getNameMapping()
- .forEach(
- (before, after) -> {
- if (derivedTableSchema.getColumn(after).isPresent()) {
- return;
- }
- Column existedColumn = derivedTableSchema.getColumn(before).get();
- newColumns.add(
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn(
- after,
- existedColumn.getType(),
- existedColumn.getComment())));
- });
- List<SchemaChangeEvent> schemaChangeEvents = new ArrayList<>();
- if (!newColumns.isEmpty()) {
- AddColumnEvent derivedSchemaChangeEvent = new AddColumnEvent(derivedTable, newColumns);
- schemaChangeEvents.add(derivedSchemaChangeEvent);
- }
- return schemaChangeEvents;
- }
-
- private List<SchemaChangeEvent> handleAlterColumnTypeEvent(
- AlterColumnTypeEvent alterColumnTypeEvent,
- Schema derivedTableSchema,
- TableId derivedTable) {
- Map<String, DataType> typeDifference = new HashMap<>();
- alterColumnTypeEvent
- .getTypeMapping()
- .forEach(
- (columnName, dataType) -> {
- Column existedColumnInDerivedTable =
- derivedTableSchema.getColumn(columnName).get();
- if (!existedColumnInDerivedTable.getType().equals(dataType)) {
- // Check type compatibility
- DataType widerType =
- getWiderType(
- columnName,
- existedColumnInDerivedTable.getType(),
- dataType);
- if (!widerType.equals(existedColumnInDerivedTable.getType())) {
- typeDifference.put(
- existedColumnInDerivedTable.getName(), widerType);
- }
- }
- });
- List<SchemaChangeEvent> schemaChangeEvents = new ArrayList<>();
- if (!typeDifference.isEmpty()) {
- AlterColumnTypeEvent derivedSchemaChangeEvent =
- new AlterColumnTypeEvent(derivedTable, typeDifference);
- schemaChangeEvents.add(derivedSchemaChangeEvent);
- }
- return schemaChangeEvents;
- }
-
- private List<SchemaChangeEvent> handleAddColumnEvent(
- AddColumnEvent addColumnEvent, Schema derivedTableSchema, TableId derivedTable) {
- List<AddColumnEvent.ColumnWithPosition> newColumns = new ArrayList<>();
- Map<String, DataType> newTypeMapping = new HashMap<>();
- // Check if new column already existed in the derived table
- for (AddColumnEvent.ColumnWithPosition addedColumn : addColumnEvent.getAddedColumns()) {
- Optional<Column> optionalColumnInDerivedTable =
- derivedTableSchema.getColumn(addedColumn.getAddColumn().getName());
- if (!optionalColumnInDerivedTable.isPresent()) {
- // Non-existed column. Use AddColumn
- newColumns.add(new AddColumnEvent.ColumnWithPosition(addedColumn.getAddColumn()));
- } else {
- // Existed column. Check type compatibility
- Column existedColumnInDerivedTable = optionalColumnInDerivedTable.get();
- if (!existedColumnInDerivedTable
- .getType()
- .equals(addedColumn.getAddColumn().getType())) {
- DataType widerType =
- getWiderType(
- existedColumnInDerivedTable.getName(),
- existedColumnInDerivedTable.getType(),
- addedColumn.getAddColumn().getType());
- if (!widerType.equals(existedColumnInDerivedTable.getType())) {
- newTypeMapping.put(existedColumnInDerivedTable.getName(), widerType);
- }
- }
- }
- }
-
- List<SchemaChangeEvent> schemaChangeEvents = new ArrayList<>();
- if (!newColumns.isEmpty()) {
- schemaChangeEvents.add(new AddColumnEvent(derivedTable, newColumns));
- }
- if (!newTypeMapping.isEmpty()) {
- schemaChangeEvents.add(new AlterColumnTypeEvent(derivedTable, newTypeMapping));
- }
- return schemaChangeEvents;
- }
-
- private List<SchemaChangeEvent> handleCreateTableEvent(
- CreateTableEvent createTableEvent, Schema derivedTableSchema, TableId derivedTable) {
- List<AddColumnEvent.ColumnWithPosition> newColumns = new ArrayList<>();
- Map<String, DataType> newTypeMapping = new HashMap<>();
- // Check if there is any columns that doesn't exist in the derived table
- // and perform add-column for non-existed columns.
- for (Column column : createTableEvent.getSchema().getColumns()) {
- Optional<Column> optionalColumnInDerivedTable =
- derivedTableSchema.getColumn(column.getName());
- if (!optionalColumnInDerivedTable.isPresent()) {
- // Non-existed column. Use AddColumn
- newColumns.add(new AddColumnEvent.ColumnWithPosition(column));
- } else {
- // Existed column. Check type compatibility
- Column existedColumnInDerivedTable = optionalColumnInDerivedTable.get();
- if (!existedColumnInDerivedTable.getType().equals(column.getType())) {
- DataType widerType =
- getWiderType(
- existedColumnInDerivedTable.getName(),
- existedColumnInDerivedTable.getType(),
- column.getType());
- if (!widerType.equals(existedColumnInDerivedTable.getType())) {
- newTypeMapping.put(existedColumnInDerivedTable.getName(), widerType);
- }
- }
- }
- }
-
- List<SchemaChangeEvent> schemaChangeEvents = new ArrayList<>();
- if (!newColumns.isEmpty()) {
- schemaChangeEvents.add(new AddColumnEvent(derivedTable, newColumns));
- }
- if (!newTypeMapping.isEmpty()) {
- schemaChangeEvents.add(new AlterColumnTypeEvent(derivedTable, newTypeMapping));
- }
- return schemaChangeEvents;
- }
-
- private DataType getWiderType(String columnName, DataType thisType, DataType thatType) {
- try {
- return SchemaUtils.inferWiderType(thisType, thatType);
- } catch (IllegalStateException e) {
- throw new IllegalStateException(
- String.format(
- "Incompatible types found for column `%s`: \"%s\" and \"%s\"",
- columnName, thisType, thatType));
- }
- }
-}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistry.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistry.java
deleted file mode 100644
index b7223c633..000000000
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistry.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
-
-import org.apache.flink.cdc.common.annotation.VisibleForTesting;
-import org.apache.flink.cdc.common.event.SchemaChangeEvent;
-import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
-import org.apache.flink.cdc.common.route.RouteRule;
-import org.apache.flink.cdc.common.sink.MetadataApplier;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.schema.event.FlushSuccessEvent;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetEvolvedSchemaRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetEvolvedSchemaResponse;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetOriginalSchemaRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetOriginalSchemaResponse;
-import org.apache.flink.cdc.runtime.operators.schema.event.SchemaChangeRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.SchemaChangeResultRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.SinkWriterRegisterEvent;
-import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
-import org.apache.flink.runtime.operators.coordination.CoordinationRequestHandler;
-import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
-import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
-import org.apache.flink.runtime.operators.coordination.OperatorEvent;
-import org.apache.flink.util.ExceptionUtils;
-import org.apache.flink.util.FlinkException;
-import org.apache.flink.util.function.ThrowingRunnable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-
-import static org.apache.flink.cdc.runtime.operators.schema.event.CoordinationResponseUtils.wrap;
-
-/**
- * The implementation of the {@link OperatorCoordinator} for the {@link SchemaOperator}.
- *
- * <p>The <code>SchemaRegister</code> provides an event loop style thread model to interact with the
- * Flink runtime. The coordinator ensures that all the state manipulations are made by its event
- * loop thread.
- *
- * <p>This <code>SchemaRegister</code> is responsible for:
- *
- * <ul>
- * <li>Apply schema changes when receiving the {@link SchemaChangeRequest} from {@link
- * SchemaOperator}
- * <li>Notify {@link SchemaOperator} to continue to push data for the table after receiving {@link
- * FlushSuccessEvent} from its registered sink writer
- * </ul>
- */
-public class SchemaRegistry implements OperatorCoordinator, CoordinationRequestHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(SchemaRegistry.class);
-
- /** The context of the coordinator. */
- private final OperatorCoordinator.Context context;
- /** The name of the operator this SchemaOperatorCoordinator is associated with. */
- private final String operatorName;
-
- /** A single-thread executor to handle async execution of the coordinator. */
- private final ExecutorService coordinatorExecutor;
-
- /**
- * Tracks the subtask failed reason to throw a more meaningful exception in {@link
- * #subtaskReset}.
- */
- private final Map<Integer, Throwable> failedReasons;
-
- /** Metadata applier for applying schema changes to external system. */
- private final MetadataApplier metadataApplier;
-
- private final List<RouteRule> routes;
-
- /** The request handler that handle all requests and events. */
- private SchemaRegistryRequestHandler requestHandler;
-
- /** Schema manager for tracking schemas of all tables. */
- private SchemaManager schemaManager;
-
- private SchemaDerivation schemaDerivation;
-
- private SchemaChangeBehavior schemaChangeBehavior;
-
- /**
- * Current parallelism. Use this to verify if Schema Registry has collected enough flush success
- * events from sink operators.
- */
- private int currentParallelism;
-
- public SchemaRegistry(
- String operatorName,
- OperatorCoordinator.Context context,
- ExecutorService executorService,
- MetadataApplier metadataApplier,
- List<RouteRule> routes) {
- this(
- operatorName,
- context,
- executorService,
- metadataApplier,
- routes,
- SchemaChangeBehavior.LENIENT);
- }
-
- public SchemaRegistry(
- String operatorName,
- OperatorCoordinator.Context context,
- ExecutorService coordinatorExecutor,
- MetadataApplier metadataApplier,
- List<RouteRule> routes,
- SchemaChangeBehavior schemaChangeBehavior) {
- this.context = context;
- this.coordinatorExecutor = coordinatorExecutor;
- this.operatorName = operatorName;
- this.failedReasons = new HashMap<>();
- this.metadataApplier = metadataApplier;
- this.routes = routes;
- this.schemaManager = new SchemaManager(schemaChangeBehavior);
- this.schemaDerivation = new SchemaDerivation(schemaManager, routes, new HashMap<>());
- this.requestHandler =
- new SchemaRegistryRequestHandler(
- metadataApplier,
- schemaManager,
- schemaDerivation,
- schemaChangeBehavior,
- context);
- this.schemaChangeBehavior = schemaChangeBehavior;
- }
-
- @Override
- public void start() throws Exception {
- LOG.info("Starting SchemaRegistry for {}.", operatorName);
- this.failedReasons.clear();
- this.currentParallelism = context.currentParallelism();
- LOG.info(
- "Started SchemaRegistry for {}. Parallelism: {}", operatorName, currentParallelism);
- }
-
- @Override
- public void close() throws Exception {
- LOG.info("SchemaRegistry for {} closed.", operatorName);
- coordinatorExecutor.shutdown();
- requestHandler.close();
- }
-
- @Override
- public void handleEventFromOperator(int subtask, int attemptNumber, OperatorEvent event) {
- runInEventLoop(
- () -> {
- try {
- if (event instanceof FlushSuccessEvent) {
- FlushSuccessEvent flushSuccessEvent = (FlushSuccessEvent) event;
- LOG.info(
- "Sink subtask {} succeed flushing for table {}.",
- flushSuccessEvent.getSubtask(),
- flushSuccessEvent.getTableId().toString());
- requestHandler.flushSuccess(
- flushSuccessEvent.getTableId(),
- flushSuccessEvent.getSubtask(),
- currentParallelism);
- } else if (event instanceof SinkWriterRegisterEvent) {
- requestHandler.registerSinkWriter(
- ((SinkWriterRegisterEvent) event).getSubtask());
- } else {
- throw new FlinkException("Unrecognized Operator Event: " + event);
- }
- } catch (Throwable t) {
- context.failJob(t);
- throw t;
- }
- },
- "handling event %s from subTask %d",
- event,
- subtask);
- }
-
- @Override
- public void checkpointCoordinator(long checkpointId, CompletableFuture<byte[]> resultFuture) {
- // we generate checkpoint in an async thread to not block the JobManager's main thread, the
- // coordinator state might be large if there are many schema changes and monitor many
- // tables.
- runInEventLoop(
- () -> {
- try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(baos)) {
- // Serialize SchemaManager
- int schemaManagerSerializerVersion = SchemaManager.SERIALIZER.getVersion();
- out.writeInt(schemaManagerSerializerVersion);
- byte[] serializedSchemaManager =
- SchemaManager.SERIALIZER.serialize(schemaManager);
- out.writeInt(serializedSchemaManager.length);
- out.write(serializedSchemaManager);
- // Serialize SchemaDerivation mapping
- SchemaDerivation.serializeDerivationMapping(schemaDerivation, out);
- resultFuture.complete(baos.toByteArray());
- } catch (Throwable t) {
- context.failJob(t);
- throw t;
- }
- },
- "taking checkpoint %d",
- checkpointId);
- }
-
- private void runInEventLoop(
- final ThrowingRunnable<Throwable> action,
- final String actionName,
- final Object... actionNameFormatParameters) {
- coordinatorExecutor.execute(
- () -> {
- try {
- action.run();
- } catch (Throwable t) {
- // if we have a JVM critical error, promote it immediately, there is a good
- // chance the logging or job failing will not succeed anymore
- ExceptionUtils.rethrowIfFatalErrorOrOOM(t);
-
- final String actionString =
- String.format(actionName, actionNameFormatParameters);
- LOG.error(
- "Uncaught exception in the SchemaEvolutionCoordinator for {} while {}. Triggering job failover.",
- operatorName,
- actionString,
- t);
- context.failJob(t);
- }
- });
- }
-
- @Override
- public void notifyCheckpointComplete(long checkpointId) {
- // do nothing
- }
-
- @Override
- public CompletableFuture<CoordinationResponse> handleCoordinationRequest(
- CoordinationRequest request) {
- CompletableFuture<CoordinationResponse> responseFuture = new CompletableFuture<>();
- runInEventLoop(
- () -> {
- try {
- if (request instanceof SchemaChangeRequest) {
- SchemaChangeRequest schemaChangeRequest = (SchemaChangeRequest) request;
- requestHandler.handleSchemaChangeRequest(
- schemaChangeRequest, responseFuture);
- } else if (request instanceof SchemaChangeResultRequest) {
- requestHandler.getSchemaChangeResult(responseFuture);
- } else if (request instanceof GetEvolvedSchemaRequest) {
- handleGetEvolvedSchemaRequest(
- ((GetEvolvedSchemaRequest) request), responseFuture);
- } else if (request instanceof GetOriginalSchemaRequest) {
- handleGetOriginalSchemaRequest(
- (GetOriginalSchemaRequest) request, responseFuture);
- } else {
- throw new IllegalArgumentException(
- "Unrecognized CoordinationRequest type: " + request);
- }
- } catch (Throwable t) {
- context.failJob(t);
- throw t;
- }
- },
- "handling coordination request %s",
- request);
- return responseFuture;
- }
-
- @Override
- public void resetToCheckpoint(long checkpointId, @Nullable byte[] checkpointData)
- throws Exception {
- if (checkpointData == null) {
- return;
- }
- try (ByteArrayInputStream bais = new ByteArrayInputStream(checkpointData);
- DataInputStream in = new DataInputStream(bais)) {
- int schemaManagerSerializerVersion = in.readInt();
-
- switch (schemaManagerSerializerVersion) {
- case 0:
- {
- int length = in.readInt();
- byte[] serializedSchemaManager = new byte[length];
- in.readFully(serializedSchemaManager);
- schemaManager =
- SchemaManager.SERIALIZER.deserialize(
- schemaManagerSerializerVersion, serializedSchemaManager);
- schemaDerivation =
- new SchemaDerivation(schemaManager, routes, Collections.emptyMap());
- requestHandler =
- new SchemaRegistryRequestHandler(
- metadataApplier,
- schemaManager,
- schemaDerivation,
- schemaManager.getBehavior(),
- context);
- break;
- }
- case 1:
- case 2:
- {
- int length = in.readInt();
- byte[] serializedSchemaManager = new byte[length];
- in.readFully(serializedSchemaManager);
- schemaManager =
- SchemaManager.SERIALIZER.deserialize(
- schemaManagerSerializerVersion, serializedSchemaManager);
- Map<TableId, Set<TableId>> derivationMapping =
- SchemaDerivation.deserializerDerivationMapping(in);
- schemaDerivation =
- new SchemaDerivation(schemaManager, routes, derivationMapping);
- requestHandler =
- new SchemaRegistryRequestHandler(
- metadataApplier,
- schemaManager,
- schemaDerivation,
- schemaChangeBehavior,
- context);
- break;
- }
- default:
- throw new IOException(
- "Unrecognized serialization version " + schemaManagerSerializerVersion);
- }
- } catch (Throwable t) {
- context.failJob(t);
- throw t;
- }
- }
-
- @Override
- public void subtaskReset(int subtask, long checkpointId) {
- Throwable rootCause = failedReasons.get(subtask);
- LOG.error(
- String.format("Subtask %d reset at checkpoint %d.", subtask, checkpointId),
- rootCause);
- }
-
- @Override
- public void executionAttemptFailed(
- int subtask, int attemptNumber, @Nullable Throwable throwable) {
- failedReasons.put(subtask, throwable);
- }
-
- @Override
- public void executionAttemptReady(
- int subtask, int attemptNumber, SubtaskGateway subtaskGateway) {
- // do nothing
- }
-
- private void handleGetEvolvedSchemaRequest(
- GetEvolvedSchemaRequest getEvolvedSchemaRequest,
- CompletableFuture<CoordinationResponse> response) {
- LOG.info("Handling evolved schema request: {}", getEvolvedSchemaRequest);
- int schemaVersion = getEvolvedSchemaRequest.getSchemaVersion();
- TableId tableId = getEvolvedSchemaRequest.getTableId();
- if (schemaVersion == GetEvolvedSchemaRequest.LATEST_SCHEMA_VERSION) {
- response.complete(
- wrap(
- new GetEvolvedSchemaResponse(
- schemaManager.getLatestEvolvedSchema(tableId).orElse(null))));
- } else {
- try {
- response.complete(
- wrap(
- new GetEvolvedSchemaResponse(
- schemaManager.getEvolvedSchema(tableId, schemaVersion))));
- } catch (IllegalArgumentException iae) {
- LOG.warn(
- "Some client is requesting an non-existed evolved schema for table {} with version {}",
- tableId,
- schemaVersion);
- response.complete(wrap(new GetEvolvedSchemaResponse(null)));
- }
- }
- }
-
- private void handleGetOriginalSchemaRequest(
- GetOriginalSchemaRequest getOriginalSchemaRequest,
- CompletableFuture<CoordinationResponse> response) {
- LOG.info("Handling original schema request: {}", getOriginalSchemaRequest);
- int schemaVersion = getOriginalSchemaRequest.getSchemaVersion();
- TableId tableId = getOriginalSchemaRequest.getTableId();
- if (schemaVersion == GetOriginalSchemaRequest.LATEST_SCHEMA_VERSION) {
- response.complete(
- wrap(
- new GetOriginalSchemaResponse(
- schemaManager.getLatestOriginalSchema(tableId).orElse(null))));
- } else {
- try {
- response.complete(
- wrap(
- new GetOriginalSchemaResponse(
- schemaManager.getOriginalSchema(tableId, schemaVersion))));
- } catch (IllegalArgumentException iae) {
- LOG.warn(
- "Some client is requesting an non-existed original schema for table {} with version {}",
- tableId,
- schemaVersion);
- response.complete(wrap(new GetOriginalSchemaResponse(null)));
- }
- }
- }
-
- // --------------------Only visible for test -----------------
-
- @VisibleForTesting
- public void handleApplyOriginalSchemaChangeEvent(SchemaChangeEvent schemaChangeEvent) {
- schemaManager.applyOriginalSchemaChange(schemaChangeEvent);
- }
-
- @VisibleForTesting
- public void handleApplyEvolvedSchemaChangeRequest(SchemaChangeEvent schemaChangeEvent) {
- schemaManager.applyEvolvedSchemaChange(schemaChangeEvent);
- }
-}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistryRequestHandler.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistryRequestHandler.java
deleted file mode 100644
index b2a8a822d..000000000
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistryRequestHandler.java
+++ /dev/null
@@ -1,517 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
-
-import org.apache.flink.cdc.common.annotation.Internal;
-import org.apache.flink.cdc.common.event.AddColumnEvent;
-import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
-import org.apache.flink.cdc.common.event.CreateTableEvent;
-import org.apache.flink.cdc.common.event.DropColumnEvent;
-import org.apache.flink.cdc.common.event.RenameColumnEvent;
-import org.apache.flink.cdc.common.event.SchemaChangeEvent;
-import org.apache.flink.cdc.common.event.SchemaChangeEventType;
-import org.apache.flink.cdc.common.event.SchemaChangeEventWithPreSchema;
-import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.common.exceptions.UnsupportedSchemaChangeEventException;
-import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
-import org.apache.flink.cdc.common.schema.Column;
-import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.common.sink.MetadataApplier;
-import org.apache.flink.cdc.common.types.DataType;
-import org.apache.flink.cdc.common.utils.Preconditions;
-import org.apache.flink.cdc.runtime.operators.schema.event.SchemaChangeProcessingResponse;
-import org.apache.flink.cdc.runtime.operators.schema.event.SchemaChangeRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.SchemaChangeResponse;
-import org.apache.flink.cdc.runtime.operators.schema.event.SchemaChangeResultResponse;
-import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
-import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.flink.cdc.runtime.operators.schema.event.CoordinationResponseUtils.wrap;
-
-/** A handler to deal with all requests and events for {@link SchemaRegistry}. */
-@Internal
-public class SchemaRegistryRequestHandler implements Closeable {
- private static final Logger LOG = LoggerFactory.getLogger(SchemaRegistryRequestHandler.class);
-
- /** The {@link MetadataApplier} for every table. */
- private final MetadataApplier metadataApplier;
- /** All active sink writers. */
- private final Set<Integer> activeSinkWriters;
- /** Schema manager holding schema for all tables. */
- private final SchemaManager schemaManager;
-
- private final SchemaDerivation schemaDerivation;
-
- /**
- * Atomic flag indicating if current RequestHandler could accept more schema changes for now.
- */
- private volatile RequestStatus schemaChangeStatus;
-
- private final List<Integer> pendingSubTaskIds;
- private final Object schemaChangeRequestLock;
-
- private volatile Throwable currentChangeException;
- private volatile List<SchemaChangeEvent> currentDerivedSchemaChangeEvents;
- private volatile List<SchemaChangeEvent> currentFinishedSchemaChanges;
- private volatile List<SchemaChangeEvent> currentIgnoredSchemaChanges;
-
- /** Sink writers which have sent flush success events for the request. */
- private final Set<Integer> flushedSinkWriters;
-
- /** Executor service to execute schema change. */
- private final ExecutorService schemaChangeThreadPool;
-
- private final SchemaChangeBehavior schemaChangeBehavior;
-
- private final OperatorCoordinator.Context context;
-
- public SchemaRegistryRequestHandler(
- MetadataApplier metadataApplier,
- SchemaManager schemaManager,
- SchemaDerivation schemaDerivation,
- SchemaChangeBehavior schemaChangeBehavior,
- OperatorCoordinator.Context context) {
- this.metadataApplier = metadataApplier;
- this.schemaManager = schemaManager;
- this.schemaDerivation = schemaDerivation;
- this.schemaChangeBehavior = schemaChangeBehavior;
- this.context = context;
-
- this.activeSinkWriters = ConcurrentHashMap.newKeySet();
- this.flushedSinkWriters = ConcurrentHashMap.newKeySet();
- this.schemaChangeThreadPool = Executors.newSingleThreadExecutor();
-
- this.currentDerivedSchemaChangeEvents = new ArrayList<>();
- this.currentFinishedSchemaChanges = new ArrayList<>();
- this.currentIgnoredSchemaChanges = new ArrayList<>();
-
- this.schemaChangeStatus = RequestStatus.IDLE;
- this.pendingSubTaskIds = new ArrayList<>();
- this.schemaChangeRequestLock = new Object();
- }
-
- /**
- * Handle the {@link SchemaChangeRequest} and wait for all sink subtasks flushing.
- *
- * @param request the received SchemaChangeRequest
- */
- public void handleSchemaChangeRequest(
- SchemaChangeRequest request, CompletableFuture<CoordinationResponse> response) {
-
- // We use requester subTask ID as the pending ticket, because there will be at most 1 schema
- // change requests simultaneously from each subTask
- int requestSubTaskId = request.getSubTaskId();
-
- synchronized (schemaChangeRequestLock) {
- // Make sure we handle the first request in the pending list to avoid out-of-order
- // waiting and blocks checkpointing mechanism.
- if (schemaChangeStatus == RequestStatus.IDLE) {
- if (pendingSubTaskIds.isEmpty()) {
- LOG.info(
- "Received schema change event request {} from table {} from subTask {}. Pending list is empty, handling this.",
- request.getSchemaChangeEvent(),
- request.getTableId().toString(),
- requestSubTaskId);
- } else if (pendingSubTaskIds.get(0) == requestSubTaskId) {
- LOG.info(
- "Received schema change event request {} from table {} from subTask {}. It is on the first of the pending list, handling this.",
- request.getSchemaChangeEvent(),
- request.getTableId().toString(),
- requestSubTaskId);
- pendingSubTaskIds.remove(0);
- } else {
- LOG.info(
- "Received schema change event request {} from table {} from subTask {}. It is not the first of the pending list ({}).",
- request.getSchemaChangeEvent(),
- request.getTableId().toString(),
- requestSubTaskId,
- pendingSubTaskIds);
- if (!pendingSubTaskIds.contains(requestSubTaskId)) {
- pendingSubTaskIds.add(requestSubTaskId);
- }
- response.complete(wrap(SchemaChangeResponse.busy()));
- return;
- }
-
- SchemaChangeEvent event = request.getSchemaChangeEvent();
-
- // If this schema change event has been requested by another subTask, ignore it.
- if (schemaManager.isOriginalSchemaChangeEventRedundant(event)) {
- LOG.info("Event {} has been addressed before, ignoring it.", event);
- clearCurrentSchemaChangeRequest();
- LOG.info(
- "SchemaChangeStatus switched from WAITING_FOR_FLUSH to IDLE for request {} due to duplicated request.",
- request);
- response.complete(wrap(SchemaChangeResponse.duplicate()));
- return;
- }
- schemaManager.applyOriginalSchemaChange(event);
- List<SchemaChangeEvent> derivedSchemaChangeEvents =
- calculateDerivedSchemaChangeEvents(request.getSchemaChangeEvent());
-
- // If this schema change event is filtered out by LENIENT mode or merging table
- // route strategies, ignore it.
- if (derivedSchemaChangeEvents.isEmpty()) {
- LOG.info("Event {} is omitted from sending to downstream, ignoring it.", event);
- clearCurrentSchemaChangeRequest();
- LOG.info(
- "SchemaChangeStatus switched from WAITING_FOR_FLUSH to IDLE for request {} due to ignored request.",
- request);
-
- response.complete(wrap(SchemaChangeResponse.ignored()));
- return;
- }
-
- LOG.info(
- "SchemaChangeStatus switched from IDLE to WAITING_FOR_FLUSH, other requests will be blocked.");
- // This request has been accepted.
- schemaChangeStatus = RequestStatus.WAITING_FOR_FLUSH;
-
- // Backfill pre-schema info for sink applying
- derivedSchemaChangeEvents.forEach(
- e -> {
- if (e instanceof SchemaChangeEventWithPreSchema) {
- SchemaChangeEventWithPreSchema pe =
- (SchemaChangeEventWithPreSchema) e;
- if (!pe.hasPreSchema()) {
- schemaManager
- .getLatestEvolvedSchema(pe.tableId())
- .ifPresent(pe::fillPreSchema);
- }
- }
- });
- currentDerivedSchemaChangeEvents = new ArrayList<>(derivedSchemaChangeEvents);
-
- response.complete(wrap(SchemaChangeResponse.accepted(derivedSchemaChangeEvents)));
- } else {
- LOG.info(
- "Schema Registry is busy processing a schema change request, could not handle request {} for now. Added {} to pending list ({}).",
- request,
- requestSubTaskId,
- pendingSubTaskIds);
- if (!pendingSubTaskIds.contains(requestSubTaskId)) {
- pendingSubTaskIds.add(requestSubTaskId);
- }
- response.complete(wrap(SchemaChangeResponse.busy()));
- }
- }
- }
-
- /**
- * Apply the schema change to the external system.
- *
- * @param tableId the table need to change schema
- * @param derivedSchemaChangeEvents list of the schema changes
- */
- private void applySchemaChange(
- TableId tableId, List<SchemaChangeEvent> derivedSchemaChangeEvents) {
- for (SchemaChangeEvent changeEvent : derivedSchemaChangeEvents) {
- if (changeEvent.getType() != SchemaChangeEventType.CREATE_TABLE) {
- if (schemaChangeBehavior == SchemaChangeBehavior.IGNORE) {
- currentIgnoredSchemaChanges.add(changeEvent);
- continue;
- }
- }
- if (!metadataApplier.acceptsSchemaEvolutionType(changeEvent.getType())) {
- LOG.info("Ignored schema change {} to table {}.", changeEvent, tableId);
- currentIgnoredSchemaChanges.add(changeEvent);
- } else {
- try {
- metadataApplier.applySchemaChange(changeEvent);
- LOG.info("Applied schema change {} to table {}.", changeEvent, tableId);
- schemaManager.applyEvolvedSchemaChange(changeEvent);
- currentFinishedSchemaChanges.add(changeEvent);
- } catch (Throwable t) {
- LOG.error(
- "Failed to apply schema change {} to table {}. Caused by: {}",
- changeEvent,
- tableId,
- t);
- if (!shouldIgnoreException(t)) {
- currentChangeException = t;
- break;
- } else {
- LOG.warn(
- "Failed to apply event {}, but keeps running in tolerant mode. Caused by: {}",
- changeEvent,
- t);
- }
- }
- }
- }
- Preconditions.checkState(
- schemaChangeStatus == RequestStatus.APPLYING,
- "Illegal schemaChangeStatus state: should be APPLYING before applySchemaChange finishes, not "
- + schemaChangeStatus);
- schemaChangeStatus = RequestStatus.FINISHED;
- LOG.info(
- "SchemaChangeStatus switched from APPLYING to FINISHED for request {}.",
- currentDerivedSchemaChangeEvents);
- }
-
- /**
- * Register a sink subtask.
- *
- * @param sinkSubtask the sink subtask to register
- */
- public void registerSinkWriter(int sinkSubtask) {
- LOG.info("Register sink subtask {}.", sinkSubtask);
- activeSinkWriters.add(sinkSubtask);
- }
-
- /**
- * Record flushed sink subtasks after receiving FlushSuccessEvent.
- *
- * @param tableId the subtask in SchemaOperator and table that the FlushEvent is about
- * @param sinkSubtask the sink subtask succeed flushing
- */
- public void flushSuccess(TableId tableId, int sinkSubtask, int parallelism) {
- flushedSinkWriters.add(sinkSubtask);
- if (activeSinkWriters.size() < parallelism) {
- LOG.info(
- "Not all active sink writers have been registered. Current {}, expected {}.",
- activeSinkWriters.size(),
- parallelism);
- return;
- }
- if (flushedSinkWriters.equals(activeSinkWriters)) {
- Preconditions.checkState(
- schemaChangeStatus == RequestStatus.WAITING_FOR_FLUSH,
- "Illegal schemaChangeStatus state: should be WAITING_FOR_FLUSH before collecting enough FlushEvents, not "
- + schemaChangeStatus);
-
- schemaChangeStatus = RequestStatus.APPLYING;
- LOG.info(
- "All sink subtask have flushed for table {}. Start to apply schema change.",
- tableId.toString());
- schemaChangeThreadPool.submit(
- () -> applySchemaChange(tableId, currentDerivedSchemaChangeEvents));
- }
- }
-
- public void getSchemaChangeResult(CompletableFuture<CoordinationResponse> response) {
- Preconditions.checkState(
- schemaChangeStatus != RequestStatus.IDLE,
- "Illegal schemaChangeStatus: should not be IDLE before getting schema change request results.");
- if (schemaChangeStatus == RequestStatus.FINISHED) {
- schemaChangeStatus = RequestStatus.IDLE;
- LOG.info(
- "SchemaChangeStatus switched from FINISHED to IDLE for request {}",
- currentDerivedSchemaChangeEvents);
-
- // This request has been finished, return it and prepare for the next request
- List<SchemaChangeEvent> finishedEvents = clearCurrentSchemaChangeRequest();
- SchemaChangeResultResponse resultResponse =
- new SchemaChangeResultResponse(finishedEvents);
- response.complete(wrap(resultResponse));
- } else {
- // Still working on schema change request, waiting it
- response.complete(wrap(new SchemaChangeProcessingResponse()));
- }
- }
-
- @Override
- public void close() throws IOException {
- if (schemaChangeThreadPool != null) {
- schemaChangeThreadPool.shutdown();
- }
- }
-
- private List<SchemaChangeEvent> calculateDerivedSchemaChangeEvents(SchemaChangeEvent event) {
- if (SchemaChangeBehavior.LENIENT.equals(schemaChangeBehavior)) {
- return schemaDerivation.applySchemaChange(event).stream()
- .flatMap(evt -> lenientizeSchemaChangeEvent(evt).stream())
- .collect(Collectors.toList());
- } else {
- return schemaDerivation.applySchemaChange(event);
- }
- }
-
- private List<SchemaChangeEvent> lenientizeSchemaChangeEvent(SchemaChangeEvent event) {
- if (event instanceof CreateTableEvent) {
- return Collections.singletonList(event);
- }
- TableId tableId = event.tableId();
- Schema evolvedSchema =
- schemaManager
- .getLatestEvolvedSchema(tableId)
- .orElseThrow(
- () ->
- new IllegalStateException(
- "Evolved schema does not exist, not ready for schema change event "
- + event));
- switch (event.getType()) {
- case ADD_COLUMN:
- {
- AddColumnEvent addColumnEvent = (AddColumnEvent) event;
- return Collections.singletonList(
- new AddColumnEvent(
- tableId,
- addColumnEvent.getAddedColumns().stream()
- .map(
- col ->
- new AddColumnEvent.ColumnWithPosition(
- Column.physicalColumn(
- col.getAddColumn()
- .getName(),
- col.getAddColumn()
- .getType()
- .nullable(),
- col.getAddColumn()
- .getComment(),
- col.getAddColumn()
- .getDefaultValueExpression())))
- .collect(Collectors.toList())));
- }
- case DROP_COLUMN:
- {
- DropColumnEvent dropColumnEvent = (DropColumnEvent) event;
- Map<String, DataType> convertNullableColumns =
- dropColumnEvent.getDroppedColumnNames().stream()
- .map(evolvedSchema::getColumn)
- .flatMap(e -> e.map(Stream::of).orElse(Stream.empty()))
- .filter(col -> !col.getType().isNullable())
- .collect(
- Collectors.toMap(
- Column::getName,
- column -> column.getType().nullable()));
-
- if (convertNullableColumns.isEmpty()) {
- return Collections.emptyList();
- } else {
- return Collections.singletonList(
- new AlterColumnTypeEvent(tableId, convertNullableColumns));
- }
- }
- case RENAME_COLUMN:
- {
- RenameColumnEvent renameColumnEvent = (RenameColumnEvent) event;
- List<AddColumnEvent.ColumnWithPosition> appendColumns = new ArrayList<>();
- Map<String, DataType> convertNullableColumns = new HashMap<>();
- renameColumnEvent
- .getNameMapping()
- .forEach(
- (key, value) -> {
- Column column =
- evolvedSchema
- .getColumn(key)
- .orElseThrow(
- () ->
- new IllegalArgumentException(
- "Non-existed column "
- + key
- + " in evolved schema."));
- if (!column.getType().isNullable()) {
- // It's a not-nullable column, we need to cast it to
- // nullable first
- convertNullableColumns.put(
- key, column.getType().nullable());
- }
- appendColumns.add(
- new AddColumnEvent.ColumnWithPosition(
- Column.physicalColumn(
- value,
- column.getType().nullable(),
- column.getComment(),
- column
- .getDefaultValueExpression())));
- });
-
- List<SchemaChangeEvent> events = new ArrayList<>();
- events.add(new AddColumnEvent(tableId, appendColumns));
- if (!convertNullableColumns.isEmpty()) {
- events.add(new AlterColumnTypeEvent(tableId, convertNullableColumns));
- }
- return events;
- }
- default:
- return Collections.singletonList(event);
- }
- }
-
- private boolean shouldIgnoreException(Throwable throwable) {
- // In IGNORE mode, will never try to apply schema change events
- // In EVOLVE and LENIENT mode, such failure will not be tolerated
- // In EXCEPTION mode, an exception will be thrown once captured
- return (throwable instanceof UnsupportedSchemaChangeEventException)
- && (schemaChangeBehavior == SchemaChangeBehavior.TRY_EVOLVE);
- }
-
- private List<SchemaChangeEvent> clearCurrentSchemaChangeRequest() {
- if (currentChangeException != null) {
- context.failJob(
- new RuntimeException("Failed to apply schema change.", currentChangeException));
- }
- List<SchemaChangeEvent> finishedSchemaChanges =
- new ArrayList<>(currentFinishedSchemaChanges);
- flushedSinkWriters.clear();
- currentDerivedSchemaChangeEvents.clear();
- currentFinishedSchemaChanges.clear();
- currentIgnoredSchemaChanges.clear();
- currentChangeException = null;
- return finishedSchemaChanges;
- }
-
- // Schema change event state could transfer in the following way:
- //
- // -------- B --------
- // | |
- // v |
- // -------- ---------------------
- // | IDLE | --- A --> | WAITING_FOR_FLUSH |
- // -------- ---------------------
- // ^ |
- // E C
- // \ v
- // ------------ ------------
- // | FINISHED | <-- D -- | APPLYING |
- // ------------ ------------
- //
- // A: When a request came to an idling request handler.
- // B: When current request is duplicate or ignored by LENIENT / routed table merging
- // strategies.
- // C: When schema registry collected enough flush success events, and actually started to apply
- // schema changes.
- // D: When schema change application finishes (successfully or with exceptions)
- // E: When current schema change request result has been retrieved by SchemaOperator, and ready
- // for the next request.
- private enum RequestStatus {
- IDLE,
- WAITING_FOR_FLUSH,
- APPLYING,
- FINISHED
- }
-}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java
new file mode 100755
index 000000000..6d49653ba
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinator.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.distributed;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.cdc.common.annotation.VisibleForTesting;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.route.RouteRule;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.common.utils.Preconditions;
+import org.apache.flink.cdc.common.utils.SchemaMergingUtils;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaDerivator;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaManager;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaRegistry;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.FlushSuccessEvent;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetOriginalSchemaRequest;
+import org.apache.flink.cdc.runtime.operators.schema.distributed.event.SchemaChangeRequest;
+import org.apache.flink.cdc.runtime.operators.schema.distributed.event.SchemaChangeResponse;
+import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
+import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
+import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.HashBasedTable;
+import org.apache.flink.shaded.guava31.com.google.common.collect.HashMultimap;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Multimap;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Table;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import static org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils.wrap;
+
+/** Coordinator node for {@link SchemaOperator}. Registry actor in distributed topology. */
+public class SchemaCoordinator extends SchemaRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaCoordinator.class);
+
+ /** Atomic finite state machine to track global schema evolving state. */
+ private transient AtomicReference<RequestStatus> evolvingStatus;
+
+ /** Request futures from pending schema mappers. */
+ private transient Map<
+ Integer, Tuple2<SchemaChangeRequest, CompletableFuture<CoordinationResponse>>>
+ pendingRequests;
+
+ /** Tracing sink writers that have flushed successfully. */
+ protected transient Set<Integer> flushedSinkWriters;
+
+ /**
+ * Transient upstream table schema. The second arity is source partition ID, because in
+ * distributed topology, schemas might vary among partitions, so we can't rely on {@code
+ * schemaManager} to store original schemas.
+ */
+ private transient Table<TableId, Integer, Schema> upstreamSchemaTable;
+
+ /**
+ * In distributed topology, one schema change event will be broadcast N-times (N = downstream
+ * parallelism). We need to effectively ignore duplicate ones since not all {@link
+ * SchemaChangeEvent}s are idempotent.
+ */
+ private transient Multimap<Tuple2<Integer, SchemaChangeEvent>, Integer>
+ alreadyHandledSchemaChangeEvents;
+
+ public SchemaCoordinator(
+ String operatorName,
+ OperatorCoordinator.Context context,
+ ExecutorService coordinatorExecutor,
+ MetadataApplier metadataApplier,
+ List<RouteRule> routingRules,
+ SchemaChangeBehavior schemaChangeBehavior,
+ Duration rpcTimeout) {
+ super(
+ context,
+ operatorName,
+ coordinatorExecutor,
+ metadataApplier,
+ routingRules,
+ schemaChangeBehavior,
+ rpcTimeout);
+ }
+
+ // -----------------
+ // Lifecycle methods
+ // -----------------
+ @Override
+ public void start() throws Exception {
+ super.start();
+ this.evolvingStatus = new AtomicReference<>(RequestStatus.IDLE);
+ this.pendingRequests = new ConcurrentHashMap<>();
+ this.flushedSinkWriters = ConcurrentHashMap.newKeySet();
+ this.upstreamSchemaTable = HashBasedTable.create();
+ this.alreadyHandledSchemaChangeEvents = HashMultimap.create();
+ LOG.info(
+ "Started SchemaRegistry for {}. Parallelism: {}", operatorName, currentParallelism);
+ }
+
+ // --------------------------
+ // Checkpoint related methods
+ // --------------------------
+ @Override
+ protected void snapshot(CompletableFuture<byte[]> resultFuture) throws Exception {
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos)) {
+ // Serialize SchemaManager
+ int schemaManagerSerializerVersion = SchemaManager.SERIALIZER.getVersion();
+ out.writeInt(schemaManagerSerializerVersion);
+ byte[] serializedSchemaManager = SchemaManager.SERIALIZER.serialize(schemaManager);
+ out.writeInt(serializedSchemaManager.length);
+ out.write(serializedSchemaManager);
+ resultFuture.complete(baos.toByteArray());
+ }
+ }
+
+ @Override
+ protected void restore(byte[] checkpointData) throws Exception {
+ try (ByteArrayInputStream bais = new ByteArrayInputStream(checkpointData);
+ DataInputStream in = new DataInputStream(bais)) {
+ int schemaManagerSerializerVersion = in.readInt();
+ int length = in.readInt();
+ byte[] serializedSchemaManager = new byte[length];
+ in.readFully(serializedSchemaManager);
+ schemaManager =
+ SchemaManager.SERIALIZER.deserialize(
+ schemaManagerSerializerVersion, serializedSchemaManager);
+ }
+ }
+
+ // -------------------------
+ // Event handler entrances (for schema mappers and sink operators)
+ // -------------------------
+ @Override
+ protected void handleGetOriginalSchemaRequest(
+ GetOriginalSchemaRequest request,
+ CompletableFuture<CoordinationResponse> responseFuture) {
+ throw new UnsupportedOperationException(
+ "In distributed topology, there's no centralized upstream schema table since they may evolve independently in various partitions.");
+ }
+
+ @Override
+ protected void handleCustomCoordinationRequest(
+ CoordinationRequest request, CompletableFuture<CoordinationResponse> responseFuture)
+ throws Exception {
+ if (request instanceof SchemaChangeRequest) {
+ handleSchemaEvolveRequest((SchemaChangeRequest) request, responseFuture);
+ } else {
+ throw new UnsupportedOperationException(
+ "Unknown coordination request type: " + request);
+ }
+ }
+
+ @Override
+ protected void handleFlushSuccessEvent(FlushSuccessEvent event) throws Exception {
+ LOG.info("Sink subtask {} succeed flushing.", event.getSinkSubTaskId());
+ flushedSinkWriters.add(event.getSinkSubTaskId());
+ }
+
+ @Override
+ protected void handleUnrecoverableError(String taskDescription, Throwable t) {
+ super.handleUnrecoverableError(taskDescription, t);
+ LOG.info("Current upstream table state: {}", upstreamSchemaTable);
+ pendingRequests.forEach(
+ (index, tuple) -> {
+ tuple.f1.completeExceptionally(t);
+ });
+ }
+
+ // -------------------------
+ // Schema evolving logic
+ // -------------------------
+
+ private void handleSchemaEvolveRequest(
+ SchemaChangeRequest request, CompletableFuture<CoordinationResponse> responseFuture)
+ throws Exception {
+ LOG.info("Coordinator received schema change request {}.", request);
+ if (!request.isNoOpRequest()) {
+ LOG.info("It's not an align request, will try to deduplicate.");
+ int eventSourcePartitionId = request.getSourceSubTaskId();
+ int handlingSinkSubTaskId = request.getSinkSubTaskId();
+ SchemaChangeEvent schemaChangeEvent = request.getSchemaChangeEvent();
+ Tuple2<Integer, SchemaChangeEvent> uniqueKey =
+ Tuple2.of(eventSourcePartitionId, schemaChangeEvent);
+ // Due to the existence of Partitioning Operator, any upstream event will be broadcast
+ // to sink N (N = sink parallelism) times.
+ // Only the first one should take effect, so we rewrite any other duplicated requests as
+ // a no-op align request.
+ alreadyHandledSchemaChangeEvents.put(uniqueKey, handlingSinkSubTaskId);
+ Collection<Integer> reportedSinkSubTasks =
+ alreadyHandledSchemaChangeEvents.get(uniqueKey);
+ if (reportedSinkSubTasks.size() == 1) {
+ LOG.info("It's a new request for {}, will handle it", uniqueKey);
+ updateUpstreamSchemaTable(
+ schemaChangeEvent.tableId(),
+ request.getSourceSubTaskId(),
+ schemaChangeEvent);
+ } else {
+ LOG.info(
+ "It's an already handled event {}. It has been handled by {}",
+ uniqueKey,
+ reportedSinkSubTasks);
+ request = SchemaChangeRequest.createNoOpRequest(handlingSinkSubTaskId);
+ }
+ // Moreover, if we've collected all sink subTasks' request, remove it from memory since
+ // no more will be possible.
+ if (reportedSinkSubTasks.size() == currentParallelism) {
+ LOG.info(
+ "All sink subTasks ({}) have already reported request {}. Remove it out of tracking.",
+ reportedSinkSubTasks,
+ uniqueKey);
+ alreadyHandledSchemaChangeEvents.removeAll(uniqueKey);
+ }
+ }
+
+ pendingRequests.put(request.getSinkSubTaskId(), Tuple2.of(request, responseFuture));
+
+ if (pendingRequests.size() == 1) {
+ Preconditions.checkState(
+ evolvingStatus.compareAndSet(
+ RequestStatus.IDLE, RequestStatus.WAITING_FOR_FLUSH),
+ "Unexpected evolving status: " + evolvingStatus.get());
+ LOG.info(
+ "Received the very-first schema change request {}. Switching from IDLE to WAITING_FOR_FLUSH.",
+ request);
+ }
+
+ // No else if, since currentParallelism might be == 1
+ if (pendingRequests.size() == currentParallelism) {
+ Preconditions.checkState(
+ evolvingStatus.compareAndSet(
+ RequestStatus.WAITING_FOR_FLUSH, RequestStatus.EVOLVING),
+ "Unexpected evolving status: " + evolvingStatus.get());
+ LOG.info(
+ "Received the last required schema change request {}. Switching from WAITING_FOR_FLUSH to EVOLVING.",
+ request);
+ startSchemaChange();
+ }
+ }
+
+ /**
+ * Tries to apply schema change event {@code schemaChangeEvent} to the combination of {@code
+ * tableId} and {@code sourcePartition}. Returns {@code true} if schema got changed, or {@code
+ * false} if nothing gets touched.
+ */
+ private void updateUpstreamSchemaTable(
+ TableId tableId, int sourcePartition, SchemaChangeEvent schemaChangeEvent) {
+ Schema oldSchema = upstreamSchemaTable.get(tableId, sourcePartition);
+ upstreamSchemaTable.put(
+ tableId,
+ sourcePartition,
+ SchemaUtils.applySchemaChangeEvent(oldSchema, schemaChangeEvent));
+ }
+
+ private void startSchemaChange() throws TimeoutException {
+ LOG.info("Starting to evolve schema.");
+ loopWhen(
+ () -> flushedSinkWriters.size() < currentParallelism,
+ () ->
+ LOG.info(
+ "Not all sink writers have successfully flushed. Expected {}, actual {}",
+ currentParallelism,
+ flushedSinkWriters),
+ rpcTimeout,
+ Duration.ofMillis(100));
+
+ LOG.info("All flushed. Going to evolve schema for pending requests: {}", pendingRequests);
+ flushedSinkWriters.clear();
+
+ // Deduce what schema change events should be applied to sink table
+ List<SchemaChangeEvent> deducedSchemaChangeEvents = deduceEvolvedSchemaChanges();
+
+ // And tries to apply it to external system
+ List<SchemaChangeEvent> successfullyAppliedSchemaChangeEvents = new ArrayList<>();
+ for (SchemaChangeEvent appliedSchemaChangeEvent : deducedSchemaChangeEvents) {
+ if (applyAndUpdateEvolvedSchemaChange(appliedSchemaChangeEvent)) {
+ successfullyAppliedSchemaChangeEvents.add(appliedSchemaChangeEvent);
+ }
+ }
+
+ // Then, we broadcast affected schema changes to mapper and release upstream
+ pendingRequests.forEach(
+ (subTaskId, tuple) -> {
+ LOG.info("Coordinator finishes pending future from {}", subTaskId);
+ tuple.f1.complete(
+ wrap(new SchemaChangeResponse(successfullyAppliedSchemaChangeEvents)));
+ });
+
+ pendingRequests.clear();
+
+ LOG.info("Finished schema evolving. Switching from EVOLVING to IDLE.");
+ Preconditions.checkState(
+ evolvingStatus.compareAndSet(RequestStatus.EVOLVING, RequestStatus.IDLE),
+ "RequestStatus should be EVOLVING when schema evolving finishes.");
+ }
+
+ private List<SchemaChangeEvent> deduceEvolvedSchemaChanges() {
+ List<SchemaChangeRequest> validSchemaChangeRequests =
+ pendingRequests.values().stream()
+ .map(e -> e.f0)
+ .filter(
+ request ->
+ !request.isNoOpRequest()) // Ignore alignment only requests
+ .collect(Collectors.toList());
+ LOG.info(
+ "Step 1 - Start deducing evolved schema change events for {}",
+ validSchemaChangeRequests);
+
+ // Firstly, based on changed upstream tables, infer a set of sink tables that might be
+ // affected by this event. Schema changes will be derived individually for each sink table.
+ Set<TableId> affectedSinkTableIds =
+ SchemaDerivator.getAffectedEvolvedTables(
+ router,
+ validSchemaChangeRequests.stream()
+ .map(rsr -> rsr.getSchemaChangeEvent().tableId())
+ .collect(Collectors.toSet()));
+ LOG.info("Step 2 - Affected sink tables are: {}", affectedSinkTableIds);
+
+ List<SchemaChangeEvent> evolvedSchemaChanges = new ArrayList<>();
+
+ // For each affected sink table, we may...
+ for (TableId affectedSinkTableId : affectedSinkTableIds) {
+ Schema currentSinkSchema =
+ schemaManager.getLatestEvolvedSchema(affectedSinkTableId).orElse(null);
+ LOG.info(
+ "Step 3.1 - For affected sink table {} with schema {}...",
+ affectedSinkTableId,
+ currentSinkSchema);
+
+ // ... reversely look up this affected sink table's upstream dependency.
+ Set<TableId> upstreamDependencies =
+ SchemaDerivator.reverseLookupDependingUpstreamTables(
+ router, affectedSinkTableId, upstreamSchemaTable);
+ Preconditions.checkState(
+ !upstreamDependencies.isEmpty(),
+ "An affected sink table's upstream dependency cannot be empty.");
+ LOG.info("Step 3.2 - upstream dependency tables are: {}", upstreamDependencies);
+
+ // Then, grab all upstream schemas from all known partitions and merge them.
+ Set<Schema> toBeMergedSchemas =
+ SchemaDerivator.reverseLookupDependingUpstreamSchemas(
+ router, affectedSinkTableId, upstreamSchemaTable);
+ LOG.info("Step 3.3 - Upstream dependency schemas are: {}.", toBeMergedSchemas);
+
+ // In distributed topology, schema will never be narrowed because current schema is
+ // in the merging base. Notice that current schema might be NULL if it's the first
+ // time we met a CreateTableEvent.
+ Schema mergedSchema = currentSinkSchema;
+ for (Schema toBeMergedSchema : toBeMergedSchemas) {
+ mergedSchema =
+ SchemaMergingUtils.getLeastCommonSchema(mergedSchema, toBeMergedSchema);
+ }
+ LOG.info("Step 3.4 - Deduced widest schema is: {}.", mergedSchema);
+
+ // Detect what schema changes we need to apply to get expected sink table.
+ List<SchemaChangeEvent> localEvolvedSchemaChanges =
+ SchemaMergingUtils.getSchemaDifference(
+ affectedSinkTableId, currentSinkSchema, mergedSchema);
+ LOG.info("Step 3.5 - Corresponding schema changes are: {}.", localEvolvedSchemaChanges);
+
+ // Finally, we normalize schema change events, including rewriting events by current
+ // schema change behavior configuration, dropping explicitly excluded schema change
+ // event types.
+ List<SchemaChangeEvent> normalizedEvents =
+ SchemaDerivator.normalizeSchemaChangeEvents(
+ currentSinkSchema,
+ localEvolvedSchemaChanges,
+ behavior,
+ metadataApplier);
+ LOG.info(
+ "Step 3.6 - After being normalized with {} behavior, final schema change events are: {}",
+ behavior,
+ normalizedEvents);
+ evolvedSchemaChanges.addAll(normalizedEvents);
+ }
+
+ return evolvedSchemaChanges;
+ }
+
+ private boolean applyAndUpdateEvolvedSchemaChange(SchemaChangeEvent schemaChangeEvent) {
+ try {
+ metadataApplier.applySchemaChange(schemaChangeEvent);
+ schemaManager.applyEvolvedSchemaChange(schemaChangeEvent);
+ LOG.info(
+ "Successfully applied schema change event {} to external system.",
+ schemaChangeEvent);
+ return true;
+ } catch (Throwable t) {
+ handleUnrecoverableError(
+ "Apply schema change event - " + schemaChangeEvent,
+ new FlinkRuntimeException(
+ "Failed to apply schema change event " + schemaChangeEvent + ".", t));
+ context.failJob(t);
+ throw t;
+ }
+ }
+
+ // -------------------------
+ // Utilities
+ // -------------------------
+
+ /**
+ * {@code IDLE}: Initial idling state, ready for requests. <br>
+ * {@code WAITING_FOR_FLUSH}: Waiting for all mappers to block & collecting enough FlushEvents.
+ * <br>
+ * {@code EVOLVING}: Applying schema change to sink.
+ */
+ private enum RequestStatus {
+ IDLE,
+ WAITING_FOR_FLUSH,
+ EVOLVING
+ }
+
+ @VisibleForTesting
+ public void emplaceOriginalSchema(TableId tableId, Integer subTaskId, Schema schema) {
+ upstreamSchemaTable.put(tableId, subTaskId, schema);
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistryProvider.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinatorProvider.java
old mode 100644
new mode 100755
similarity index 56%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistryProvider.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinatorProvider.java
index bc261e40f..f028f4cc9
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaRegistryProvider.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaCoordinatorProvider.java
@@ -15,24 +15,24 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
+package org.apache.flink.cdc.runtime.operators.schema.distributed;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
import org.apache.flink.cdc.common.route.RouteRule;
import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.runtime.operators.schema.common.CoordinatorExecutorThreadFactory;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
-import org.apache.flink.util.FatalExitExceptionHandler;
+import java.time.Duration;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-/** Provider of {@link SchemaRegistry}. */
+/** Provider for {@link SchemaCoordinator}. */
@Internal
-public class SchemaRegistryProvider implements OperatorCoordinator.Provider {
+public class SchemaCoordinatorProvider implements OperatorCoordinator.Provider {
private static final long serialVersionUID = 1L;
private final OperatorID operatorID;
@@ -40,18 +40,21 @@ public class SchemaRegistryProvider implements OperatorCoordinator.Provider {
private final MetadataApplier metadataApplier;
private final List<RouteRule> routingRules;
private final SchemaChangeBehavior schemaChangeBehavior;
+ private final Duration rpcTimeout;
- public SchemaRegistryProvider(
+ public SchemaCoordinatorProvider(
OperatorID operatorID,
String operatorName,
MetadataApplier metadataApplier,
List<RouteRule> routingRules,
- SchemaChangeBehavior schemaChangeBehavior) {
+ SchemaChangeBehavior schemaChangeBehavior,
+ Duration rpcTimeout) {
this.operatorID = operatorID;
this.operatorName = operatorName;
this.metadataApplier = metadataApplier;
this.routingRules = routingRules;
this.schemaChangeBehavior = schemaChangeBehavior;
+ this.rpcTimeout = rpcTimeout;
}
@Override
@@ -66,50 +69,13 @@ public class SchemaRegistryProvider implements OperatorCoordinator.Provider {
"schema-evolution-coordinator", context.getUserCodeClassloader());
ExecutorService coordinatorExecutor =
Executors.newSingleThreadExecutor(coordinatorThreadFactory);
- return new SchemaRegistry(
+ return new SchemaCoordinator(
operatorName,
context,
coordinatorExecutor,
metadataApplier,
routingRules,
- schemaChangeBehavior);
- }
-
- /** A thread factory class that provides some helper methods. */
- public static class CoordinatorExecutorThreadFactory implements ThreadFactory {
-
- private final String coordinatorThreadName;
- private final ClassLoader cl;
- private final Thread.UncaughtExceptionHandler errorHandler;
-
- private Thread t;
-
- CoordinatorExecutorThreadFactory(
- final String coordinatorThreadName, final ClassLoader contextClassLoader) {
- this(coordinatorThreadName, contextClassLoader, FatalExitExceptionHandler.INSTANCE);
- }
-
- CoordinatorExecutorThreadFactory(
- final String coordinatorThreadName,
- final ClassLoader contextClassLoader,
- final Thread.UncaughtExceptionHandler errorHandler) {
- this.coordinatorThreadName = coordinatorThreadName;
- this.cl = contextClassLoader;
- this.errorHandler = errorHandler;
- }
-
- @Override
- public synchronized Thread newThread(Runnable r) {
- if (t != null) {
- throw new Error(
- "This indicates that a fatal error has happened and caused the "
- + "coordinator executor thread to exit. Check the earlier logs"
- + "to see the root cause of the problem.");
- }
- t = new Thread(r, coordinatorThreadName);
- t.setContextClassLoader(cl);
- t.setUncaughtExceptionHandler(errorHandler);
- return t;
- }
+ schemaChangeBehavior,
+ rpcTimeout);
}
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java
new file mode 100755
index 000000000..eca0193c8
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.distributed;
+
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.FlushEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.exceptions.SchemaEvolveException;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.route.RouteRule;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils;
+import org.apache.flink.cdc.runtime.operators.schema.common.SchemaDerivator;
+import org.apache.flink.cdc.runtime.operators.schema.common.TableIdRouter;
+import org.apache.flink.cdc.runtime.operators.schema.common.metrics.SchemaOperatorMetrics;
+import org.apache.flink.cdc.runtime.operators.schema.distributed.event.SchemaChangeRequest;
+import org.apache.flink.cdc.runtime.operators.schema.distributed.event.SchemaChangeResponse;
+import org.apache.flink.cdc.runtime.partitioning.PartitioningEvent;
+import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
+import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
+import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.StreamTask;
+import org.apache.flink.util.SerializedValue;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.HashBasedTable;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Table;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+/** This operator merges upstream inferred schema into a centralized Schema Registry. */
+public class SchemaOperator extends AbstractStreamOperator<Event>
+ implements OneInputStreamOperator<PartitioningEvent, Event>, Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaOperator.class);
+
+ // Final fields that are set upon construction
+ private final Duration rpcTimeOut;
+ private final String timezone;
+ private final SchemaChangeBehavior schemaChangeBehavior;
+ private final List<RouteRule> routingRules;
+
+ public SchemaOperator(
+ List<RouteRule> routingRules,
+ Duration rpcTimeOut,
+ SchemaChangeBehavior schemaChangeBehavior,
+ String timezone) {
+ this.routingRules = routingRules;
+ this.rpcTimeOut = rpcTimeOut;
+ this.schemaChangeBehavior = schemaChangeBehavior;
+ this.timezone = timezone;
+ }
+
+ // Transient fields that are set when operator is running
+ private transient TaskOperatorEventGateway toCoordinator;
+ private transient int subTaskId;
+
+ // Trace schema by both TableId and source partition
+ private transient volatile Table<TableId, Integer, Schema> upstreamSchemaTable;
+ private transient volatile Map<TableId, Schema> evolvedSchemaMap;
+ private transient TableIdRouter tableIdRouter;
+ private transient SchemaDerivator derivator;
+ private transient SchemaOperatorMetrics schemaOperatorMetrics;
+
+ @Override
+ public void open() throws Exception {
+ super.open();
+ subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
+ upstreamSchemaTable = HashBasedTable.create();
+ evolvedSchemaMap = new HashMap<>();
+ tableIdRouter = new TableIdRouter(routingRules);
+ derivator = new SchemaDerivator();
+ this.schemaOperatorMetrics =
+ new SchemaOperatorMetrics(
+ getRuntimeContext().getMetricGroup(), schemaChangeBehavior);
+ }
+
+ @Override
+ public void setup(
+ StreamTask<?, ?> containingTask,
+ StreamConfig config,
+ Output<StreamRecord<Event>> output) {
+ super.setup(containingTask, config, output);
+ this.toCoordinator = containingTask.getEnvironment().getOperatorCoordinatorEventGateway();
+ }
+
+ @Override
+ public void processElement(StreamRecord<PartitioningEvent> streamRecord) throws Exception {
+ // Unpack partitioned events
+ PartitioningEvent partitioningEvent = streamRecord.getValue();
+ Event event = partitioningEvent.getPayload();
+ int sourcePartition = partitioningEvent.getSourcePartition();
+
+ if (event instanceof SchemaChangeEvent) {
+ schemaOperatorMetrics.increaseSchemaChangeEvents(1);
+
+ SchemaChangeEvent schemaChangeEvent = (SchemaChangeEvent) event;
+ TableId tableId = schemaChangeEvent.tableId();
+
+ // First, update upstream schema map unconditionally and it will never fail
+ Schema beforeSchema = upstreamSchemaTable.get(tableId, sourcePartition);
+ Schema afterSchema =
+ SchemaUtils.applySchemaChangeEvent(beforeSchema, schemaChangeEvent);
+ upstreamSchemaTable.put(tableId, sourcePartition, afterSchema);
+
+ // Check if we need to send a schema change request to the coordinator
+ if (!(schemaChangeEvent instanceof CreateTableEvent)) {
+ if (schemaChangeBehavior == SchemaChangeBehavior.IGNORE) {
+ LOG.info("{}> Schema change event {} has been ignored.", subTaskId, event);
+ schemaOperatorMetrics.increaseIgnoredSchemaChangeEvents(1);
+ return;
+ } else if (schemaChangeBehavior == SchemaChangeBehavior.EXCEPTION) {
+ throw new SchemaEvolveException(
+ schemaChangeEvent,
+ "Unexpected schema change events occurred in EXCEPTION mode. Job will fail now.");
+ }
+ }
+
+ // Then, notify this information to the coordinator
+ requestSchemaChange(
+ new SchemaChangeRequest(sourcePartition, subTaskId, schemaChangeEvent));
+ schemaOperatorMetrics.increaseFinishedSchemaChangeEvents(1);
+ } else if (event instanceof DataChangeEvent) {
+ DataChangeEvent dataChangeEvent = (DataChangeEvent) event;
+ TableId tableId = dataChangeEvent.tableId();
+
+ // First, we obtain the upstream schema corresponding to this data change event
+ Schema upstreamSchema =
+ upstreamSchemaTable.get(dataChangeEvent.tableId(), sourcePartition);
+
+ // Then, for each routing terminus, coerce data records to the expected schema
+ for (TableId sinkTableId : tableIdRouter.route(tableId)) {
+ Schema evolvedSchema = evolvedSchemaMap.get(sinkTableId);
+
+ DataChangeEvent coercedDataRecord =
+ derivator
+ .coerceDataRecord(
+ timezone,
+ DataChangeEvent.route(dataChangeEvent, sinkTableId),
+ upstreamSchema,
+ evolvedSchema)
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ String.format(
+ "Unable to coerce data record from %s (schema: %s) to %s (schema: %s)",
+ tableId,
+ upstreamSchema,
+ sinkTableId,
+ evolvedSchema)));
+ output.collect(new StreamRecord<>(coercedDataRecord));
+ }
+ } else {
+ throw new IllegalStateException(
+ subTaskId + "> SchemaOperator received an unexpected event: " + event);
+ }
+ }
+
+ private void requestSchemaChange(SchemaChangeRequest schemaChangeRequest) {
+ LOG.info("{}> Sent FlushEvent to downstream...", subTaskId);
+ output.collect(new StreamRecord<>(new FlushEvent(subTaskId)));
+
+ LOG.info("{}> Sending evolve request...", subTaskId);
+ SchemaChangeResponse response = sendRequestToCoordinator(schemaChangeRequest);
+
+ LOG.info("{}> Evolve request response: {}", subTaskId, response);
+
+ // Update local evolved schema cache
+ response.getSchemaEvolveResult()
+ .forEach(
+ schemaChangeEvent ->
+ evolvedSchemaMap.compute(
+ schemaChangeEvent.tableId(),
+ (tableId, schema) ->
+ SchemaUtils.applySchemaChangeEvent(
+ schema, schemaChangeEvent)));
+
+ // And emit schema change events to downstream
+ response.getSchemaEvolveResult().forEach(evt -> output.collect(new StreamRecord<>(evt)));
+ LOG.info(
+ "{}> Successfully updated evolved schema cache. Current state: {}",
+ subTaskId,
+ evolvedSchemaMap);
+ }
+
+ private <REQUEST extends CoordinationRequest, RESPONSE extends CoordinationResponse>
+ RESPONSE sendRequestToCoordinator(REQUEST request) {
+ try {
+ CompletableFuture<CoordinationResponse> responseFuture =
+ toCoordinator.sendRequestToCoordinator(
+ getOperatorID(), new SerializedValue<>(request));
+ return CoordinationResponseUtils.unwrap(
+ responseFuture.get(rpcTimeOut.toMillis(), TimeUnit.MILLISECONDS));
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Failed to send request to coordinator: " + request.toString(), e);
+ }
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperatorFactory.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperatorFactory.java
new file mode 100755
index 000000000..52378c15f
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperatorFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.distributed;
+
+import org.apache.flink.cdc.common.annotation.Internal;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.route.RouteRule;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.runtime.partitioning.PartitioningEvent;
+import org.apache.flink.runtime.jobgraph.OperatorID;
+import org.apache.flink.runtime.operators.coordination.OperatorCoordinator;
+import org.apache.flink.streaming.api.operators.CoordinatedOperatorFactory;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperatorFactory;
+import org.apache.flink.streaming.api.operators.SimpleOperatorFactory;
+
+import java.time.Duration;
+import java.util.List;
+
+/** Factory to create {@link SchemaOperator}. */
+@Internal
+public class SchemaOperatorFactory extends SimpleOperatorFactory<Event>
+ implements CoordinatedOperatorFactory<Event>,
+ OneInputStreamOperatorFactory<PartitioningEvent, Event> {
+ private static final long serialVersionUID = 1L;
+
+ private final MetadataApplier metadataApplier;
+ private final List<RouteRule> routingRules;
+ private final SchemaChangeBehavior schemaChangeBehavior;
+ private final Duration rpcTimeout;
+
+ public SchemaOperatorFactory(
+ MetadataApplier metadataApplier,
+ List<RouteRule> routingRules,
+ Duration rpcTimeout,
+ SchemaChangeBehavior schemaChangeBehavior,
+ String timezone) {
+ super(new SchemaOperator(routingRules, rpcTimeout, schemaChangeBehavior, timezone));
+ this.metadataApplier = metadataApplier;
+ this.routingRules = routingRules;
+ this.schemaChangeBehavior = schemaChangeBehavior;
+ this.rpcTimeout = rpcTimeout;
+ }
+
+ @Override
+ public OperatorCoordinator.Provider getCoordinatorProvider(
+ String operatorName, OperatorID operatorID) {
+ return new SchemaCoordinatorProvider(
+ operatorID,
+ operatorName,
+ metadataApplier,
+ routingRules,
+ schemaChangeBehavior,
+ rpcTimeout);
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/event/SchemaChangeRequest.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/event/SchemaChangeRequest.java
new file mode 100644
index 000000000..e287f7874
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/event/SchemaChangeRequest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.distributed.event;
+
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.utils.Preconditions;
+import org.apache.flink.cdc.runtime.operators.schema.distributed.SchemaCoordinator;
+import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
+
+/** Schema operator's request to {@link SchemaCoordinator} for merging an incompatible schema. */
+public class SchemaChangeRequest implements CoordinationRequest {
+ // Indicating which source subTask does this schema change event comes from.
+ private final int sourceSubTaskId;
+
+ // Indicating which schema mapper initiates this schema change request.
+ private final int sinkSubTaskId;
+
+ // A schema change event is uniquely bound to a sourceSubTaskId.
+ private final SchemaChangeEvent schemaChangeEvent;
+
+ public static SchemaChangeRequest createNoOpRequest(int sinkSubTaskId) {
+ return new SchemaChangeRequest(-1, sinkSubTaskId, null);
+ }
+
+ public SchemaChangeRequest(
+ int sourceSubTaskId, int sinkSubTaskId, SchemaChangeEvent schemaChangeEvent) {
+ this.sourceSubTaskId = sourceSubTaskId;
+ this.sinkSubTaskId = sinkSubTaskId;
+ this.schemaChangeEvent = schemaChangeEvent;
+ }
+
+ // Checking if this schema request was invalidated since it has been submitted by another
+ // downstream sink subTask before.
+ public boolean isNoOpRequest() {
+ return sourceSubTaskId == -1 || schemaChangeEvent == null;
+ }
+
+ public int getSourceSubTaskId() {
+ Preconditions.checkState(
+ !isNoOpRequest(), "Unable to fetch source subTaskId for an align event.");
+ return sourceSubTaskId;
+ }
+
+ public int getSinkSubTaskId() {
+ return sinkSubTaskId;
+ }
+
+ public SchemaChangeEvent getSchemaChangeEvent() {
+ Preconditions.checkState(
+ !isNoOpRequest(), "Unable to fetch source subTaskId for an align event.");
+ return schemaChangeEvent;
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/event/SchemaChangeResponse.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/event/SchemaChangeResponse.java
new file mode 100644
index 000000000..1c0b7eaab
--- /dev/null
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/event/SchemaChangeResponse.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.distributed.event;
+
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.runtime.operators.schema.distributed.SchemaCoordinator;
+import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
+
+import java.util.List;
+import java.util.Objects;
+
+/** Response from a {@link SchemaCoordinator} to broadcast a coordination consensus. */
+public class SchemaChangeResponse implements CoordinationResponse {
+
+ private final List<SchemaChangeEvent> schemaEvolveResult;
+
+ public SchemaChangeResponse(List<SchemaChangeEvent> schemaEvolveResult) {
+ this.schemaEvolveResult = schemaEvolveResult;
+ }
+
+ public List<SchemaChangeEvent> getSchemaEvolveResult() {
+ return schemaEvolveResult;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SchemaChangeResponse)) {
+ return false;
+ }
+ SchemaChangeResponse that = (SchemaChangeResponse) o;
+ return Objects.equals(schemaEvolveResult, that.schemaEvolveResult);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(schemaEvolveResult);
+ }
+
+ @Override
+ public String toString() {
+ return "SchemaChangeResponse{" + "schemaEvolveResult=" + schemaEvolveResult + '}';
+ }
+}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeProcessingResponse.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeProcessingResponse.java
deleted file mode 100644
index bf8d84f35..000000000
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeProcessingResponse.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.event;
-
-import org.apache.flink.cdc.common.event.SchemaChangeEvent;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
-import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
-
-/**
- * The response for {@link SchemaChangeResultRequest} from {@link SchemaRegistry} to {@link
- * SchemaOperator} if not apply {@link SchemaChangeEvent} in time.
- */
-public class SchemaChangeProcessingResponse implements CoordinationResponse {
-
- private static final long serialVersionUID = 1L;
-}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeResultRequest.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeResultRequest.java
deleted file mode 100644
index e53762c19..000000000
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeResultRequest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.event;
-
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
-import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
-
-/**
- * request from {@link SchemaOperator} to {@link SchemaRegistry} for getting result of applying
- * schema change.
- */
-public class SchemaChangeResultRequest implements CoordinationRequest {
-
- private static final long serialVersionUID = 1L;
-}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeResultResponse.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeResultResponse.java
deleted file mode 100644
index 7039ef086..000000000
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/SchemaChangeResultResponse.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.event;
-
-import org.apache.flink.cdc.common.event.SchemaChangeEvent;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
-import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
-
-import java.util.List;
-import java.util.Objects;
-
-/**
- * The response for {@link SchemaChangeResultRequest} from {@link SchemaRegistry} to {@link
- * SchemaOperator}.
- */
-public class SchemaChangeResultResponse implements CoordinationResponse {
-
- private static final long serialVersionUID = 1L;
-
- /**
- * Whether the SchemaOperator need to buffer data and the SchemaOperatorCoordinator need to wait
- * for flushing.
- */
- private final List<SchemaChangeEvent> finishedSchemaChangeEvents;
-
- public SchemaChangeResultResponse(List<SchemaChangeEvent> finishedSchemaChangeEvents) {
- this.finishedSchemaChangeEvents = finishedSchemaChangeEvents;
- }
-
- public List<SchemaChangeEvent> getFinishedSchemaChangeEvents() {
- return finishedSchemaChangeEvents;
- }
-
- @Override
- public String toString() {
- return "ReleaseUpstreamResponse{"
- + "finishedSchemaChangeEvents="
- + finishedSchemaChangeEvents
- + '}';
- }
-
- @Override
- public boolean equals(Object object) {
- if (this == object) {
- return true;
- }
- if (object == null || getClass() != object.getClass()) {
- return false;
- }
- SchemaChangeResultResponse that = (SchemaChangeResultResponse) object;
- return Objects.equals(finishedSchemaChangeEvents, that.finishedSchemaChangeEvents);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(finishedSchemaChangeEvents);
- }
-}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java
index 3767ba321..c0c6c5b6e 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java
@@ -113,7 +113,7 @@ public class DataSinkFunctionOperator extends StreamSink<Event> {
private void handleFlushEvent(FlushEvent event) throws Exception {
userFunction.finish();
schemaEvolutionClient.notifyFlushSuccess(
- getRuntimeContext().getIndexOfThisSubtask(), event.getTableId());
+ getRuntimeContext().getIndexOfThisSubtask(), event.getSourceSubTaskId());
}
private void emitLatestSchema(TableId tableId) throws Exception {
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java
index 554d79702..9c438f51c 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java
@@ -199,7 +199,7 @@ public class DataSinkWriterOperator<CommT> extends AbstractStreamOperator<Commit
private void handleFlushEvent(FlushEvent event) throws Exception {
copySinkWriter.flush(false);
schemaEvolutionClient.notifyFlushSuccess(
- getRuntimeContext().getIndexOfThisSubtask(), event.getTableId());
+ getRuntimeContext().getIndexOfThisSubtask(), event.getSourceSubTaskId());
}
private void emitLatestSchema(TableId tableId) throws Exception {
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/SchemaEvolutionClient.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/SchemaEvolutionClient.java
index aa256787f..20f89e961 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/SchemaEvolutionClient.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/SchemaEvolutionClient.java
@@ -19,14 +19,13 @@ package org.apache.flink.cdc.runtime.operators.sink;
import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.schema.coordinator.SchemaRegistry;
-import org.apache.flink.cdc.runtime.operators.schema.event.FlushSuccessEvent;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetEvolvedSchemaRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetEvolvedSchemaResponse;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetOriginalSchemaRequest;
-import org.apache.flink.cdc.runtime.operators.schema.event.GetOriginalSchemaResponse;
-import org.apache.flink.cdc.runtime.operators.schema.event.SinkWriterRegisterEvent;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.FlushSuccessEvent;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaRequest;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaResponse;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetOriginalSchemaRequest;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.GetOriginalSchemaResponse;
+import org.apache.flink.cdc.runtime.operators.schema.common.event.SinkWriterRegisterEvent;
+import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.util.SerializedValue;
@@ -34,11 +33,11 @@ import org.apache.flink.util.SerializedValue;
import java.io.IOException;
import java.util.Optional;
-import static org.apache.flink.cdc.runtime.operators.schema.event.CoordinationResponseUtils.unwrap;
+import static org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils.unwrap;
/**
- * Client for {@link DataSinkWriterOperator} interact with {@link SchemaRegistry} when table schema
- * evolution happened.
+ * Client for {@link DataSinkWriterOperator} interact with Schema Registry (Could be distributed or
+ * regular) when table schema evolution happened.
*/
public class SchemaEvolutionClient {
@@ -53,16 +52,17 @@ public class SchemaEvolutionClient {
this.schemaOperatorID = schemaOperatorID;
}
- /** send {@link SinkWriterRegisterEvent} to {@link SchemaRegistry}. */
+ /** send {@link SinkWriterRegisterEvent} to Schema Registry. */
public void registerSubtask(int subtask) throws IOException {
toCoordinator.sendOperatorEventToCoordinator(
schemaOperatorID, new SerializedValue<>(new SinkWriterRegisterEvent(subtask)));
}
- /** send {@link FlushSuccessEvent} to {@link SchemaRegistry}. */
- public void notifyFlushSuccess(int subtask, TableId tableId) throws IOException {
+ /** send {@link FlushSuccessEvent} to Schema Registry. */
+ public void notifyFlushSuccess(int subtask, int sourceSubTaskId) throws IOException {
toCoordinator.sendOperatorEventToCoordinator(
- schemaOperatorID, new SerializedValue<>(new FlushSuccessEvent(subtask, tableId)));
+ schemaOperatorID,
+ new SerializedValue<>(new FlushSuccessEvent(subtask, sourceSubTaskId)));
}
public Optional<Schema> getLatestEvolvedSchema(TableId tableId) throws Exception {
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java
index 56e243d31..d60721027 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java
@@ -33,6 +33,7 @@ import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.schema.Selectors;
import org.apache.flink.cdc.common.source.SupportedMetadataColumn;
import org.apache.flink.cdc.common.udf.UserDefinedFunctionContext;
+import org.apache.flink.cdc.common.utils.SchemaMergingUtils;
import org.apache.flink.cdc.common.utils.SchemaUtils;
import org.apache.flink.cdc.runtime.operators.transform.converter.PostTransformConverter;
import org.apache.flink.cdc.runtime.operators.transform.converter.PostTransformConverters;
@@ -381,7 +382,7 @@ public class PostTransformOperator extends AbstractStreamOperator<Event>
return schema;
}
- return SchemaUtils.inferWiderSchema(newSchemas);
+ return SchemaMergingUtils.getCommonSchema(newSchemas);
}
private List<Object> getUdfFunctionInstances() {
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java
index 4a3c51583..fd1d1d8c5 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java
@@ -34,6 +34,7 @@ import org.apache.flink.cdc.common.schema.Column;
import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.schema.Selectors;
import org.apache.flink.cdc.common.source.SupportedMetadataColumn;
+import org.apache.flink.cdc.common.utils.Preconditions;
import org.apache.flink.cdc.common.utils.SchemaUtils;
import org.apache.flink.cdc.runtime.parser.TransformParser;
import org.apache.flink.runtime.state.StateInitializationContext;
@@ -75,6 +76,7 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
private List<UserDefinedFunctionDescriptor> udfDescriptors;
private Map<TableId, PreTransformProcessor> preTransformProcessorMap;
private Map<TableId, Boolean> hasAsteriskMap;
+ private final boolean canContainDistributedTables;
public static PreTransformOperator.Builder newBuilder() {
return new PreTransformOperator.Builder();
@@ -83,6 +85,7 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
/** Builder of {@link PreTransformOperator}. */
public static class Builder {
private final List<TransformRule> transformRules = new ArrayList<>();
+ private boolean canContainDistributedTables;
private final List<Tuple3<String, String, Map<String, String>>> udfFunctions =
new ArrayList<>();
@@ -130,20 +133,30 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
return this;
}
+ public PreTransformOperator.Builder canContainDistributedTables(
+ boolean canContainDistributedTables) {
+ this.canContainDistributedTables = canContainDistributedTables;
+ return this;
+ }
+
public PreTransformOperator build() {
- return new PreTransformOperator(transformRules, udfFunctions);
+ return new PreTransformOperator(
+ transformRules, udfFunctions, canContainDistributedTables);
}
}
private PreTransformOperator(
List<TransformRule> transformRules,
- List<Tuple3<String, String, Map<String, String>>> udfFunctions) {
- this.transformRules = transformRules;
+ List<Tuple3<String, String, Map<String, String>>> udfFunctions,
+ boolean canContainDistributedTables) {
this.preTransformChangeInfoMap = new ConcurrentHashMap<>();
this.preTransformProcessorMap = new ConcurrentHashMap<>();
this.schemaMetadataTransformers = new ArrayList<>();
this.chainingStrategy = ChainingStrategy.ALWAYS;
+
+ this.transformRules = transformRules;
this.udfFunctions = udfFunctions;
+ this.canContainDistributedTables = canContainDistributedTables;
}
@Override
@@ -186,6 +199,12 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
@Override
public void initializeState(StateInitializationContext context) throws Exception {
super.initializeState(context);
+ if (canContainDistributedTables) {
+ // In distributed mode, we don't have a globally consistent schema for each partition.
+ // It's not meaningful to persist them to state. Instead, we rely on each source
+ // partition to send fresh CreateTableEvent to instantiate each event flow.
+ return;
+ }
OperatorStateStore stateStore = context.getOperatorStateStore();
ListStateDescriptor<byte[]> descriptor =
new ListStateDescriptor<>("originalSchemaState", byte[].class);
@@ -216,6 +235,10 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
super.snapshotState(context);
+ if (canContainDistributedTables) {
+ // Same reason in this#initializeState.
+ return;
+ }
state.update(
new ArrayList<>(
preTransformChangeInfoMap.values().stream()
@@ -443,20 +466,25 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
private DataChangeEvent processDataChangeEvent(DataChangeEvent dataChangeEvent) {
if (!transforms.isEmpty()) {
- PreTransformProcessor preTransformProcessor =
- preTransformProcessorMap.get(dataChangeEvent.tableId());
+ TableId tableId = dataChangeEvent.tableId();
+ PreTransformProcessor processor = preTransformProcessorMap.get(tableId);
+ Preconditions.checkArgument(
+ processor != null,
+ "Transform operator receives a data change event from table %s without a full schema view. "
+ + "This might happen if source with distributed tables doesn't emit CreateTableEvent first after fail-over. "
+ + "This is likely a bug, please consider filing an issue.",
+ tableId);
+
BinaryRecordData before = (BinaryRecordData) dataChangeEvent.before();
BinaryRecordData after = (BinaryRecordData) dataChangeEvent.after();
if (before != null) {
- BinaryRecordData projectedBefore =
- preTransformProcessor.processFillDataField(before);
+ BinaryRecordData projectedBefore = processor.processFillDataField(before);
dataChangeEvent = DataChangeEvent.projectBefore(dataChangeEvent, projectedBefore);
}
if (after != null) {
- BinaryRecordData projectedAfter = preTransformProcessor.processFillDataField(after);
+ BinaryRecordData projectedAfter = processor.processFillDataField(after);
dataChangeEvent = DataChangeEvent.projectAfter(dataChangeEvent, projectedAfter);
}
- return dataChangeEvent;
}
return dataChangeEvent;
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PrePartitionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java
similarity index 52%
copy from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PrePartitionOperator.java
copy to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java
index 938e6950d..3cce8923b 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PrePartitionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java
@@ -20,52 +20,42 @@ package org.apache.flink.cdc.runtime.partitioning;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.event.DataChangeEvent;
import org.apache.flink.cdc.common.event.Event;
-import org.apache.flink.cdc.common.event.FlushEvent;
import org.apache.flink.cdc.common.event.SchemaChangeEvent;
import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.function.HashFunction;
import org.apache.flink.cdc.common.function.HashFunctionProvider;
import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
-import org.apache.flink.cdc.runtime.operators.sink.SchemaEvolutionClient;
+import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.cdc.runtime.serializer.event.EventSerializer;
-import org.apache.flink.runtime.jobgraph.OperatorID;
-import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.runtime.state.StateSnapshotContext;
import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.shaded.guava31.com.google.common.cache.CacheBuilder;
-import org.apache.flink.shaded.guava31.com.google.common.cache.CacheLoader;
-import org.apache.flink.shaded.guava31.com.google.common.cache.LoadingCache;
-
import java.io.Serializable;
-import java.time.Duration;
-import java.util.Optional;
+import java.util.HashMap;
+import java.util.Map;
-/** Operator for processing events from {@link SchemaOperator} before {@link EventPartitioner}. */
+/** Operator for processing events from upstream before flowing to {@link SchemaOperator}. */
@Internal
-public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
+public class DistributedPrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
implements OneInputStreamOperator<Event, PartitioningEvent>, Serializable {
-
private static final long serialVersionUID = 1L;
- private static final Duration CACHE_EXPIRE_DURATION = Duration.ofDays(1);
- private final OperatorID schemaOperatorId;
private final int downstreamParallelism;
private final HashFunctionProvider<DataChangeEvent> hashFunctionProvider;
- private transient SchemaEvolutionClient schemaEvolutionClient;
- private transient LoadingCache<TableId, HashFunction<DataChangeEvent>> cachedHashFunctions;
+ // Schema and HashFunctionMap used in schema inferencing mode.
+ private transient Map<TableId, Schema> schemaMap;
+ private transient Map<TableId, HashFunction<DataChangeEvent>> hashFunctionMap;
- public PrePartitionOperator(
- OperatorID schemaOperatorId,
- int downstreamParallelism,
- HashFunctionProvider<DataChangeEvent> hashFunctionProvider) {
+ private transient int subTaskId;
+
+ public DistributedPrePartitionOperator(
+ int downstreamParallelism, HashFunctionProvider<DataChangeEvent> hashFunctionProvider) {
this.chainingStrategy = ChainingStrategy.ALWAYS;
- this.schemaOperatorId = schemaOperatorId;
this.downstreamParallelism = downstreamParallelism;
this.hashFunctionProvider = hashFunctionProvider;
}
@@ -73,36 +63,45 @@ public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEve
@Override
public void open() throws Exception {
super.open();
- TaskOperatorEventGateway toCoordinator =
- getContainingTask().getEnvironment().getOperatorCoordinatorEventGateway();
- schemaEvolutionClient = new SchemaEvolutionClient(toCoordinator, schemaOperatorId);
- cachedHashFunctions = createCache();
+ subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
+ schemaMap = new HashMap<>();
+ hashFunctionMap = new HashMap<>();
}
@Override
public void processElement(StreamRecord<Event> element) throws Exception {
Event event = element.getValue();
if (event instanceof SchemaChangeEvent) {
+ SchemaChangeEvent schemaChangeEvent = (SchemaChangeEvent) event;
+ TableId tableId = schemaChangeEvent.tableId();
+
+ // Update schema map
+ schemaMap.compute(
+ tableId,
+ (tId, oldSchema) ->
+ SchemaUtils.applySchemaChangeEvent(oldSchema, schemaChangeEvent));
+
// Update hash function
- TableId tableId = ((SchemaChangeEvent) event).tableId();
- cachedHashFunctions.put(tableId, recreateHashFunction(tableId));
+ hashFunctionMap.put(tableId, recreateHashFunction(tableId));
+
// Broadcast SchemaChangeEvent
broadcastEvent(event);
- } else if (event instanceof FlushEvent) {
- // Broadcast FlushEvent
- broadcastEvent(event);
} else if (event instanceof DataChangeEvent) {
// Partition DataChangeEvent by table ID and primary keys
- partitionBy(((DataChangeEvent) event));
+ partitionBy((DataChangeEvent) event);
+ } else {
+ throw new IllegalStateException(
+ subTaskId + "> PrePartition operator received an unexpected event: " + event);
}
}
- private void partitionBy(DataChangeEvent dataChangeEvent) throws Exception {
+ private void partitionBy(DataChangeEvent dataChangeEvent) {
output.collect(
new StreamRecord<>(
- new PartitioningEvent(
+ PartitioningEvent.ofDistributed(
dataChangeEvent,
- cachedHashFunctions
+ subTaskId,
+ hashFunctionMap
.get(dataChangeEvent.tableId())
.hashcode(dataChangeEvent)
% downstreamParallelism)));
@@ -113,40 +112,13 @@ public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEve
// Deep-copying each event is required since downstream subTasks might run in the same
// JVM
Event copiedEvent = EventSerializer.INSTANCE.copy(toBroadcast);
- output.collect(new StreamRecord<>(new PartitioningEvent(copiedEvent, i)));
+ output.collect(
+ new StreamRecord<>(PartitioningEvent.ofDistributed(copiedEvent, subTaskId, i)));
}
}
- private Schema loadLatestSchemaFromRegistry(TableId tableId) {
- Optional<Schema> schema;
- try {
- schema = schemaEvolutionClient.getLatestEvolvedSchema(tableId);
- } catch (Exception e) {
- throw new RuntimeException(
- String.format("Failed to request latest schema for table \"%s\"", tableId), e);
- }
- if (!schema.isPresent()) {
- throw new IllegalStateException(
- String.format(
- "Schema is never registered or outdated for table \"%s\"", tableId));
- }
- return schema.get();
- }
-
private HashFunction<DataChangeEvent> recreateHashFunction(TableId tableId) {
- return hashFunctionProvider.getHashFunction(tableId, loadLatestSchemaFromRegistry(tableId));
- }
-
- private LoadingCache<TableId, HashFunction<DataChangeEvent>> createCache() {
- return CacheBuilder.newBuilder()
- .expireAfterAccess(CACHE_EXPIRE_DURATION)
- .build(
- new CacheLoader<TableId, HashFunction<DataChangeEvent>>() {
- @Override
- public HashFunction<DataChangeEvent> load(TableId key) {
- return recreateHashFunction(key);
- }
- });
+ return hashFunctionProvider.getHashFunction(tableId, schemaMap.get(tableId));
}
@Override
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PartitioningEvent.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PartitioningEvent.java
index 076a84392..b32c21bf7 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PartitioningEvent.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PartitioningEvent.java
@@ -29,10 +29,28 @@ import java.util.Objects;
@Internal
public class PartitioningEvent implements Event {
private final Event payload;
+ private final int sourcePartition;
private final int targetPartition;
- public PartitioningEvent(Event payload, int targetPartition) {
+ /**
+ * For partitioning events with regular topology, source partition information is not necessary.
+ */
+ public static PartitioningEvent ofRegular(Event payload, int targetPartition) {
+ return new PartitioningEvent(payload, -1, targetPartition);
+ }
+
+ /**
+ * For distributed topology, we need to track its upstream source subTask ID to correctly
+ * distinguish events from different partitions.
+ */
+ public static PartitioningEvent ofDistributed(
+ Event payload, int sourcePartition, int targetPartition) {
+ return new PartitioningEvent(payload, sourcePartition, targetPartition);
+ }
+
+ private PartitioningEvent(Event payload, int sourcePartition, int targetPartition) {
this.payload = payload;
+ this.sourcePartition = sourcePartition;
this.targetPartition = targetPartition;
}
@@ -40,6 +58,10 @@ public class PartitioningEvent implements Event {
return payload;
}
+ public int getSourcePartition() {
+ return sourcePartition;
+ }
+
public int getTargetPartition() {
return targetPartition;
}
@@ -53,12 +75,14 @@ public class PartitioningEvent implements Event {
return false;
}
PartitioningEvent that = (PartitioningEvent) o;
- return targetPartition == that.targetPartition && Objects.equals(payload, that.payload);
+ return sourcePartition == that.sourcePartition
+ && targetPartition == that.targetPartition
+ && Objects.equals(payload, that.payload);
}
@Override
public int hashCode() {
- return Objects.hash(payload, targetPartition);
+ return Objects.hash(payload, sourcePartition, targetPartition);
}
@Override
@@ -66,6 +90,8 @@ public class PartitioningEvent implements Event {
return "PartitioningEvent{"
+ "payload="
+ payload
+ + ", sourcePartition="
+ + sourcePartition
+ ", targetPartition="
+ targetPartition
+ '}';
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PrePartitionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java
similarity index 93%
rename from flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PrePartitionOperator.java
rename to flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java
index 938e6950d..38f177472 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/PrePartitionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java
@@ -26,7 +26,7 @@ import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.function.HashFunction;
import org.apache.flink.cdc.common.function.HashFunctionProvider;
import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.runtime.operators.schema.SchemaOperator;
+import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.cdc.runtime.operators.sink.SchemaEvolutionClient;
import org.apache.flink.cdc.runtime.serializer.event.EventSerializer;
import org.apache.flink.runtime.jobgraph.OperatorID;
@@ -45,9 +45,12 @@ import java.io.Serializable;
import java.time.Duration;
import java.util.Optional;
-/** Operator for processing events from {@link SchemaOperator} before {@link EventPartitioner}. */
+/**
+ * Operator for processing events from {@link SchemaOperator} before {@link EventPartitioner} with
+ * regular topology.
+ */
@Internal
-public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
+public class RegularPrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
implements OneInputStreamOperator<Event, PartitioningEvent>, Serializable {
private static final long serialVersionUID = 1L;
@@ -60,7 +63,7 @@ public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEve
private transient SchemaEvolutionClient schemaEvolutionClient;
private transient LoadingCache<TableId, HashFunction<DataChangeEvent>> cachedHashFunctions;
- public PrePartitionOperator(
+ public RegularPrePartitionOperator(
OperatorID schemaOperatorId,
int downstreamParallelism,
HashFunctionProvider<DataChangeEvent> hashFunctionProvider) {
@@ -100,7 +103,7 @@ public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEve
private void partitionBy(DataChangeEvent dataChangeEvent) throws Exception {
output.collect(
new StreamRecord<>(
- new PartitioningEvent(
+ PartitioningEvent.ofRegular(
dataChangeEvent,
cachedHashFunctions
.get(dataChangeEvent.tableId())
@@ -113,7 +116,7 @@ public class PrePartitionOperator extends AbstractStreamOperator<PartitioningEve
// Deep-copying each event is required since downstream subTasks might run in the same
// JVM
Event copiedEvent = EventSerializer.INSTANCE.copy(toBroadcast);
- output.collect(new StreamRecord<>(new PartitioningEvent(copiedEvent, i)));
+ output.collect(new StreamRecord<>(PartitioningEvent.ofRegular(copiedEvent, i)));
}
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/EventSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/EventSerializer.java
index c8f34020d..8a17589fc 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/EventSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/EventSerializer.java
@@ -61,7 +61,8 @@ public final class EventSerializer extends TypeSerializerSingleton<Event> {
@Override
public Event copy(Event from) {
if (from instanceof FlushEvent) {
- return new FlushEvent(tableIdSerializer.copy(((FlushEvent) from).getTableId()));
+ FlushEvent flushEvent = (FlushEvent) from;
+ return new FlushEvent(((FlushEvent) from).getSourceSubTaskId());
} else if (from instanceof SchemaChangeEvent) {
return schemaChangeEventSerializer.copy((SchemaChangeEvent) from);
} else if (from instanceof DataChangeEvent) {
@@ -84,7 +85,7 @@ public final class EventSerializer extends TypeSerializerSingleton<Event> {
public void serialize(Event record, DataOutputView target) throws IOException {
if (record instanceof FlushEvent) {
enumSerializer.serialize(EventClass.FLUSH_EVENT, target);
- tableIdSerializer.serialize(((FlushEvent) record).getTableId(), target);
+ target.writeInt(((FlushEvent) record).getSourceSubTaskId());
} else if (record instanceof SchemaChangeEvent) {
enumSerializer.serialize(EventClass.SCHEME_CHANGE_EVENT, target);
schemaChangeEventSerializer.serialize((SchemaChangeEvent) record, target);
@@ -101,7 +102,7 @@ public final class EventSerializer extends TypeSerializerSingleton<Event> {
EventClass eventClass = enumSerializer.deserialize(source);
switch (eventClass) {
case FLUSH_EVENT:
- return new FlushEvent(tableIdSerializer.deserialize(source));
+ return new FlushEvent(source.readInt());
case DATA_CHANGE_EVENT:
return dataChangeEventSerializer.deserialize(source);
case SCHEME_CHANGE_EVENT:
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/PartitioningEventSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/PartitioningEventSerializer.java
index 0cce884af..44217674e 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/PartitioningEventSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/PartitioningEventSerializer.java
@@ -43,13 +43,15 @@ public class PartitioningEventSerializer extends TypeSerializerSingleton<Partiti
@Override
public PartitioningEvent createInstance() {
- return new PartitioningEvent(null, -1);
+ return PartitioningEvent.ofDistributed(null, -1, -1);
}
@Override
public PartitioningEvent copy(PartitioningEvent from) {
- return new PartitioningEvent(
- eventSerializer.copy(from.getPayload()), from.getTargetPartition());
+ return PartitioningEvent.ofDistributed(
+ eventSerializer.copy(from.getPayload()),
+ from.getSourcePartition(),
+ from.getTargetPartition());
}
@Override
@@ -65,14 +67,16 @@ public class PartitioningEventSerializer extends TypeSerializerSingleton<Partiti
@Override
public void serialize(PartitioningEvent record, DataOutputView target) throws IOException {
eventSerializer.serialize(record.getPayload(), target);
+ target.writeInt(record.getSourcePartition());
target.writeInt(record.getTargetPartition());
}
@Override
public PartitioningEvent deserialize(DataInputView source) throws IOException {
Event payload = eventSerializer.deserialize(source);
+ int sourcePartition = source.readInt();
int targetPartition = source.readInt();
- return new PartitioningEvent(payload, targetPartition);
+ return PartitioningEvent.ofDistributed(payload, sourcePartition, targetPartition);
}
@Override
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/SchemaChangeEventSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/SchemaChangeEventSerializer.java
index 4f556b5ca..7a5da9c72 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/SchemaChangeEventSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/event/SchemaChangeEventSerializer.java
@@ -67,6 +67,11 @@ public final class SchemaChangeEventSerializer extends TypeSerializerSingleton<S
public TableId tableId() {
return TableId.tableId("unknown", "unknown", "unknown");
}
+
+ @Override
+ public SchemaChangeEvent copy(TableId newTableId) {
+ return null;
+ }
};
}
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaDerivatorTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaDerivatorTest.java
new file mode 100644
index 000000000..f921f024c
--- /dev/null
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaDerivatorTest.java
@@ -0,0 +1,602 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.cdc.common.event.AddColumnEvent;
+import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
+import org.apache.flink.cdc.common.event.CreateTableEvent;
+import org.apache.flink.cdc.common.event.DropColumnEvent;
+import org.apache.flink.cdc.common.event.DropTableEvent;
+import org.apache.flink.cdc.common.event.RenameColumnEvent;
+import org.apache.flink.cdc.common.event.SchemaChangeEvent;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.event.TruncateTableEvent;
+import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
+import org.apache.flink.cdc.common.schema.Column;
+import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.common.types.DataTypes;
+
+import org.apache.flink.shaded.guava31.com.google.common.collect.HashBasedTable;
+import org.apache.flink.shaded.guava31.com.google.common.collect.Table;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
+
+/** Unit test for {@link SchemaDerivator}. */
+public class SchemaDerivatorTest extends SchemaTestBase {
+ private static final Set<TableId> ALL_UPSTREAM_TABLE_IDS =
+ IntStream.rangeClosed(0, 5)
+ .boxed()
+ .flatMap(
+ dbIdx ->
+ IntStream.rangeClosed(1, 3)
+ .mapToObj(
+ tblIdx ->
+ TableId.tableId(
+ "db_" + dbIdx,
+ "table_" + tblIdx)))
+ .collect(Collectors.toSet());
+
+ private static final Table<TableId, Integer, Schema> DUMMY_UPSTREAM_SCHEMA_TABLE =
+ HashBasedTable.create();
+
+ static {
+ ALL_UPSTREAM_TABLE_IDS.forEach(
+ tbl -> {
+ for (int i = 0; i < 2; i++) {
+ DUMMY_UPSTREAM_SCHEMA_TABLE.put(
+ tbl,
+ i,
+ Schema.newBuilder()
+ .physicalColumn(
+ "id",
+ DataTypes.INT(),
+ String.format("%s @ %s", tbl, i))
+ .build());
+ }
+ });
+ }
+
+ private static final TableId NORMALIZE_TEST_TABLE_ID = TableId.parse("foo.bar.baz");
+ private static final Schema NORMALIZE_TEST_SCHEMA =
+ Schema.newBuilder()
+ .physicalColumn("id", DataTypes.INT())
+ .physicalColumn("name", DataTypes.VARCHAR(128))
+ .physicalColumn("age", DataTypes.FLOAT())
+ .physicalColumn("notes", DataTypes.STRING())
+ .build();
+ private static final MetadataApplier MOCKED_METADATA_APPLIER =
+ schemaChangeEvent -> {
+ throw new UnsupportedOperationException(
+ "Mocked metadata applier doesn't really do anything.");
+ };
+
+ private static Set<String> getAffected(String... tables) {
+ return SchemaDerivator.getAffectedEvolvedTables(
+ TABLE_ID_ROUTER,
+ Arrays.stream(tables).map(TableId::parse).collect(Collectors.toSet()))
+ .stream()
+ .map(TableId::toString)
+ .collect(Collectors.toSet());
+ }
+
+ private static Set<String> reverseLookupTable(String tableId) {
+ return SchemaDerivator.reverseLookupDependingUpstreamTables(
+ TABLE_ID_ROUTER, TableId.parse(tableId), DUMMY_UPSTREAM_SCHEMA_TABLE)
+ .stream()
+ .map(TableId::toString)
+ .collect(Collectors.toSet());
+ }
+
+ private static Set<String> reverseLookupSchema(String tableId) {
+ return SchemaDerivator.reverseLookupDependingUpstreamSchemas(
+ TABLE_ID_ROUTER, TableId.parse(tableId), DUMMY_UPSTREAM_SCHEMA_TABLE)
+ .stream()
+ .map(schema -> schema.getColumn("id").get().getComment())
+ .collect(Collectors.toSet());
+ }
+
+ private static List<SchemaChangeEvent> normalizeEvent(
+ SchemaChangeBehavior behavior, SchemaChangeEvent... events) {
+ return SchemaDerivator.normalizeSchemaChangeEvents(
+ NORMALIZE_TEST_SCHEMA,
+ Arrays.stream(events).collect(Collectors.toList()),
+ behavior,
+ MOCKED_METADATA_APPLIER);
+ }
+
+ @Test
+ void testGetAffectedEvolvedTables() {
+ assertThat(getAffected()).isEmpty();
+
+ // No routing rule, behaves like one-to-one routing
+ assertThat(getAffected("db_0.table_1")).containsExactlyInAnyOrder("db_0.table_1");
+ assertThat(getAffected("db_0.table_2")).containsExactlyInAnyOrder("db_0.table_2");
+ assertThat(getAffected("db_0.table_3")).containsExactlyInAnyOrder("db_0.table_3");
+ assertThat(getAffected("db_0.table_1", "db_0.table_2"))
+ .containsExactlyInAnyOrder("db_0.table_1", "db_0.table_2");
+ assertThat(getAffected("db_0.table_1", "db_0.table_3"))
+ .containsExactlyInAnyOrder("db_0.table_1", "db_0.table_3");
+ assertThat(getAffected("db_0.table_2", "db_0.table_3"))
+ .containsExactlyInAnyOrder("db_0.table_2", "db_0.table_3");
+ assertThat(getAffected("db_0.table_1", "db_0.table_2", "db_0.table_3"))
+ .containsExactlyInAnyOrder("db_0.table_1", "db_0.table_2", "db_0.table_3");
+
+ // One-to-one routing
+ assertThat(getAffected("db_1.table_1")).containsExactlyInAnyOrder("db_1.table_1");
+ assertThat(getAffected("db_1.table_2")).containsExactlyInAnyOrder("db_1.table_2");
+ assertThat(getAffected("db_1.table_3")).containsExactlyInAnyOrder("db_1.table_3");
+ assertThat(getAffected("db_1.table_1", "db_1.table_2"))
+ .containsExactlyInAnyOrder("db_1.table_1", "db_1.table_2");
+ assertThat(getAffected("db_1.table_1", "db_1.table_3"))
+ .containsExactlyInAnyOrder("db_1.table_1", "db_1.table_3");
+ assertThat(getAffected("db_1.table_2", "db_1.table_3"))
+ .containsExactlyInAnyOrder("db_1.table_2", "db_1.table_3");
+ assertThat(getAffected("db_1.table_1", "db_1.table_2", "db_1.table_3"))
+ .containsExactlyInAnyOrder("db_1.table_1", "db_1.table_2", "db_1.table_3");
+
+ // One-to-one routing, but twisted
+ assertThat(getAffected("db_2.table_1")).containsExactlyInAnyOrder("db_2.table_2");
+ assertThat(getAffected("db_2.table_2")).containsExactlyInAnyOrder("db_2.table_3");
+ assertThat(getAffected("db_2.table_3")).containsExactlyInAnyOrder("db_2.table_1");
+ assertThat(getAffected("db_2.table_1", "db_2.table_2"))
+ .containsExactlyInAnyOrder("db_2.table_2", "db_2.table_3");
+ assertThat(getAffected("db_2.table_1", "db_2.table_3"))
+ .containsExactlyInAnyOrder("db_2.table_2", "db_2.table_1");
+ assertThat(getAffected("db_2.table_2", "db_2.table_3"))
+ .containsExactlyInAnyOrder("db_2.table_3", "db_2.table_1");
+ assertThat(getAffected("db_2.table_1", "db_2.table_2", "db_2.table_3"))
+ .containsExactlyInAnyOrder("db_2.table_2", "db_2.table_3", "db_2.table_1");
+
+ // Merging-table routing
+ assertThat(getAffected("db_3.table_1")).containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(getAffected("db_3.table_2")).containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(getAffected("db_3.table_3")).containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(getAffected("db_3.table_1", "db_3.table_2"))
+ .containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(getAffected("db_3.table_1", "db_3.table_3"))
+ .containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(getAffected("db_3.table_2", "db_3.table_3"))
+ .containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(getAffected("db_3.table_1", "db_3.table_2", "db_3.table_3"))
+ .containsExactlyInAnyOrder("db_3.table_merged");
+
+ // Broadcasting routing
+ assertThat(getAffected("db_4.table_1"))
+ .containsExactlyInAnyOrder("db_4.table_a", "db_4.table_b", "db_4.table_c");
+ assertThat(getAffected("db_4.table_2"))
+ .containsExactlyInAnyOrder("db_4.table_b", "db_4.table_c");
+ assertThat(getAffected("db_4.table_3")).containsExactlyInAnyOrder("db_4.table_c");
+ assertThat(getAffected("db_4.table_1", "db_4.table_2"))
+ .containsExactlyInAnyOrder("db_4.table_a", "db_4.table_b", "db_4.table_c");
+ assertThat(getAffected("db_4.table_1", "db_4.table_3"))
+ .containsExactlyInAnyOrder("db_4.table_a", "db_4.table_b", "db_4.table_c");
+ assertThat(getAffected("db_4.table_2", "db_4.table_3"))
+ .containsExactlyInAnyOrder("db_4.table_b", "db_4.table_c");
+ assertThat(getAffected("db_4.table_1", "db_4.table_2", "db_4.table_3"))
+ .containsExactlyInAnyOrder("db_4.table_a", "db_4.table_b", "db_4.table_c");
+
+ // RepSym routing
+ assertThat(getAffected("db_5.table_1", "db_5.table_2"))
+ .containsExactlyInAnyOrder(
+ "db_5.prefix_table_1_suffix", "db_5.prefix_table_2_suffix");
+ assertThat(getAffected("db_5.table_1", "db_5.table_3"))
+ .containsExactlyInAnyOrder(
+ "db_5.prefix_table_1_suffix", "db_5.prefix_table_3_suffix");
+ assertThat(getAffected("db_5.table_2", "db_5.table_3"))
+ .containsExactlyInAnyOrder(
+ "db_5.prefix_table_2_suffix", "db_5.prefix_table_3_suffix");
+ assertThat(getAffected("db_5.table_1", "db_5.table_2", "db_5.table_3"))
+ .containsExactlyInAnyOrder(
+ "db_5.prefix_table_1_suffix",
+ "db_5.prefix_table_2_suffix",
+ "db_5.prefix_table_3_suffix");
+ }
+
+ @Test
+ void testReverseLookupDependingUpstreamTables() {
+ assertThat(reverseLookupTable("db_0.table_1")).containsExactlyInAnyOrder("db_0.table_1");
+ assertThat(reverseLookupTable("db_0.table_2")).containsExactlyInAnyOrder("db_0.table_2");
+ assertThat(reverseLookupTable("db_0.table_3")).containsExactlyInAnyOrder("db_0.table_3");
+
+ assertThat(reverseLookupTable("db_1.table_1")).containsExactlyInAnyOrder("db_1.table_1");
+ assertThat(reverseLookupTable("db_1.table_2")).containsExactlyInAnyOrder("db_1.table_2");
+ assertThat(reverseLookupTable("db_1.table_3")).containsExactlyInAnyOrder("db_1.table_3");
+
+ assertThat(reverseLookupTable("db_2.table_1")).containsExactlyInAnyOrder("db_2.table_3");
+ assertThat(reverseLookupTable("db_2.table_2")).containsExactlyInAnyOrder("db_2.table_1");
+ assertThat(reverseLookupTable("db_2.table_3")).containsExactlyInAnyOrder("db_2.table_2");
+
+ assertThat(reverseLookupTable("db_3.table_merged"))
+ .containsExactlyInAnyOrder("db_3.table_1", "db_3.table_2", "db_3.table_3");
+
+ assertThat(reverseLookupTable("db_4.table_a")).containsExactlyInAnyOrder("db_4.table_1");
+ assertThat(reverseLookupTable("db_4.table_b"))
+ .containsExactlyInAnyOrder("db_4.table_1", "db_4.table_2");
+ assertThat(reverseLookupTable("db_4.table_c"))
+ .containsExactlyInAnyOrder("db_4.table_1", "db_4.table_2", "db_4.table_3");
+
+ assertThat(reverseLookupTable("db_5.prefix_table_1_suffix"))
+ .containsExactlyInAnyOrder("db_5.table_1");
+ assertThat(reverseLookupTable("db_5.prefix_table_2_suffix"))
+ .containsExactlyInAnyOrder("db_5.table_2");
+ assertThat(reverseLookupTable("db_5.prefix_table_3_suffix"))
+ .containsExactlyInAnyOrder("db_5.table_3");
+ }
+
+ @Test
+ void testReverseLookupDependingUpstreamSchemas() {
+ assertThat(reverseLookupSchema("db_0.table_1"))
+ .containsExactlyInAnyOrder("db_0.table_1 @ 0", "db_0.table_1 @ 1");
+ assertThat(reverseLookupSchema("db_0.table_2"))
+ .containsExactlyInAnyOrder("db_0.table_2 @ 0", "db_0.table_2 @ 1");
+ assertThat(reverseLookupSchema("db_0.table_3"))
+ .containsExactlyInAnyOrder("db_0.table_3 @ 0", "db_0.table_3 @ 1");
+
+ assertThat(reverseLookupSchema("db_1.table_1"))
+ .containsExactlyInAnyOrder("db_1.table_1 @ 0", "db_1.table_1 @ 1");
+ assertThat(reverseLookupSchema("db_1.table_2"))
+ .containsExactlyInAnyOrder("db_1.table_2 @ 0", "db_1.table_2 @ 1");
+ assertThat(reverseLookupSchema("db_1.table_3"))
+ .containsExactlyInAnyOrder("db_1.table_3 @ 0", "db_1.table_3 @ 1");
+
+ assertThat(reverseLookupSchema("db_2.table_1"))
+ .containsExactlyInAnyOrder("db_2.table_3 @ 0", "db_2.table_3 @ 1");
+ assertThat(reverseLookupSchema("db_2.table_2"))
+ .containsExactlyInAnyOrder("db_2.table_1 @ 0", "db_2.table_1 @ 1");
+ assertThat(reverseLookupSchema("db_2.table_3"))
+ .containsExactlyInAnyOrder("db_2.table_2 @ 0", "db_2.table_2 @ 1");
+
+ assertThat(reverseLookupSchema("db_3.table_merged"))
+ .containsExactlyInAnyOrder(
+ "db_3.table_1 @ 0",
+ "db_3.table_1 @ 1",
+ "db_3.table_2 @ 0",
+ "db_3.table_2 @ 1",
+ "db_3.table_3 @ 0",
+ "db_3.table_3 @ 1");
+
+ assertThat(reverseLookupSchema("db_4.table_a"))
+ .containsExactlyInAnyOrder("db_4.table_1 @ 0", "db_4.table_1 @ 1");
+ assertThat(reverseLookupSchema("db_4.table_b"))
+ .containsExactlyInAnyOrder(
+ "db_4.table_1 @ 0",
+ "db_4.table_1 @ 1",
+ "db_4.table_2 @ 0",
+ "db_4.table_2 @ 1");
+ assertThat(reverseLookupSchema("db_4.table_c"))
+ .containsExactlyInAnyOrder(
+ "db_4.table_1 @ 0",
+ "db_4.table_1 @ 1",
+ "db_4.table_2 @ 0",
+ "db_4.table_2 @ 1",
+ "db_4.table_3 @ 0",
+ "db_4.table_3 @ 1");
+
+ assertThat(reverseLookupSchema("db_5.prefix_table_1_suffix"))
+ .containsExactlyInAnyOrder("db_5.table_1 @ 0", "db_5.table_1 @ 1");
+ assertThat(reverseLookupSchema("db_5.prefix_table_2_suffix"))
+ .containsExactlyInAnyOrder("db_5.table_2 @ 0", "db_5.table_2 @ 1");
+ assertThat(reverseLookupSchema("db_5.prefix_table_3_suffix"))
+ .containsExactlyInAnyOrder("db_5.table_3 @ 0", "db_5.table_3 @ 1");
+ }
+
+ @Test
+ void testNormalizeSchemaChangeEventsInEvolveMode() {
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new CreateTableEvent(
+ NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA)))
+ .containsExactly(
+ new CreateTableEvent(NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id")))))
+ .containsExactly(
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id"))));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()))))
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()),
+ Collections.singletonMap("age", DataTypes.FLOAT())));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new RenameColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", "aging"))))
+ .containsExactly(
+ new RenameColumnEvent(
+ NORMALIZE_TEST_TABLE_ID, Collections.singletonMap("age", "aging")));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new DropColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList("notes"))))
+ .containsExactly(
+ new DropColumnEvent(
+ NORMALIZE_TEST_TABLE_ID, Collections.singletonList("notes")));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .containsExactly(new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.EVOLVE,
+ new DropTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .containsExactly(new DropTableEvent(NORMALIZE_TEST_TABLE_ID));
+ }
+
+ @Test
+ void testNormalizeSchemaChangeEventsInTryEvolveMode() {
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new CreateTableEvent(
+ NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA)))
+ .containsExactly(
+ new CreateTableEvent(NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id")))))
+ .containsExactly(
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id"))));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()))))
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()),
+ Collections.singletonMap("age", DataTypes.FLOAT())));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new RenameColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", "aging"))))
+ .containsExactly(
+ new RenameColumnEvent(
+ NORMALIZE_TEST_TABLE_ID, Collections.singletonMap("age", "aging")));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new DropColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList("notes"))))
+ .containsExactly(
+ new DropColumnEvent(
+ NORMALIZE_TEST_TABLE_ID, Collections.singletonList("notes")));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .containsExactly(new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.TRY_EVOLVE,
+ new DropTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .containsExactly(new DropTableEvent(NORMALIZE_TEST_TABLE_ID));
+ }
+
+ @Test
+ void testNormalizeSchemaChangeEventsInLenientMode() {
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new CreateTableEvent(
+ NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA)))
+ .containsExactly(
+ new CreateTableEvent(NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id")))))
+ .containsExactly(
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.LAST,
+ null))));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()))))
+ .containsExactly(
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()),
+ Collections.singletonMap("age", DataTypes.FLOAT())));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new RenameColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", "aging"))))
+ .containsExactly(
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn("aging", DataTypes.FLOAT()),
+ AddColumnEvent.ColumnPosition.LAST,
+ null))));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new DropColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList("notes"))))
+ .isEmpty();
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .containsExactly(new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.LENIENT,
+ new DropTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .containsExactly(new DropTableEvent(NORMALIZE_TEST_TABLE_ID));
+ }
+
+ @Test
+ void testNormalizeSchemaChangeEventsInIgnoreMode() {
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new CreateTableEvent(
+ NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA)))
+ .containsExactly(
+ new CreateTableEvent(NORMALIZE_TEST_TABLE_ID, NORMALIZE_TEST_SCHEMA));
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new AddColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList(
+ new AddColumnEvent.ColumnWithPosition(
+ Column.physicalColumn(
+ "added_flag", DataTypes.BOOLEAN()),
+ AddColumnEvent.ColumnPosition.AFTER,
+ "id")))))
+ .isEmpty();
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new AlterColumnTypeEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", DataTypes.DOUBLE()))))
+ .isEmpty();
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new RenameColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonMap("age", "aging"))))
+ .isEmpty();
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new DropColumnEvent(
+ NORMALIZE_TEST_TABLE_ID,
+ Collections.singletonList("notes"))))
+ .isEmpty();
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new TruncateTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .isEmpty();
+
+ assertThat(
+ normalizeEvent(
+ SchemaChangeBehavior.IGNORE,
+ new DropTableEvent(NORMALIZE_TEST_TABLE_ID)))
+ .isEmpty();
+ }
+}
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaManagerTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaManagerTest.java
similarity index 97%
rename from flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaManagerTest.java
rename to flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaManagerTest.java
index 5adf50dcf..743f38d33 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaManagerTest.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaManagerTest.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
+package org.apache.flink.cdc.runtime.operators.schema.common;
import org.apache.flink.cdc.common.event.AddColumnEvent;
import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
@@ -34,6 +34,7 @@ import java.util.Arrays;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Unit test for {@link SchemaManager}. */
@@ -65,15 +66,11 @@ class SchemaManagerTest {
.isPresent()
.contains(CUSTOMERS_SCHEMA);
- // Cannot apply CreateTableEvent multiple times
- assertThatThrownBy(
+ assertThatCode(
() ->
schemaManager.applyEvolvedSchemaChange(
new CreateTableEvent(CUSTOMERS, CUSTOMERS_SCHEMA)))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage(
- "Unable to apply CreateTableEvent to an existing schema for table \"%s\"",
- CUSTOMERS);
+ .doesNotThrowAnyException();
}
@Test
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaTestBase.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaTestBase.java
new file mode 100644
index 000000000..8086f650e
--- /dev/null
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/SchemaTestBase.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.cdc.common.data.binary.BinaryRecordData;
+import org.apache.flink.cdc.common.data.binary.BinaryStringData;
+import org.apache.flink.cdc.common.event.DataChangeEvent;
+import org.apache.flink.cdc.common.event.Event;
+import org.apache.flink.cdc.common.event.TableId;
+import org.apache.flink.cdc.common.route.RouteRule;
+import org.apache.flink.cdc.common.types.DataType;
+import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.runtime.partitioning.PartitioningEvent;
+import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** Some common schema testing utilities & functions. */
+public abstract class SchemaTestBase {
+ protected static final List<RouteRule> ROUTING_RULES =
+ Arrays.asList(
+ // Simple 1-to-1 routing rules
+ new RouteRule("db_1.table_1", "db_1.table_1"),
+ new RouteRule("db_1.table_2", "db_1.table_2"),
+ new RouteRule("db_1.table_3", "db_1.table_3"),
+
+ // Re-routed rules
+ new RouteRule("db_2.table_1", "db_2.table_2"),
+ new RouteRule("db_2.table_2", "db_2.table_3"),
+ new RouteRule("db_2.table_3", "db_2.table_1"),
+
+ // Merging tables
+ new RouteRule("db_3.table_\\.*", "db_3.table_merged"),
+
+ // Broadcast tables
+ new RouteRule("db_4.table_1", "db_4.table_a"),
+ new RouteRule("db_4.table_1", "db_4.table_b"),
+ new RouteRule("db_4.table_1", "db_4.table_c"),
+ new RouteRule("db_4.table_2", "db_4.table_b"),
+ new RouteRule("db_4.table_2", "db_4.table_c"),
+ new RouteRule("db_4.table_3", "db_4.table_c"),
+
+ // RepSym routes
+ new RouteRule("db_5.table_\\.*", "db_5.prefix_<>_suffix", "<>"),
+
+ // Irrelevant routes
+ new RouteRule("foo", "bar", null));
+
+ protected static final TableIdRouter TABLE_ID_ROUTER = new TableIdRouter(ROUTING_RULES);
+
+ protected static BinaryRecordData genBinRec(String rowType, Object... fields) {
+ return (new BinaryRecordDataGenerator(quickGenRow(rowType).toArray(new DataType[0])))
+ .generate(
+ Arrays.stream(fields)
+ .map(
+ e ->
+ (e instanceof String)
+ ? BinaryStringData.fromString((String) e)
+ : e)
+ .toArray());
+ }
+
+ protected static List<DataType> quickGenRow(String crypticExpr) {
+ List<DataType> rowTypes = new ArrayList<>(crypticExpr.length());
+ for (char c : crypticExpr.toLowerCase().toCharArray()) {
+ switch (c) {
+ case 'b':
+ rowTypes.add(DataTypes.BOOLEAN());
+ break;
+ case 'i':
+ rowTypes.add(DataTypes.INT());
+ break;
+ case 'l':
+ rowTypes.add(DataTypes.BIGINT());
+ break;
+ case 'f':
+ rowTypes.add(DataTypes.FLOAT());
+ break;
+ case 'd':
+ rowTypes.add(DataTypes.DOUBLE());
+ break;
+ case 's':
+ rowTypes.add(DataTypes.STRING());
+ break;
+ default:
+ throw new IllegalStateException("Unexpected type char: " + c);
+ }
+ }
+ return rowTypes;
+ }
+
+ protected static DataChangeEvent genInsert(TableId tableId, String rowType, Object... fields) {
+ return DataChangeEvent.insertEvent(tableId, genBinRec(rowType, fields));
+ }
+
+ protected static StreamRecord<PartitioningEvent> wrap(Event payload) {
+ return wrap(payload, 0, 0);
+ }
+
+ protected static StreamRecord<PartitioningEvent> wrap(Event payload, int from, int to) {
+ return new StreamRecord<>(PartitioningEvent.ofDistributed(payload, from, to));
+ }
+}
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/TableIdRouterTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/TableIdRouterTest.java
new file mode 100644
index 000000000..a6d6c89a2
--- /dev/null
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/common/TableIdRouterTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators.schema.common;
+
+import org.apache.flink.cdc.common.event.TableId;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat;
+
+/** Unit test for {@link TableIdRouter}. */
+public class TableIdRouterTest extends SchemaTestBase {
+
+ private static List<String> testRoute(String tableId) {
+ return TABLE_ID_ROUTER.route(TableId.parse(tableId)).stream()
+ .map(TableId::toString)
+ .collect(Collectors.toList());
+ }
+
+ @Test
+ void testImplicitRoute() {
+ assertThat(testRoute("db_0.table_1")).containsExactlyInAnyOrder("db_0.table_1");
+ assertThat(testRoute("db_0.table_2")).containsExactlyInAnyOrder("db_0.table_2");
+ assertThat(testRoute("db_0.table_3")).containsExactlyInAnyOrder("db_0.table_3");
+ }
+
+ @Test
+ void testOneToOneRoute() {
+ assertThat(testRoute("db_1.table_1")).containsExactlyInAnyOrder("db_1.table_1");
+ assertThat(testRoute("db_1.table_2")).containsExactlyInAnyOrder("db_1.table_2");
+ assertThat(testRoute("db_1.table_3")).containsExactlyInAnyOrder("db_1.table_3");
+ }
+
+ @Test
+ void testTwistedOneToOneRoute() {
+ assertThat(testRoute("db_2.table_1")).containsExactlyInAnyOrder("db_2.table_2");
+ assertThat(testRoute("db_2.table_2")).containsExactlyInAnyOrder("db_2.table_3");
+ assertThat(testRoute("db_2.table_3")).containsExactlyInAnyOrder("db_2.table_1");
+ }
+
+ @Test
+ void testMergingTablesRoute() {
+ assertThat(testRoute("db_3.table_1")).containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(testRoute("db_3.table_2")).containsExactlyInAnyOrder("db_3.table_merged");
+ assertThat(testRoute("db_3.table_3")).containsExactlyInAnyOrder("db_3.table_merged");
+ }
+
+ @Test
+ void testBroadcastingRoute() {
+ assertThat(testRoute("db_4.table_1"))
+ .containsExactlyInAnyOrder("db_4.table_a", "db_4.table_b", "db_4.table_c");
+ assertThat(testRoute("db_4.table_2"))
+ .containsExactlyInAnyOrder("db_4.table_b", "db_4.table_c");
+ assertThat(testRoute("db_4.table_3")).containsExactlyInAnyOrder("db_4.table_c");
+ }
+
+ @Test
+ void testRepSymRoute() {
+ assertThat(testRoute("db_5.table_1"))
+ .containsExactlyInAnyOrder("db_5.prefix_table_1_suffix");
+ assertThat(testRoute("db_5.table_2"))
+ .containsExactlyInAnyOrder("db_5.prefix_table_2_suffix");
+ assertThat(testRoute("db_5.table_3"))
+ .containsExactlyInAnyOrder("db_5.prefix_table_3_suffix");
+ }
+}
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaDerivationTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaDerivationTest.java
deleted file mode 100644
index 6b7cc5d61..000000000
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/coordinator/SchemaDerivationTest.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.cdc.runtime.operators.schema.coordinator;
-
-import org.apache.flink.cdc.common.event.AddColumnEvent;
-import org.apache.flink.cdc.common.event.AlterColumnTypeEvent;
-import org.apache.flink.cdc.common.event.CreateTableEvent;
-import org.apache.flink.cdc.common.event.DropColumnEvent;
-import org.apache.flink.cdc.common.event.RenameColumnEvent;
-import org.apache.flink.cdc.common.event.SchemaChangeEvent;
-import org.apache.flink.cdc.common.event.TableId;
-import org.apache.flink.cdc.common.route.RouteRule;
-import org.apache.flink.cdc.common.schema.Column;
-import org.apache.flink.cdc.common.schema.PhysicalColumn;
-import org.apache.flink.cdc.common.schema.Schema;
-import org.apache.flink.cdc.common.types.DataType;
-import org.apache.flink.cdc.common.types.DataTypes;
-
-import org.apache.flink.shaded.guava31.com.google.common.collect.ImmutableMap;
-
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import static org.apache.flink.cdc.common.testutils.assertions.EventAssertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Unit test for {@link SchemaDerivation}. */
-class SchemaDerivationTest {
-
- private static final TableId TABLE_1 = TableId.tableId("mydb", "myschema", "mytable1");
- private static final TableId TABLE_2 = TableId.tableId("mydb", "myschema", "mytable2");
- private static final TableId MERGED_TABLE = TableId.tableId("mydb", "myschema", "mytables");
-
- private static final Schema SCHEMA =
- Schema.newBuilder()
- .column(Column.physicalColumn("id", DataTypes.BIGINT()))
- .column(Column.physicalColumn("name", DataTypes.STRING()))
- .column(Column.physicalColumn("age", DataTypes.INT()))
- .build();
-
- private static final Schema COMPATIBLE_SCHEMA =
- Schema.newBuilder()
- .column(Column.physicalColumn("id", DataTypes.BIGINT()))
- .column(Column.physicalColumn("name", DataTypes.STRING()))
- .column(Column.physicalColumn("age", DataTypes.BIGINT()))
- .column(Column.physicalColumn("gender", DataTypes.STRING()))
- .build();
-
- private static final Schema INCOMPATIBLE_SCHEMA =
- Schema.newBuilder()
- .column(Column.physicalColumn("id", DataTypes.BIGINT()))
- .column(Column.physicalColumn("name", DataTypes.STRING()))
- .column(Column.physicalColumn("age", DataTypes.STRING()))
- .column(Column.physicalColumn("gender", DataTypes.STRING()))
- .build();
-
- private static final List<RouteRule> ROUTES =
- Collections.singletonList(
- new RouteRule("mydb.myschema.mytable[0-9]", MERGED_TABLE.toString(), null));
-
- @Test
- void testOneToOneMapping() {
- SchemaDerivation schemaDerivation =
- new SchemaDerivation(new SchemaManager(), ROUTES, new HashMap<>());
-
- // Create table
- List<SchemaChangeEvent> derivedChangesAfterCreateTable =
- schemaDerivation.applySchemaChange(new CreateTableEvent(TABLE_1, SCHEMA));
- assertThat(derivedChangesAfterCreateTable).hasSize(1);
- assertThat(derivedChangesAfterCreateTable.get(0))
- .asCreateTableEvent()
- .hasTableId(MERGED_TABLE)
- .hasSchema(SCHEMA);
-
- // Add column
- AddColumnEvent.ColumnWithPosition newCol1 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col1", DataTypes.STRING(), null));
- AddColumnEvent.ColumnWithPosition newCol2 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col2", DataTypes.STRING(), null));
- AddColumnEvent.ColumnWithPosition newCol3 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col3", DataTypes.STRING(), null, "abc"));
- List<AddColumnEvent.ColumnWithPosition> newColumns =
- Arrays.asList(newCol1, newCol2, newCol3);
- List<SchemaChangeEvent> derivedChangesAfterAddColumn =
- schemaDerivation.applySchemaChange(new AddColumnEvent(TABLE_1, newColumns));
- assertThat(derivedChangesAfterAddColumn).hasSize(1);
- assertThat(derivedChangesAfterAddColumn.get(0))
- .asAddColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsAddedColumns(newCol1, newCol2, newCol3);
-
- // Alter column type
- ImmutableMap<String, DataType> typeMapping = ImmutableMap.of("age", DataTypes.BIGINT());
- List<SchemaChangeEvent> derivedChangesAfterAlterTableType =
- schemaDerivation.applySchemaChange(new AlterColumnTypeEvent(TABLE_1, typeMapping));
- assertThat(derivedChangesAfterAlterTableType).hasSize(1);
- assertThat(derivedChangesAfterAlterTableType.get(0))
- .asAlterColumnTypeEvent()
- .hasTableId(MERGED_TABLE)
- .containsTypeMapping(typeMapping);
-
- // Drop column
- List<String> droppedColumns = Arrays.asList("new_col1", "new_col2");
- List<SchemaChangeEvent> derivedChangesAfterDropColumn =
- schemaDerivation.applySchemaChange(new DropColumnEvent(TABLE_1, droppedColumns));
- assertThat(derivedChangesAfterDropColumn).hasSize(1);
- assertThat(derivedChangesAfterDropColumn.get(0))
- .asDropColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsDroppedColumns("new_col1", "new_col2");
-
- // Rename column
- Map<String, String> renamedColumns = ImmutableMap.of("name", "last_name");
- List<SchemaChangeEvent> derivedChangesAfterRenameColumn =
- schemaDerivation.applySchemaChange(new RenameColumnEvent(TABLE_1, renamedColumns));
- assertThat(derivedChangesAfterRenameColumn).hasSize(1);
- assertThat(derivedChangesAfterRenameColumn.get(0))
- .asRenameColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsNameMapping(renamedColumns);
- }
-
- @Test
- void testMergingTablesWithExactSameSchema() {
- SchemaManager schemaManager = new SchemaManager();
- SchemaDerivation schemaDerivation =
- new SchemaDerivation(schemaManager, ROUTES, new HashMap<>());
-
- // Create table 1
- List<SchemaChangeEvent> derivedChangesAfterCreateTable =
- schemaDerivation.applySchemaChange(new CreateTableEvent(TABLE_1, SCHEMA));
- assertThat(derivedChangesAfterCreateTable).hasSize(1);
- assertThat(derivedChangesAfterCreateTable.get(0))
- .asCreateTableEvent()
- .hasTableId(MERGED_TABLE)
- .hasSchema(SCHEMA);
- derivedChangesAfterCreateTable.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Create table 2
- assertThat(schemaDerivation.applySchemaChange(new CreateTableEvent(TABLE_2, SCHEMA)))
- .isEmpty();
-
- // Add column for table 1
- AddColumnEvent.ColumnWithPosition newCol1 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col1", DataTypes.STRING(), null));
- AddColumnEvent.ColumnWithPosition newCol2 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col2", DataTypes.STRING(), null));
- List<AddColumnEvent.ColumnWithPosition> newColumns = Arrays.asList(newCol1, newCol2);
- List<SchemaChangeEvent> derivedChangesAfterAddColumn =
- schemaDerivation.applySchemaChange(new AddColumnEvent(TABLE_1, newColumns));
- assertThat(derivedChangesAfterAddColumn).hasSize(1);
- assertThat(derivedChangesAfterAddColumn.get(0))
- .asAddColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsAddedColumns(newCol1, newCol2);
- derivedChangesAfterAddColumn.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Add column for table 2
- assertThat(schemaDerivation.applySchemaChange(new AddColumnEvent(TABLE_2, newColumns)))
- .isEmpty();
-
- // Alter column type for table 1
- ImmutableMap<String, DataType> typeMapping = ImmutableMap.of("age", DataTypes.BIGINT());
- List<SchemaChangeEvent> derivedChangesAfterAlterColumnType =
- schemaDerivation.applySchemaChange(new AlterColumnTypeEvent(TABLE_1, typeMapping));
- assertThat(derivedChangesAfterAlterColumnType).hasSize(1);
- assertThat(derivedChangesAfterAlterColumnType.get(0))
- .asAlterColumnTypeEvent()
- .hasTableId(MERGED_TABLE)
- .containsTypeMapping(typeMapping);
- derivedChangesAfterAlterColumnType.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Alter column type for table 2
- assertThat(
- schemaDerivation.applySchemaChange(
- new AlterColumnTypeEvent(TABLE_2, typeMapping)))
- .isEmpty();
-
- // Drop column for table 1
- List<String> droppedColumns = Arrays.asList("new_col1", "new_col2");
- assertThat(schemaDerivation.applySchemaChange(new DropColumnEvent(TABLE_1, droppedColumns)))
- .isEmpty();
- // Drop column for table 2
- assertThat(schemaDerivation.applySchemaChange(new DropColumnEvent(TABLE_2, droppedColumns)))
- .isEmpty();
-
- // Rename column for table 1
- Map<String, String> renamedColumns = ImmutableMap.of("name", "last_name");
- List<SchemaChangeEvent> derivedChangesAfterRenameColumn =
- schemaDerivation.applySchemaChange(new RenameColumnEvent(TABLE_1, renamedColumns));
- assertThat(derivedChangesAfterRenameColumn).hasSize(1);
- assertThat(derivedChangesAfterRenameColumn.get(0))
- .asAddColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsAddedColumns(
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("last_name", DataTypes.STRING(), null)));
- derivedChangesAfterRenameColumn.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Rename column for table 2
- assertThat(
- schemaDerivation.applySchemaChange(
- new RenameColumnEvent(TABLE_2, renamedColumns)))
- .isEmpty();
- }
-
- @Test
- void testMergingTableWithDifferentSchemas() {
- SchemaManager schemaManager = new SchemaManager();
- SchemaDerivation schemaDerivation =
- new SchemaDerivation(schemaManager, ROUTES, new HashMap<>());
- // Create table 1
- List<SchemaChangeEvent> derivedChangesAfterCreateTable =
- schemaDerivation.applySchemaChange(new CreateTableEvent(TABLE_1, SCHEMA));
- assertThat(derivedChangesAfterCreateTable).hasSize(1);
- assertThat(derivedChangesAfterCreateTable.get(0))
- .asCreateTableEvent()
- .hasTableId(MERGED_TABLE)
- .hasSchema(SCHEMA);
- derivedChangesAfterCreateTable.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Create table 2
- List<SchemaChangeEvent> derivedChangesAfterCreateTable2 =
- schemaDerivation.applySchemaChange(
- new CreateTableEvent(TABLE_2, COMPATIBLE_SCHEMA));
- assertThat(derivedChangesAfterCreateTable2).hasSize(2);
- assertThat(derivedChangesAfterCreateTable2)
- .containsExactlyInAnyOrder(
- new AddColumnEvent(
- MERGED_TABLE,
- Collections.singletonList(
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn(
- "gender", DataTypes.STRING(), null)))),
- new AlterColumnTypeEvent(
- MERGED_TABLE, ImmutableMap.of("age", DataTypes.BIGINT())));
- derivedChangesAfterCreateTable2.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Add column for table 1
- AddColumnEvent.ColumnWithPosition newCol1 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col1", DataTypes.VARCHAR(255), null));
- AddColumnEvent.ColumnWithPosition newCol2 =
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("new_col2", DataTypes.VARCHAR(255), null));
- List<AddColumnEvent.ColumnWithPosition> newColumns = Arrays.asList(newCol1, newCol2);
- List<SchemaChangeEvent> derivedChangesAfterAddColumn =
- schemaDerivation.applySchemaChange(new AddColumnEvent(TABLE_1, newColumns));
- assertThat(derivedChangesAfterAddColumn).hasSize(1);
- assertThat(derivedChangesAfterAddColumn.get(0))
- .asAddColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsAddedColumns(newCol1, newCol2);
- derivedChangesAfterAddColumn.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Add column for table 2
- List<SchemaChangeEvent> derivedChangesAfterAddColumnForTable2 =
- schemaDerivation.applySchemaChange(
- new AddColumnEvent(
- TABLE_2,
- Arrays.asList(
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn(
- "new_col1", DataTypes.STRING(), null)),
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn(
- "new_col2", DataTypes.STRING(), null)))));
- assertThat(derivedChangesAfterAddColumnForTable2).hasSize(1);
- assertThat(derivedChangesAfterAddColumnForTable2.get(0))
- .asAlterColumnTypeEvent()
- .containsTypeMapping(
- ImmutableMap.of(
- "new_col1", DataTypes.STRING(), "new_col2", DataTypes.STRING()));
- derivedChangesAfterAddColumnForTable2.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Alter column type for table 1
- ImmutableMap<String, DataType> typeMapping = ImmutableMap.of("age", DataTypes.BIGINT());
- List<SchemaChangeEvent> derivedChangesAfterAlterColumnType =
- schemaDerivation.applySchemaChange(new AlterColumnTypeEvent(TABLE_1, typeMapping));
- assertThat(derivedChangesAfterAlterColumnType).isEmpty();
- // Alter column type for table 2
- List<SchemaChangeEvent> derivedChangesAfterAlterColumnTypeForTable2 =
- schemaDerivation.applySchemaChange(
- new AlterColumnTypeEvent(
- TABLE_2, ImmutableMap.of("age", DataTypes.TINYINT())));
- assertThat(derivedChangesAfterAlterColumnTypeForTable2).isEmpty();
-
- // Drop column for table 1
- List<String> droppedColumns = Arrays.asList("new_col1", "new_col2");
- assertThat(schemaDerivation.applySchemaChange(new DropColumnEvent(TABLE_1, droppedColumns)))
- .isEmpty();
- // Drop column for table 2
- assertThat(schemaDerivation.applySchemaChange(new DropColumnEvent(TABLE_2, droppedColumns)))
- .isEmpty();
-
- // Rename column for table 1
- Map<String, String> renamedColumns = ImmutableMap.of("name", "last_name");
- List<SchemaChangeEvent> derivedChangesAfterRenameColumn =
- schemaDerivation.applySchemaChange(new RenameColumnEvent(TABLE_1, renamedColumns));
- assertThat(derivedChangesAfterRenameColumn).hasSize(1);
- assertThat(derivedChangesAfterRenameColumn.get(0))
- .asAddColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsAddedColumns(
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("last_name", DataTypes.STRING(), null)));
- derivedChangesAfterRenameColumn.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Rename column for table 2
- List<SchemaChangeEvent> derivedChangesAfterRenameColumnForTable2 =
- schemaDerivation.applySchemaChange(
- new RenameColumnEvent(TABLE_2, ImmutableMap.of("name", "first_name")));
- assertThat(derivedChangesAfterRenameColumnForTable2).hasSize(1);
- assertThat(derivedChangesAfterRenameColumnForTable2.get(0))
- .asAddColumnEvent()
- .hasTableId(MERGED_TABLE)
- .containsAddedColumns(
- new AddColumnEvent.ColumnWithPosition(
- new PhysicalColumn("first_name", DataTypes.STRING(), null)));
- derivedChangesAfterRenameColumnForTable2.forEach(schemaManager::applyEvolvedSchemaChange);
-
- assertThat(schemaManager.getLatestEvolvedSchema(MERGED_TABLE))
- .contains(
- Schema.newBuilder()
- .column(Column.physicalColumn("id", DataTypes.BIGINT()))
- .column(Column.physicalColumn("name", DataTypes.STRING()))
- .column(Column.physicalColumn("age", DataTypes.BIGINT()))
- .column(Column.physicalColumn("gender", DataTypes.STRING()))
- .column(Column.physicalColumn("new_col1", DataTypes.STRING()))
- .column(Column.physicalColumn("new_col2", DataTypes.STRING()))
- .column(Column.physicalColumn("last_name", DataTypes.STRING()))
- .column(Column.physicalColumn("first_name", DataTypes.STRING()))
- .build());
- }
-
- @Test
- void testIncompatibleTypes() {
- SchemaManager schemaManager = new SchemaManager();
- SchemaDerivation schemaDerivation =
- new SchemaDerivation(schemaManager, ROUTES, new HashMap<>());
- // Create table 1
- List<SchemaChangeEvent> derivedChangesAfterCreateTable =
- schemaDerivation.applySchemaChange(new CreateTableEvent(TABLE_1, SCHEMA));
- assertThat(derivedChangesAfterCreateTable).hasSize(1);
- assertThat(derivedChangesAfterCreateTable.get(0))
- .asCreateTableEvent()
- .hasTableId(MERGED_TABLE)
- .hasSchema(SCHEMA);
- derivedChangesAfterCreateTable.forEach(schemaManager::applyEvolvedSchemaChange);
-
- // Create table 2
- assertThatThrownBy(
- () ->
- schemaDerivation.applySchemaChange(
... 1695 lines suppressed ...