You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ku...@apache.org on 2026/03/13 07:01:47 UTC
(flink-cdc) branch master updated: [FLINK-38729] Add support for Flink 2.2.0 (#4294)
This is an automated email from the ASF dual-hosted git repository.
kunni pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/master by this push:
new 323b2bc67 [FLINK-38729] Add support for Flink 2.2.0 (#4294)
323b2bc67 is described below
commit 323b2bc67f9b86ba6a4dfc7e5b3dfc0766a7b633
Author: Kunni <de...@gmail.com>
AuthorDate: Fri Mar 13 15:01:42 2026 +0800
[FLINK-38729] Add support for Flink 2.2.0 (#4294)
Co-authored-by: xiaoxiong.duan@zznode.com <xi...@zznode.com>
---
.github/workflows/flink_cdc_base.yml | 2 +-
.github/workflows/flink_cdc_ci.yml | 22 ++
.github/workflows/flink_cdc_ci_nightly.yml | 24 ++
.github/workflows/modules.py | 11 +-
flink-cdc-cli/pom.xml | 29 +++
.../java/org/apache/flink/cdc/cli/CliFrontend.java | 9 +-
.../flink/cdc/cli/utils/ConfigurationUtils.java | 14 +-
.../org/apache/flink/cdc/cli/CliFrontendTest.java | 3 +-
flink-cdc-common/pom.xml | 29 +++
flink-cdc-composer/pom.xml | 43 ++++
.../flink/translator/DataSinkTranslatorTest.java | 6 +
.../source/DistributedSourceFunction.java | 4 +-
.../flink-cdc-pipeline-connector-values/pom.xml | 35 +++
.../cdc/connectors/values/sink/ValuesDataSink.java | 15 +-
.../values/source/ValuesDataSourceITCase.java | 4 +-
flink-cdc-dist/pom.xml | 36 +++
.../flink/cdc/pipeline/tests/FlussE2eITCase.java | 2 +
.../cdc/pipeline/tests/MaxComputeE2eITCase.java | 2 +
.../cdc/pipeline/tests/MySqlToDorisE2eITCase.java | 2 +
.../cdc/pipeline/tests/MySqlToHudiE2eITCase.java | 2 +
.../pipeline/tests/MySqlToIcebergE2eITCase.java | 2 +
.../cdc/pipeline/tests/MySqlToPaimonE2eITCase.java | 2 +
.../flink/cdc/pipeline/tests/MysqlE2eITCase.java | 2 +
.../tests/MysqlE2eWithYarnApplicationITCase.java | 2 +
.../cdc/pipeline/tests/MysqlToKafkaE2eITCase.java | 2 +
.../cdc/pipeline/tests/OceanBaseE2eITCase.java | 2 +
.../flink/cdc/pipeline/tests/OracleE2eITCase.java | 2 +
.../cdc/pipeline/tests/PostgresE2eITCase.java | 2 +
.../flink/cdc/pipeline/tests/RouteE2eITCase.java | 2 +
.../cdc/pipeline/tests/SchemaEvolveE2eITCase.java | 2 +
.../tests/SchemaEvolvingTransformE2eITCase.java | 2 +
.../cdc/pipeline/tests/TransformE2eITCase.java | 2 +
.../flink/cdc/pipeline/tests/UdfE2eITCase.java | 2 +
.../flink/cdc/pipeline/tests/ValuesE2eITCase.java | 260 +++++++++++++++++++++
.../migration/MySqlToPaimonMigrationITCase.java | 2 +
.../tests/migration/YamlJobMigrationITCase.java | 2 +
.../tests/specs/FlinkPipelineSpecsITCase.java | 2 +
.../cdc/pipeline/tests/utils/TarballFetcher.java | 30 ++-
.../pom.xml | 24 +-
.../typeutils/TypeSerializerSnapshotAdapter.java | 27 +++
.../operators/AbstractStreamOperatorAdapter.java | 28 +++
.../apache/flink/runtime/event/WatermarkEvent.java | 27 +++
.../pom.xml | 24 +-
.../typeutils/TypeSerializerSnapshotAdapter.java | 37 +++
.../org/apache/flink/api/connector/sink2/Sink.java | 74 ++++++
.../connector/sink2/TwoPhaseCommittingSink.java | 28 +++
.../operators/AbstractStreamOperatorAdapter.java | 43 ++++
.../apache/flink/core/execution/RestoreMode.java | 56 +++++
.../runtime/jobgraph/SavepointConfigOptions.java | 59 +++++
.../connector/sink2/WithPostCommitTopology.java | 27 +++
.../api/connector/sink2/WithPreCommitTopology.java | 28 +++
.../api/connector/sink2/WithPreWriteTopology.java | 29 +++
.../streaming/api/functions/sink/SinkFunction.java | 28 +++
.../source/RichParallelSourceFunction.java | 43 ++++
.../api/functions/source/SourceFunction.java | 28 +++
.../api/operators/YieldingOperatorFactory.java | 28 +++
.../flink/table/api/ValidationException.java | 38 +++
flink-cdc-runtime/pom.xml | 64 +++++
.../schema/distributed/SchemaOperator.java | 6 +-
.../schema/regular/BatchSchemaOperator.java | 4 +-
.../operators/schema/regular/SchemaOperator.java | 6 +-
.../sink/BatchDataSinkFunctionOperator.java | 2 -
.../sink/BatchDataSinkWriterOperator.java | 47 +++-
.../operators/sink/DataSinkFunctionOperator.java | 8 +-
.../operators/sink/DataSinkWriterOperator.java | 54 ++++-
.../operators/transform/PostTransformOperator.java | 4 +-
.../operators/transform/PreTransformOperator.java | 27 ++-
.../BatchRegularPrePartitionOperator.java | 5 +-
.../DistributedPrePartitionOperator.java | 7 +-
.../partitioning/RegularPrePartitionOperator.java | 4 +-
.../cdc/runtime/serializer/EnumSerializer.java | 3 +-
.../NestedSerializersSnapshotDelegate.java | 2 +-
.../serializer/NullableSerializerWrapper.java | 3 +-
.../serializer/data/ArrayDataSerializer.java | 7 +-
.../serializer/data/DecimalDataSerializer.java | 3 +-
.../data/LocalZonedTimestampDataSerializer.java | 3 +-
.../runtime/serializer/data/MapDataSerializer.java | 4 +-
.../serializer/data/TimestampDataSerializer.java | 3 +-
.../data/ZonedTimestampDataSerializer.java | 3 +-
.../flink/cdc/runtime/typeutils/EventTypeInfo.java | 10 +-
.../typeutils/PartitioningEventTypeInfo.java | 10 +-
.../schema/distributed/SchemaEvolveTest.java | 7 +-
.../operators/sink/DataSinkOperatorAdapter.java | 10 +-
.../cdc/runtime/serializer/EnumSerializerTest.java | 2 +-
.../cdc/runtime/serializer/SerializerTestBase.java | 2 +-
.../DistributedEventOperatorTestHarness.java | 8 +-
.../operators/RegularEventOperatorTestHarness.java | 19 +-
pom.xml | 52 ++++-
88 files changed, 1562 insertions(+), 119 deletions(-)
diff --git a/.github/workflows/flink_cdc_base.yml b/.github/workflows/flink_cdc_base.yml
index c5157abc9..982ea81e3 100644
--- a/.github/workflows/flink_cdc_base.yml
+++ b/.github/workflows/flink_cdc_base.yml
@@ -108,7 +108,7 @@ jobs:
build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }${{ inputs.custom-maven-parameter }}"
- mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} -Duser.timezone=$jvm_timezone verify
+ mvn --no-snapshot-updates -B -DskipTests ${{ inputs.custom-maven-parameter }} -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules -DspecifiedParallelism=${{ inputs.parallelism }} -Duser.timezone=$jvm_timezone verify
- name: Print JVM thread dumps when cancelled
if: ${{ failure() }}
diff --git a/.github/workflows/flink_cdc_ci.yml b/.github/workflows/flink_cdc_ci.yml
index 5398ca43e..f7514f7e0 100644
--- a/.github/workflows/flink_cdc_ci.yml
+++ b/.github/workflows/flink_cdc_ci.yml
@@ -63,6 +63,14 @@ jobs:
with:
java-versions: "[11]"
modules: "['core']"
+ common_2_x:
+ name: Common Unit Tests 2.x
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[11]"
+ flink-versions: "['2.2.0']"
+ custom-maven-parameter: "-Pflink2"
+ modules: "['core_2.x']"
pipeline-ut:
name: Pipeline Unit Tests
uses: ./.github/workflows/flink_cdc_base.yml
@@ -88,6 +96,20 @@ jobs:
flink-versions: "['1.19.3', '1.20.3']"
modules: "['pipeline_e2e']"
parallelism: ${{ matrix.parallelism }}
+ pipeline_e2e_2_x:
+ strategy:
+ max-parallel: 2
+ fail-fast: false
+ matrix:
+ parallelism: [ 1, 4 ]
+ name: Pipeline E2E Tests 2.x (${{ matrix.parallelism }}-Parallelism)
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[11]"
+ flink-versions: "['2.2.0']"
+ custom-maven-parameter: "-Pflink2"
+ modules: "['pipeline_e2e']"
+ parallelism: ${{ matrix.parallelism }}
source_e2e:
name: Source E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
diff --git a/.github/workflows/flink_cdc_ci_nightly.yml b/.github/workflows/flink_cdc_ci_nightly.yml
index d48016764..5b01fc62c 100644
--- a/.github/workflows/flink_cdc_ci_nightly.yml
+++ b/.github/workflows/flink_cdc_ci_nightly.yml
@@ -54,6 +54,15 @@ jobs:
with:
java-versions: "[17]"
modules: "['core']"
+ common_2_x:
+ if: github.repository == 'apache/flink-cdc'
+ name: Common Unit Tests 2.x
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[17]"
+ flink-versions: "['2.2.0']"
+ custom-maven-parameter: "-Pflink2"
+ modules: "['core_2.x']"
pipeline-ut:
if: github.repository == 'apache/flink-cdc'
name: Pipeline Unit Tests
@@ -82,6 +91,21 @@ jobs:
flink-versions: "['1.19.3', '1.20.3']"
modules: "['pipeline_e2e']"
parallelism: ${{ matrix.parallelism }}
+ pipeline_e2e_2_x:
+ if: github.repository == 'apache/flink-cdc'
+ strategy:
+ max-parallel: 2
+ fail-fast: false
+ matrix:
+ parallelism: [ 1, 4 ]
+ name: Pipeline E2E Tests 2.x (${{ matrix.parallelism }}-Parallelism)
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[17]"
+ custom-maven-parameter: "-Pflink2"
+ flink-versions: "['2.2.0']"
+ modules: "['pipeline_e2e_2.x']"
+ parallelism: ${{ matrix.parallelism }}
source_e2e:
if: github.repository == 'apache/flink-cdc'
name: Source E2E Tests
diff --git a/.github/workflows/modules.py b/.github/workflows/modules.py
index d2d051c76..86e94164f 100755
--- a/.github/workflows/modules.py
+++ b/.github/workflows/modules.py
@@ -17,6 +17,14 @@ MODULES_CORE = [
"flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
]
+MODULES_CORE_2_X = [
+ "flink-cdc-cli",
+ "flink-cdc-common",
+ "flink-cdc-composer",
+ "flink-cdc-runtime",
+ "flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
+]
+
MODULES_PIPELINE_CONNECTORS = [
"flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris",
"flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch",
@@ -142,6 +150,7 @@ MODULES_SOURCE_E2E = [
ALL_MODULES = set(
MODULES_CORE +
+ MODULES_CORE_2_X +
MODULES_PIPELINE_CONNECTORS +
MODULES_MYSQL_SOURCE +
MODULES_MYSQL_PIPELINE +
@@ -173,7 +182,7 @@ test_modules = set()
compile_modules = set()
for module in INPUT_MODULES.split(', '):
- module_list = set(globals()['MODULES_' + module.upper().replace('-', '_')])
+ module_list = set(globals()['MODULES_' + module.upper().replace('-', '_').replace('.', '_')])
test_modules |= module_list
if module == 'source_e2e' or module == 'pipeline_e2e':
compile_modules |= ALL_MODULES
diff --git a/flink-cdc-cli/pom.xml b/flink-cdc-cli/pom.xml
index 35a69cbd2..fe335fc28 100644
--- a/flink-cdc-cli/pom.xml
+++ b/flink-cdc-cli/pom.xml
@@ -82,4 +82,33 @@ limitations under the License.
</dependencies>
+ <profiles>
+ <profile>
+ <id>flink2</id>
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java b/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java
index 5f327eca2..0f170367a 100644
--- a/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java
+++ b/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/CliFrontend.java
@@ -27,7 +27,6 @@ import org.apache.flink.cdc.common.utils.StringUtils;
import org.apache.flink.cdc.composer.PipelineExecution;
import org.apache.flink.configuration.DeploymentOptions;
import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.jobgraph.SavepointConfigOptions;
import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings;
import org.apache.flink.shaded.guava31.com.google.common.base.Joiner;
@@ -173,7 +172,13 @@ public class CliFrontend {
commandLine.getOptionValue(SAVEPOINT_CLAIM_MODE),
ConfigurationUtils.getClaimModeClass());
} else {
- restoreMode = SavepointConfigOptions.RESTORE_MODE.defaultValue();
+ try {
+ restoreMode =
+ ConfigurationUtils.getClaimModeClass().getField("DEFAULT").get(null);
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(
+ "Failed to get DEFAULT value from RestoreMode class.", e);
+ }
}
// allowNonRestoredState is always false because all operators are predefined.
diff --git a/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java b/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java
index 7280c0679..eafd4da09 100644
--- a/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java
+++ b/flink-cdc-cli/src/main/java/org/apache/flink/cdc/cli/utils/ConfigurationUtils.java
@@ -67,12 +67,16 @@ public class ConfigurationUtils {
public static Class<?> getClaimModeClass() {
try {
- return Class.forName("org.apache.flink.core.execution.RestoreMode");
- } catch (ClassNotFoundException ignored) {
+ return Class.forName("org.apache.flink.core.execution.RecoveryClaimMode");
+ } catch (ClassNotFoundException classNotFoundException) {
try {
- return Class.forName("org.apache.flink.runtime.jobgraph.RestoreMode");
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(e);
+ return Class.forName("org.apache.flink.core.execution.RestoreMode");
+ } catch (ClassNotFoundException ignored) {
+ try {
+ return Class.forName("org.apache.flink.runtime.jobgraph.RestoreMode");
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
}
}
}
diff --git a/flink-cdc-cli/src/test/java/org/apache/flink/cdc/cli/CliFrontendTest.java b/flink-cdc-cli/src/test/java/org/apache/flink/cdc/cli/CliFrontendTest.java
index b9f42cefc..70aeebbfa 100644
--- a/flink-cdc-cli/src/test/java/org/apache/flink/cdc/cli/CliFrontendTest.java
+++ b/flink-cdc-cli/src/test/java/org/apache/flink/cdc/cli/CliFrontendTest.java
@@ -107,7 +107,8 @@ class CliFrontendTest {
"-n");
assertThat(executor.getFlinkConfig().get(SAVEPOINT_PATH))
.isEqualTo(flinkHome() + "/savepoints/savepoint-1");
- assertThat(executor.getFlinkConfig().get(RESTORE_MODE)).isEqualTo(RestoreMode.NO_CLAIM);
+ assertThat(executor.getFlinkConfig().get(RESTORE_MODE).toString())
+ .isEqualTo(RestoreMode.NO_CLAIM.toString());
assertThat(executor.getFlinkConfig().get(SAVEPOINT_IGNORE_UNCLAIMED_STATE)).isTrue();
}
diff --git a/flink-cdc-common/pom.xml b/flink-cdc-common/pom.xml
index 95f7d5fe0..80237d26d 100644
--- a/flink-cdc-common/pom.xml
+++ b/flink-cdc-common/pom.xml
@@ -44,4 +44,33 @@ limitations under the License.
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>flink2</id>
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/flink-cdc-composer/pom.xml b/flink-cdc-composer/pom.xml
index ca2812acf..4161491ba 100644
--- a/flink-cdc-composer/pom.xml
+++ b/flink-cdc-composer/pom.xml
@@ -96,4 +96,47 @@ limitations under the License.
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>flink2</id>
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>flink1</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink1-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslatorTest.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslatorTest.java
index 8ca8eb5cf..001f0ba64 100644
--- a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslatorTest.java
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/translator/DataSinkTranslatorTest.java
@@ -18,6 +18,7 @@
package org.apache.flink.cdc.composer.flink.translator;
import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
import org.apache.flink.api.dag.Transformation;
import org.apache.flink.cdc.common.event.Event;
import org.apache.flink.runtime.jobgraph.OperatorID;
@@ -87,5 +88,10 @@ class DataSinkTranslatorTest {
public SinkWriter<Event> createWriter(InitContext context) throws IOException {
return null;
}
+
+ @Override
+ public SinkWriter<Event> createWriter(WriterInitContext context) throws IOException {
+ return null;
+ }
}
}
diff --git a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java
index 58bbe3f42..d95034e0b 100644
--- a/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java
+++ b/flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/testsource/source/DistributedSourceFunction.java
@@ -79,8 +79,8 @@ public class DistributedSourceFunction extends RichParallelSourceFunction<Event>
public void open(Configuration parameters) throws Exception {
super.open(parameters);
iotaCounter = 0;
- subTaskId = getRuntimeContext().getIndexOfThisSubtask();
- parallelism = getRuntimeContext().getNumberOfParallelSubtasks();
+ subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
+ parallelism = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks();
if (distributedTables) {
tables =
IntStream.range(0, numOfTables)
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/pom.xml b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/pom.xml
index 3eda26dad..d28b7fa01 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/pom.xml
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/pom.xml
@@ -107,4 +107,39 @@ limitations under the License.
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>flink2</id>
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java</artifactId>
+ <version>${flink.2.x.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSink.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSink.java
index d5511b7ba..c15594622 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSink.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/main/java/org/apache/flink/cdc/connectors/values/sink/ValuesDataSink.java
@@ -19,6 +19,7 @@ package org.apache.flink.cdc.connectors.values.sink;
import org.apache.flink.api.connector.sink2.Sink;
import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.data.RecordData;
import org.apache.flink.cdc.common.event.ChangeEvent;
@@ -36,6 +37,7 @@ import org.apache.flink.cdc.common.sink.MetadataApplier;
import org.apache.flink.cdc.common.utils.SchemaUtils;
import org.apache.flink.cdc.connectors.values.ValuesDatabase;
+import java.io.IOException;
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
@@ -101,8 +103,17 @@ public class ValuesDataSink implements DataSink, Serializable {
return new ValuesSinkWriter(
materializedInMemory,
print,
- context.getSubtaskId(),
- context.getNumberOfParallelSubtasks());
+ context.getTaskInfo().getIndexOfThisSubtask(),
+ context.getTaskInfo().getNumberOfParallelSubtasks());
+ }
+
+ @Override
+ public SinkWriter<Event> createWriter(WriterInitContext context) throws IOException {
+ return new ValuesSinkWriter(
+ materializedInMemory,
+ print,
+ context.getTaskInfo().getIndexOfThisSubtask(),
+ context.getTaskInfo().getNumberOfParallelSubtasks());
}
}
diff --git a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/test/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceITCase.java b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/test/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceITCase.java
index 7bea12a18..640fa9cc3 100644
--- a/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/test/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceITCase.java
+++ b/flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values/src/test/java/org/apache/flink/cdc/connectors/values/source/ValuesDataSourceITCase.java
@@ -18,7 +18,6 @@
package org.apache.flink.cdc.connectors.values.source;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.cdc.common.data.binary.BinaryStringData;
import org.apache.flink.cdc.common.event.CreateTableEvent;
import org.apache.flink.cdc.common.event.DataChangeEvent;
@@ -40,6 +39,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import java.util.ArrayList;
import java.util.List;
@@ -49,6 +49,7 @@ import java.util.List;
* different enumeration situations of {@link
* org.apache.flink.cdc.connectors.values.source.ValuesDataSourceHelper}.
*/
+@Timeout(value = 60, unit = java.util.concurrent.TimeUnit.SECONDS)
class ValuesDataSourceITCase {
@BeforeEach
@@ -68,7 +69,6 @@ class ValuesDataSourceITCase {
private void executeDataStreamJob(ValuesDataSourceHelper.EventSetId type) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.enableCheckpointing(3000);
- env.setRestartStrategy(RestartStrategies.noRestart());
FlinkSourceProvider sourceProvider =
(FlinkSourceProvider) new ValuesDataSource(type).getEventSourceProvider();
CloseableIterator<Event> events =
diff --git a/flink-cdc-dist/pom.xml b/flink-cdc-dist/pom.xml
index 132bea2cd..8c918fddc 100644
--- a/flink-cdc-dist/pom.xml
+++ b/flink-cdc-dist/pom.xml
@@ -54,6 +54,42 @@ limitations under the License.
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>flink2</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-guava</artifactId>
+ <version>${flink.2.x.shaded.guava.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>flink1</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink1-compat</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-guava</artifactId>
+ <version>${flink.1.x.shaded.guava.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<plugin>
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/FlussE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/FlussE2eITCase.java
index 8f12e20c1..05b34dab4 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/FlussE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/FlussE2eITCase.java
@@ -26,6 +26,7 @@ import org.apache.flink.shaded.guava31.com.google.common.collect.ImmutableMap;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
@@ -48,6 +49,7 @@ import java.util.List;
import java.util.stream.Collectors;
/** An End-to-end test case for Fluss pipeline connector. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
@Testcontainers
public class FlussE2eITCase extends PipelineTestEnvironment {
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MaxComputeE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MaxComputeE2eITCase.java
index 59d966bc2..f8ae049f5 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MaxComputeE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MaxComputeE2eITCase.java
@@ -34,6 +34,7 @@ import com.aliyun.odps.task.SQLTask;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.GenericContainer;
@@ -56,6 +57,7 @@ import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
/** End-to-end tests for maxcompute cdc pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class MaxComputeE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MaxComputeE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToDorisE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToDorisE2eITCase.java
index 434ccbcf6..351e3ccea 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToDorisE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToDorisE2eITCase.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
@@ -49,6 +50,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/** End-to-end tests for mysql cdc to Doris pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class MySqlToDorisE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MySqlToDorisE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToHudiE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToHudiE2eITCase.java
index 76936b698..d1838041d 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToHudiE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToHudiE2eITCase.java
@@ -35,6 +35,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.BindMode;
@@ -64,6 +65,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/** End-to-end tests for mysql cdc to Iceberg pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
public class MySqlToHudiE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MySqlToHudiE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java
index c63c54ad6..6614d448a 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToIcebergE2eITCase.java
@@ -38,6 +38,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +71,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/** End-to-end tests for mysql cdc to Iceberg pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
public class MySqlToIcebergE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MySqlToIcebergE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToPaimonE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToPaimonE2eITCase.java
index d263dfd53..f0e7cb082 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToPaimonE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MySqlToPaimonE2eITCase.java
@@ -26,6 +26,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.Container;
@@ -48,6 +49,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/** End-to-end tests for mysql cdc to Paimon pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class MySqlToPaimonE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MySqlToPaimonE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java
index b85d77ebc..372b9f7dd 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eITCase.java
@@ -24,6 +24,7 @@ import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +37,7 @@ import java.time.Duration;
import java.util.function.Function;
/** End-to-end tests for mysql cdc pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class MysqlE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MysqlE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eWithYarnApplicationITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eWithYarnApplicationITCase.java
index cab98c6af..7de50469b 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eWithYarnApplicationITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlE2eWithYarnApplicationITCase.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.Network;
@@ -50,6 +51,7 @@ import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
/** End-to-end tests for mysql cdc pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
public class MysqlE2eWithYarnApplicationITCase extends PipelineTestOnYarnEnvironment {
private static final Logger LOG =
LoggerFactory.getLogger(MysqlE2eWithYarnApplicationITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlToKafkaE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlToKafkaE2eITCase.java
index a8093f0f7..5090d56f4 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlToKafkaE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/MysqlToKafkaE2eITCase.java
@@ -38,6 +38,7 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.KafkaContainer;
@@ -71,6 +72,7 @@ import static org.apache.flink.util.DockerImageVersions.KAFKA;
import static org.assertj.core.api.Assertions.assertThat;
/** End-to-end tests for mysql cdc to Kafka pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class MysqlToKafkaE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MysqlToKafkaE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OceanBaseE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OceanBaseE2eITCase.java
index 2bebedb3b..7bbf61378 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OceanBaseE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OceanBaseE2eITCase.java
@@ -29,6 +29,7 @@ import org.apache.commons.text.StringEscapeUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.output.Slf4jLogConsumer;
@@ -51,6 +52,7 @@ import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
/** OceanBase flink cdc pipeline connector sink integrate test. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class OceanBaseE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(OceanBaseE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OracleE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OracleE2eITCase.java
index 0be4734e7..f474815e4 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OracleE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/OracleE2eITCase.java
@@ -27,6 +27,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.DockerClientFactory;
@@ -61,6 +62,7 @@ import static org.apache.flink.cdc.connectors.oracle.source.OracleSourceTestBase
import static org.apache.flink.cdc.connectors.oracle.source.OracleSourceTestBase.TOP_USER;
/** End-to-end tests for Oracle cdc pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
public class OracleE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(OracleE2eITCase.class);
protected static final Pattern COMMENT_PATTERN = Pattern.compile("^(.*)--.*$");
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/PostgresE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/PostgresE2eITCase.java
index e4b33b3ff..69a65682f 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/PostgresE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/PostgresE2eITCase.java
@@ -24,6 +24,7 @@ import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.PostgreSQLContainer;
@@ -42,6 +43,7 @@ import static org.apache.flink.cdc.connectors.postgres.PostgresTestBase.getJdbcC
import static org.apache.flink.cdc.connectors.postgres.PostgresTestBase.getSlotName;
/** End-to-end tests for postgres cdc pipeline job. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
public class PostgresE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(PostgresE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java
index 2514711c0..c9169388d 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/RouteE2eITCase.java
@@ -23,6 +23,7 @@ import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
@@ -37,6 +38,7 @@ import java.util.function.Function;
import java.util.stream.IntStream;
/** E2e tests for routing features. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class RouteE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(RouteE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java
index a6ade384c..7595dd6d1 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolveE2eITCase.java
@@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +37,7 @@ import java.util.Collections;
import java.util.List;
/** E2e tests for Schema Evolution cases. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class SchemaEvolveE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(SchemaEvolveE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java
index 130dba0c4..5410eb903 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/SchemaEvolvingTransformE2eITCase.java
@@ -24,6 +24,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +37,7 @@ import java.util.Collections;
import java.util.List;
/** E2e tests for Schema Evolution cases. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class SchemaEvolvingTransformE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG =
LoggerFactory.getLogger(SchemaEvolvingTransformE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java
index 351bd76fd..925bc6059 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/TransformE2eITCase.java
@@ -27,6 +27,7 @@ import org.apache.flink.cdc.runtime.operators.transform.PreTransformOperator;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
@@ -51,6 +52,7 @@ import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
/** E2e tests for the {@link PreTransformOperator} and {@link PostTransformOperator}. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class TransformE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(TransformE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/UdfE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/UdfE2eITCase.java
index ca8e3d764..c55a67c76 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/UdfE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/UdfE2eITCase.java
@@ -23,6 +23,7 @@ import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -39,6 +40,7 @@ import java.util.function.Function;
import java.util.stream.Stream;
/** E2e tests for User-defined functions. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class UdfE2eITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(TransformE2eITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/ValuesE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/ValuesE2eITCase.java
new file mode 100644
index 000000000..0f7bc28cb
--- /dev/null
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/ValuesE2eITCase.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.pipeline.tests;
+
+import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
+
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.Duration;
+
+/** End-to-end tests for values cdc pipeline job. */
+class ValuesE2eITCase extends PipelineTestEnvironment {
+ private static final Logger LOG = LoggerFactory.getLogger(ValuesE2eITCase.class);
+
+ @Test
+ void testValuesSingleSplitSingleTable() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: SINGLE_SPLIT_SINGLE_TABLE\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, ], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @Test
+ void testValuesSingleSplitMultiTables() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: SINGLE_SPLIT_MULTI_TABLES\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, 2], after=[2, x], op=UPDATE, meta=()}");
+ }
+
+ @Test
+ void testValuesMultiSplitsSingleTable() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: MULTI_SPLITS_SINGLE_TABLE\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[4, 4], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[5, 5], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[6, 6], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, 2], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[4, 4], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[6, 6], after=[], op=DELETE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1, ], after=[1, 1, x], op=UPDATE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[3, 3, ], after=[3, 3, x], op=UPDATE, meta=()}");
+ }
+
+ @Test
+ void testValuesSingleSplitSingleTableWithDefaultValue() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: SINGLE_SPLIT_SINGLE_TABLE_WITH_DEFAULT_VALUE\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`col3` STRING, position=LAST, existedColumnName=null}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={col2=newCol2, col3=newCol3}}",
+ "DropColumnEvent{tableId=default_namespace.default_schema.table1, droppedColumnNames=[newCol2]}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[1, 1], after=[], op=DELETE, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[2, ], after=[2, x], op=UPDATE, meta=()}",
+ "AddColumnEvent{tableId=default_namespace.default_schema.table1, addedColumns=[ColumnWithPosition{column=`colWithDefault` STRING 'flink', position=LAST, existedColumnName=null}]}",
+ "RenameColumnEvent{tableId=default_namespace.default_schema.table1, nameMapping={colWithDefault=newColWithDefault}}");
+ }
+
+ @Test
+ void testValuesSingleSplitSingleBatchTable() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: SINGLE_SPLIT_SINGLE_BATCH_TABLE\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d\n"
+ + " execution.runtime-mode: BATCH",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @Test
+ void testValuesSingleSplitMultiBatchTable() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: SINGLE_SPLIT_MULTI_BATCH_TABLE\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d\n"
+ + " execution.runtime-mode: BATCH",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "CreateTableEvent{tableId=default_namespace.default_schema.table2, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table2, before=[], after=[3, 3], op=INSERT, meta=()}");
+ }
+
+ @Test
+ void testValuesMultiSplitsSingleBatchTable() throws Exception {
+ String pipelineJob =
+ String.format(
+ "source:\n"
+ + " type: values\n"
+ + " event-set.id: MULTI_SPLITS_SINGLE_BATCH_TABLE\n"
+ + "\n"
+ + "sink:\n"
+ + " type: values\n"
+ + " print.enabled: true\n"
+ + "\n"
+ + "pipeline:\n"
+ + " parallelism: %d\n"
+ + " execution.runtime-mode: BATCH",
+ parallelism);
+
+ submitPipelineJob(pipelineJob);
+ waitUntilJobFinished(Duration.ofSeconds(30));
+ LOG.info("Pipeline job has finished");
+
+ validateResult(
+ "CreateTableEvent{tableId=default_namespace.default_schema.table1, schema=columns={`col1` STRING,`col2` STRING}, primaryKeys=col1, options=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[1, 1], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[2, 2], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[3, 3], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[4, 4], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[5, 5], op=INSERT, meta=()}",
+ "DataChangeEvent{tableId=default_namespace.default_schema.table1, before=[], after=[6, 6], op=INSERT, meta=()}");
+ }
+}
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/MySqlToPaimonMigrationITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/MySqlToPaimonMigrationITCase.java
index bcedcf7b0..bcfac098f 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/MySqlToPaimonMigrationITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/MySqlToPaimonMigrationITCase.java
@@ -28,6 +28,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.slf4j.Logger;
@@ -52,6 +53,7 @@ import java.util.stream.Collectors;
/**
* E2e cases for stopping & restarting jobs of `MySQL source to Paimon sink` from previous state.
*/
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class MySqlToPaimonMigrationITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(MySqlToPaimonMigrationITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/YamlJobMigrationITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/YamlJobMigrationITCase.java
index bdf970ea6..ef2112935 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/YamlJobMigrationITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/migration/YamlJobMigrationITCase.java
@@ -29,6 +29,7 @@ import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.slf4j.Logger;
@@ -43,6 +44,7 @@ import java.time.Duration;
import java.util.function.Function;
/** E2e cases for stopping & restarting jobs from previous state. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
class YamlJobMigrationITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(YamlJobMigrationITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/specs/FlinkPipelineSpecsITCase.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/specs/FlinkPipelineSpecsITCase.java
index b7ac0507c..0d5515cb3 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/specs/FlinkPipelineSpecsITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/specs/FlinkPipelineSpecsITCase.java
@@ -23,6 +23,7 @@ import org.apache.flink.cdc.pipeline.tests.utils.PipelineTestEnvironment;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -50,6 +51,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
/** YAML-spec based pipeline test cases. */
+@EnabledIfSystemProperty(named = "specifiedFlinkVersion", matches = "^1.*")
public class FlinkPipelineSpecsITCase extends PipelineTestEnvironment {
private static final Logger LOG = LoggerFactory.getLogger(FlinkPipelineSpecsITCase.class);
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/TarballFetcher.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/TarballFetcher.java
index 1d5788918..450f683d6 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/TarballFetcher.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/TarballFetcher.java
@@ -19,10 +19,16 @@ package org.apache.flink.cdc.pipeline.tests.utils;
import org.apache.flink.cdc.common.test.utils.TestUtils;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.ExecCreateCmdResponse;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testcontainers.DockerClientFactory;
import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.FrameConsumerResultCallback;
+import org.testcontainers.containers.output.OutputFrame;
+import org.testcontainers.containers.output.ToStringConsumer;
import org.testcontainers.utility.MountableFile;
import java.net.URL;
@@ -60,11 +66,11 @@ public abstract class TarballFetcher {
container.copyFileToContainer(
MountableFile.forHostPath(
- TestUtils.getResource("flink-cdc.sh", "flink-cdc-dist", "src"), 755),
+ TestUtils.getResource("flink-cdc.sh", "flink-cdc-dist", "src")),
version.workDir() + "/bin/flink-cdc.sh");
container.copyFileToContainer(
MountableFile.forHostPath(
- TestUtils.getResource("flink-cdc.yaml", "flink-cdc-dist", "src"), 755),
+ TestUtils.getResource("flink-cdc.yaml", "flink-cdc-dist", "src")),
version.workDir() + "/conf/flink-cdc.yaml");
container.copyFileToContainer(
MountableFile.forHostPath(TestUtils.getResource("flink-cdc-dist.jar")),
@@ -78,6 +84,9 @@ public abstract class TarballFetcher {
TestUtils.getResource("values-cdc-pipeline-connector.jar")),
version.workDir() + "/lib/values-cdc-pipeline-connector.jar");
+ // Ensure the script has execute permission
+ runInContainerAsRoot(container, "chmod", "+x", version.workDir() + "/bin/flink-cdc.sh");
+
} else {
LOG.info("CDC {} is a released version, download it from the Internet...", version);
@@ -109,6 +118,23 @@ public abstract class TarballFetcher {
container.copyFileToContainer(MountableFile.forHostPath(tempFile), containerPath);
}
+ private static void runInContainerAsRoot(GenericContainer<?> container, String... command)
+ throws InterruptedException {
+ ToStringConsumer stdoutConsumer = new ToStringConsumer();
+ ToStringConsumer stderrConsumer = new ToStringConsumer();
+ DockerClient dockerClient = DockerClientFactory.instance().client();
+ ExecCreateCmdResponse execCreateCmdResponse =
+ dockerClient
+ .execCreateCmd(container.getContainerId())
+ .withUser("root")
+ .withCmd(command)
+ .exec();
+ FrameConsumerResultCallback callback = new FrameConsumerResultCallback();
+ callback.addConsumer(OutputFrame.OutputType.STDOUT, stdoutConsumer);
+ callback.addConsumer(OutputFrame.OutputType.STDERR, stderrConsumer);
+ dockerClient.execStartCmd(execCreateCmdResponse.getId()).exec(callback).awaitCompletion();
+ }
+
/** Enum for all released Flink CDC version tags. */
public enum CdcVersion {
V3_1_1("3.1.1"),
diff --git a/flink-cdc-common/pom.xml b/flink-cdc-flink1-compat/pom.xml
similarity index 68%
copy from flink-cdc-common/pom.xml
copy to flink-cdc-flink1-compat/pom.xml
index 95f7d5fe0..8b21cdc2e 100644
--- a/flink-cdc-common/pom.xml
+++ b/flink-cdc-flink1-compat/pom.xml
@@ -25,23 +25,11 @@ limitations under the License.
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>flink-cdc-common</artifactId>
+ <artifactId>flink-cdc-flink1-compat</artifactId>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <id>test-jar</id>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <properties>
+ <flink.version>${flink.1.x.version}</flink.version>
+ <shaded.guava.version>${flink.1.x.shaded.guava.version}</shaded.guava.version>
+ </properties>
-</project>
\ No newline at end of file
+</project>
diff --git a/flink-cdc-flink1-compat/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotAdapter.java b/flink-cdc-flink1-compat/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotAdapter.java
new file mode 100644
index 000000000..b868b4fba
--- /dev/null
+++ b/flink-cdc-flink1-compat/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotAdapter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 1.20. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface TypeSerializerSnapshotAdapter<T> extends TypeSerializerSnapshot<T> {}
diff --git a/flink-cdc-flink1-compat/src/main/java/org/apache/flink/cdc/runtime/operators/AbstractStreamOperatorAdapter.java b/flink-cdc-flink1-compat/src/main/java/org/apache/flink/cdc/runtime/operators/AbstractStreamOperatorAdapter.java
new file mode 100644
index 000000000..73c578b04
--- /dev/null
+++ b/flink-cdc-flink1-compat/src/main/java/org/apache/flink/cdc/runtime/operators/AbstractStreamOperatorAdapter.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+
+/**
+ * Compatibility adapter for Flink 1.20. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public class AbstractStreamOperatorAdapter<OUT> extends AbstractStreamOperator<OUT> {}
diff --git a/flink-cdc-flink1-compat/src/main/java/org/apache/flink/runtime/event/WatermarkEvent.java b/flink-cdc-flink1-compat/src/main/java/org/apache/flink/runtime/event/WatermarkEvent.java
new file mode 100644
index 000000000..b9d7f7fb6
--- /dev/null
+++ b/flink-cdc-flink1-compat/src/main/java/org/apache/flink/runtime/event/WatermarkEvent.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.event;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 1.20. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public class WatermarkEvent {}
diff --git a/flink-cdc-common/pom.xml b/flink-cdc-flink2-compat/pom.xml
similarity index 68%
copy from flink-cdc-common/pom.xml
copy to flink-cdc-flink2-compat/pom.xml
index 95f7d5fe0..4170b6f90 100644
--- a/flink-cdc-common/pom.xml
+++ b/flink-cdc-flink2-compat/pom.xml
@@ -25,23 +25,11 @@ limitations under the License.
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>flink-cdc-common</artifactId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <id>test-jar</id>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ <shaded.guava.version>${flink.2.x.shaded.guava.version}</shaded.guava.version>
+ </properties>
-</project>
\ No newline at end of file
+</project>
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotAdapter.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotAdapter.java
new file mode 100644
index 000000000..5060d9219
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/common/typeutils/TypeSerializerSnapshotAdapter.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.common.typeutils;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface TypeSerializerSnapshotAdapter<T> extends TypeSerializerSnapshot<T> {
+
+ TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(
+ TypeSerializer<T> typeSerializer);
+
+ @Override
+ default TypeSerializerSchemaCompatibility<T> resolveSchemaCompatibility(
+ TypeSerializerSnapshot<T> typeSerializerSnapshot) {
+ return resolveSchemaCompatibility(typeSerializerSnapshot.restoreSerializer());
+ }
+}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/connector/sink2/Sink.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/connector/sink2/Sink.java
new file mode 100644
index 000000000..0b06d3f51
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/connector/sink2/Sink.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.connector.sink2;
+
+import org.apache.flink.annotation.Experimental;
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.operators.MailboxExecutor;
+import org.apache.flink.api.common.operators.ProcessingTimeService;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
+import org.apache.flink.util.UserCodeClassLoader;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Optional;
+import java.util.function.Consumer;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface Sink<InputT> extends Serializable {
+
+ SinkWriter<InputT> createWriter(WriterInitContext var1) throws IOException;
+
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ SinkWriter<InputT> createWriter(InitContext var1) throws IOException;
+
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ @Internal
+ public interface InitContext extends org.apache.flink.api.connector.sink2.InitContext {
+ UserCodeClassLoader getUserCodeClassLoader();
+
+ MailboxExecutor getMailboxExecutor();
+
+ ProcessingTimeService getProcessingTimeService();
+
+ SinkWriterMetricGroup metricGroup();
+
+ SerializationSchema.InitializationContext asSerializationSchemaInitializationContext();
+
+ boolean isObjectReuseEnabled();
+
+ <IN> TypeSerializer<IN> createInputSerializer();
+
+ @Experimental
+ default <MetaT> Optional<Consumer<MetaT>> metadataConsumer() {
+ return Optional.empty();
+ }
+ }
+}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/connector/sink2/TwoPhaseCommittingSink.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/connector/sink2/TwoPhaseCommittingSink.java
new file mode 100644
index 000000000..cae29dbfe
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/api/connector/sink2/TwoPhaseCommittingSink.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.connector.sink2;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface TwoPhaseCommittingSink<InputT, CommT>
+ extends Sink<InputT>, SupportsCommitter<CommT> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/cdc/runtime/operators/AbstractStreamOperatorAdapter.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/cdc/runtime/operators/AbstractStreamOperatorAdapter.java
new file mode 100644
index 000000000..e893db104
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/cdc/runtime/operators/AbstractStreamOperatorAdapter.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.cdc.runtime.operators;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.streaming.api.graph.StreamConfig;
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.ChainingStrategy;
+import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+import org.apache.flink.streaming.runtime.tasks.StreamTask;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public class AbstractStreamOperatorAdapter<OUT> extends AbstractStreamOperator<OUT> {
+ protected ChainingStrategy chainingStrategy;
+
+ @Override
+ public void setup(
+ StreamTask<?, ?> containingTask,
+ StreamConfig config,
+ Output<StreamRecord<OUT>> output) {
+ super.setup(containingTask, config, output);
+ }
+}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/core/execution/RestoreMode.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/core/execution/RestoreMode.java
new file mode 100644
index 000000000..6e7cca53f
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/core/execution/RestoreMode.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.core.execution;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.DescribedEnum;
+import org.apache.flink.configuration.description.InlineElement;
+import org.apache.flink.configuration.description.TextElement;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ *
+ * <p>Copy from <a
+ * href="https://github.com/apache/flink/blob/release-1.20.3/flink-core/src/main/java/org/apache/flink/core/execution/RestoreMode.java">...</a>.
+ */
+@Internal
+public enum RestoreMode implements DescribedEnum {
+ CLAIM(
+ "Flink will take ownership of the given snapshot. It will clean the snapshot once it is subsumed by newer ones."),
+ NO_CLAIM(
+ "Flink will not claim ownership of the snapshot files. However it will make sure it does not depend on any artefacts from the restored snapshot. In order to do that, Flink will take the first checkpoint as a full one, which means it might reupload/duplicate files that are part of the restored checkpoint."),
+ /**
+ * @deprecated
+ */
+ @Deprecated
+ LEGACY(
+ "This is the mode in which Flink worked until 1.15. It will not claim ownership of the snapshot and will not delete the files. However, it can directly depend on the existence of the files of the restored checkpoint. It might not be safe to delete checkpoints that were restored in legacy mode. This mode is deprecated, please use CLAIM or NO_CLAIM mode to get a clear state file ownership.");
+
+ private final String description;
+ public static final RestoreMode DEFAULT = NO_CLAIM;
+
+ private RestoreMode(String description) {
+ this.description = description;
+ }
+
+ @Internal
+ public InlineElement getDescription() {
+ return TextElement.text(this.description);
+ }
+}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java
new file mode 100644
index 000000000..83494815c
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.jobgraph;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.docs.Documentation;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+import org.apache.flink.core.execution.RestoreMode;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ *
+ * <p>Copy from <a
+ * href="https://github.com/apache/flink/blob/release-1.20.3/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/SavepointConfigOptions.java">...</a>.
+ */
+@Deprecated
+@Documentation.ExcludeFromDocumentation("Hidden for deprecated.")
+@Internal
+public class SavepointConfigOptions {
+ public static final ConfigOption<String> SAVEPOINT_PATH =
+ ConfigOptions.key("execution.savepoint.path")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "Path to a savepoint to restore the job from (for example hdfs:///flink/savepoint-1537).");
+ public static final ConfigOption<Boolean> SAVEPOINT_IGNORE_UNCLAIMED_STATE =
+ ConfigOptions.key("execution.savepoint.ignore-unclaimed-state")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "Allow to skip savepoint state that cannot be restored. Allow this if you removed an operator from your pipeline after the savepoint was triggered.");
+ public static final ConfigOption<RestoreMode> RESTORE_MODE;
+
+ static {
+ RESTORE_MODE =
+ ConfigOptions.key("execution.savepoint-restore-mode")
+ .enumType(RestoreMode.class)
+ .defaultValue(RestoreMode.DEFAULT)
+ .withDescription(
+ "Describes the mode how Flink should restore from the given savepoint or retained checkpoint.");
+ }
+}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPostCommitTopology.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPostCommitTopology.java
new file mode 100644
index 000000000..2fe0127b9
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPostCommitTopology.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.connector.sink2;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface WithPostCommitTopology<InputT, CommT> extends SupportsPostCommitTopology<CommT> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPreCommitTopology.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPreCommitTopology.java
new file mode 100644
index 000000000..de18bdb86
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPreCommitTopology.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.connector.sink2;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface WithPreCommitTopology<InputT, CommT>
+ extends SupportsPreCommitTopology<CommT, CommT> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPreWriteTopology.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPreWriteTopology.java
new file mode 100644
index 000000000..8a1d8c2f2
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/connector/sink2/WithPreWriteTopology.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.connector.sink2;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.connector.sink2.Sink;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface WithPreWriteTopology<InputT>
+ extends Sink<InputT>, SupportsPreWriteTopology<InputT> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/sink/SinkFunction.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/sink/SinkFunction.java
new file mode 100644
index 000000000..ff1a27611
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/sink/SinkFunction.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.functions.sink;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface SinkFunction<IN>
+ extends org.apache.flink.streaming.api.functions.sink.legacy.SinkFunction<IN> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java
new file mode 100644
index 000000000..c45677998
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/source/RichParallelSourceFunction.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.functions.source;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.functions.AbstractRichFunction;
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.source.legacy.ParallelSourceFunction;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Deprecated
+@Internal
+public abstract class RichParallelSourceFunction<OUT> extends AbstractRichFunction
+ implements ParallelSourceFunction<OUT>, SourceFunction<OUT> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ super.open(openContext);
+ open(new Configuration());
+ }
+
+ public void open(Configuration parameters) throws Exception {}
+}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
new file mode 100644
index 000000000..6147d72d2
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/functions/source/SourceFunction.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.functions.source;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface SourceFunction<T>
+ extends org.apache.flink.streaming.api.functions.source.legacy.SourceFunction<T> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/operators/YieldingOperatorFactory.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/operators/YieldingOperatorFactory.java
new file mode 100644
index 000000000..0cc855d45
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/streaming/api/operators/YieldingOperatorFactory.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.streaming.api.operators;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public interface YieldingOperatorFactory<OUT>
+ extends org.apache.flink.streaming.api.operators.legacy.YieldingOperatorFactory<OUT> {}
diff --git a/flink-cdc-flink2-compat/src/main/java/org/apache/flink/table/api/ValidationException.java b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/table/api/ValidationException.java
new file mode 100644
index 000000000..20cc03305
--- /dev/null
+++ b/flink-cdc-flink2-compat/src/main/java/org/apache/flink/table/api/ValidationException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.api;
+
+import org.apache.flink.annotation.Internal;
+
+import javax.annotation.Nullable;
+
+/**
+ * Compatibility adapter for Flink 2.2. This class is part of the multi-version compatibility layer
+ * that allows Flink CDC to work across different Flink versions.
+ */
+@Internal
+public class ValidationException extends RuntimeException {
+
+ public ValidationException(String message) {
+ super(message);
+ }
+
+ public ValidationException(String message, @Nullable Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/flink-cdc-runtime/pom.xml b/flink-cdc-runtime/pom.xml
index bb1c78fbb..fc474013c 100644
--- a/flink-cdc-runtime/pom.xml
+++ b/flink-cdc-runtime/pom.xml
@@ -103,4 +103,68 @@ limitations under the License.
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>flink2</id>
+ <properties>
+ <flink.version>${flink.2.x.version}</flink.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink2-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>flink1</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink1-compat</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>flink2-exclude-flink1</id>
+ <activation>
+ <property>
+ <name>flink.version</name>
+ <value>${flink.2.x.version}</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cdc-flink1-compat</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>*</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
</project>
\ No newline at end of file
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java
index 15c0571ba..9ec2b2fc9 100755
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaOperator.java
@@ -29,6 +29,7 @@ import org.apache.flink.cdc.common.route.RouteRule;
import org.apache.flink.cdc.common.route.TableIdRouter;
import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils;
import org.apache.flink.cdc.runtime.operators.schema.common.SchemaDerivator;
import org.apache.flink.cdc.runtime.operators.schema.common.metrics.SchemaOperatorMetrics;
@@ -39,7 +40,6 @@ import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -61,7 +61,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
/** This operator merges upstream inferred schema into a centralized Schema Registry. */
-public class SchemaOperator extends AbstractStreamOperator<Event>
+public class SchemaOperator extends AbstractStreamOperatorAdapter<Event>
implements OneInputStreamOperator<PartitioningEvent, Event>, Serializable {
private static final Logger LOG = LoggerFactory.getLogger(SchemaOperator.class);
@@ -97,7 +97,7 @@ public class SchemaOperator extends AbstractStreamOperator<Event>
@Override
public void open() throws Exception {
super.open();
- subTaskId = getRuntimeContext().getIndexOfThisSubtask();
+ subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
upstreamSchemaTable = HashBasedTable.create();
evolvedSchemaMap = new HashMap<>();
tableIdRouter = new TableIdRouter(routingRules);
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/BatchSchemaOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/BatchSchemaOperator.java
index df990425a..8ed7d0916 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/BatchSchemaOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/BatchSchemaOperator.java
@@ -28,10 +28,10 @@ import org.apache.flink.cdc.common.route.RouteRule;
import org.apache.flink.cdc.common.route.TableIdRouter;
import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.sink.MetadataApplier;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.common.SchemaDerivator;
import org.apache.flink.cdc.runtime.operators.schema.common.SchemaManager;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
@@ -49,7 +49,7 @@ import java.util.stream.Collectors;
/** The operator will apply create table event and router mapper in batch mode. */
@Internal
-public class BatchSchemaOperator extends AbstractStreamOperator<Event>
+public class BatchSchemaOperator extends AbstractStreamOperatorAdapter<Event>
implements OneInputStreamOperator<Event, Event>, Serializable {
private static final long serialVersionUID = 1L;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/SchemaOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/SchemaOperator.java
index 901ba168d..7dd9a210a 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/SchemaOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/regular/SchemaOperator.java
@@ -29,6 +29,7 @@ import org.apache.flink.cdc.common.route.RouteRule;
import org.apache.flink.cdc.common.route.TableIdRouter;
import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils;
import org.apache.flink.cdc.runtime.operators.schema.common.SchemaDerivator;
import org.apache.flink.cdc.runtime.operators.schema.common.metrics.SchemaOperatorMetrics;
@@ -38,7 +39,6 @@ import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.runtime.operators.coordination.CoordinationRequest;
import org.apache.flink.runtime.operators.coordination.CoordinationResponse;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
@@ -65,7 +65,7 @@ import static org.apache.flink.cdc.common.pipeline.PipelineOptions.DEFAULT_SCHEM
* SchemaChangeEvent}s and block the stream for tables before their schema changes finish.
*/
@Internal
-public class SchemaOperator extends AbstractStreamOperator<Event>
+public class SchemaOperator extends AbstractStreamOperatorAdapter<Event>
implements OneInputStreamOperator<Event, Event>, Serializable {
private static final long serialVersionUID = 1L;
@@ -131,7 +131,7 @@ public class SchemaOperator extends AbstractStreamOperator<Event>
this.schemaOperatorMetrics =
new SchemaOperatorMetrics(
getRuntimeContext().getMetricGroup(), schemaChangeBehavior);
- this.subTaskId = getRuntimeContext().getIndexOfThisSubtask();
+ this.subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
this.originalSchemaMap = new HashMap<>();
this.evolvedSchemaMap = new HashMap<>();
this.router = new TableIdRouter(routingRules);
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkFunctionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkFunctionOperator.java
index 560909006..5b3925aad 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkFunctionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkFunctionOperator.java
@@ -21,7 +21,6 @@ import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.event.Event;
import org.apache.flink.cdc.runtime.operators.sink.exception.SinkWrapperException;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.StreamSink;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -37,7 +36,6 @@ public class BatchDataSinkFunctionOperator extends StreamSink<Event> {
public BatchDataSinkFunctionOperator(SinkFunction<Event> userFunction) {
super(userFunction);
- this.chainingStrategy = ChainingStrategy.ALWAYS;
}
@Override
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkWriterOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkWriterOperator.java
index 52900fef7..e89f3f814 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkWriterOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/BatchDataSinkWriterOperator.java
@@ -23,6 +23,7 @@ import org.apache.flink.api.connector.sink2.SinkWriter;
import org.apache.flink.cdc.common.annotation.Internal;
import org.apache.flink.cdc.common.event.Event;
import org.apache.flink.cdc.common.event.FlushEvent;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.sink.exception.SinkWrapperException;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateSnapshotContext;
@@ -33,6 +34,7 @@ import org.apache.flink.streaming.api.operators.BoundedOneInput;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
@@ -41,6 +43,7 @@ import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
/**
* An operator that processes records to be written into a {@link Sink} in batch mode.
@@ -53,7 +56,7 @@ import java.lang.reflect.Field;
*/
@Internal
public class BatchDataSinkWriterOperator<CommT>
- extends AbstractStreamOperator<CommittableMessage<CommT>>
+ extends AbstractStreamOperatorAdapter<CommittableMessage<CommT>>
implements OneInputStreamOperator<Event, CommittableMessage<CommT>>, BoundedOneInput {
private final Sink<Event> sink;
@@ -88,8 +91,7 @@ public class BatchDataSinkWriterOperator<CommT>
Output<StreamRecord<CommittableMessage<CommT>>> output) {
super.setup(containingTask, config, output);
flinkWriterOperator = createFlinkWriterOperator();
- this.<AbstractStreamOperator<CommittableMessage<CommT>>>getFlinkWriterOperator()
- .setup(containingTask, config, output);
+ invokeSetup(flinkWriterOperator, containingTask, config, output);
}
@Override
@@ -161,19 +163,56 @@ public class BatchDataSinkWriterOperator<CommT>
// -------------------------- Reflection helper functions --------------------------
+ private void invokeSetup(
+ Object operator,
+ StreamTask<?, ?> containingTask,
+ StreamConfig config,
+ Output<?> output) {
+ try {
+ Method setupMethod =
+ AbstractStreamOperator.class.getDeclaredMethod(
+ "setup", StreamTask.class, StreamConfig.class, Output.class);
+ setupMethod.setAccessible(true);
+ setupMethod.invoke(operator, containingTask, config, output);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to invoke setup on flinkWriterOperator", e);
+ }
+ }
+
private Object createFlinkWriterOperator() {
+ Class<?> flinkWriterClass;
try {
- Class<?> flinkWriterClass =
+ flinkWriterClass =
getRuntimeContext()
.getUserCodeClassLoader()
.loadClass(
"org.apache.flink.streaming.runtime.operators.sink.SinkWriterOperator");
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Failed to load SinkWriterOperator class", e);
+ }
+
+ try {
Constructor<?> constructor =
flinkWriterClass.getDeclaredConstructor(
Sink.class, ProcessingTimeService.class, MailboxExecutor.class);
constructor.setAccessible(true);
return constructor.newInstance(sink, processingTimeService, mailboxExecutor);
+ } catch (NoSuchMethodException e) {
+ // Constructor with 3 parameters not found, try the 4-parameter version
+ try {
+ Constructor<?> constructor =
+ flinkWriterClass.getDeclaredConstructor(
+ StreamOperatorParameters.class,
+ Sink.class,
+ ProcessingTimeService.class,
+ MailboxExecutor.class);
+ constructor.setAccessible(true);
+ return constructor.newInstance(null, sink, processingTimeService, mailboxExecutor);
+ } catch (Exception ex) {
+ throw new RuntimeException("Failed to create SinkWriterOperator in Flink", ex);
+ }
} catch (Exception e) {
+ // Other exceptions (e.g., InvocationTargetException) indicate real failures
throw new RuntimeException("Failed to create SinkWriterOperator in Flink", e);
}
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java
index df0e7802b..4286b3548 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkFunctionOperator.java
@@ -30,7 +30,6 @@ import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.Output;
import org.apache.flink.streaming.api.operators.StreamSink;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -62,7 +61,6 @@ public class DataSinkFunctionOperator extends StreamSink<Event> {
super(userFunction);
this.schemaOperatorID = schemaOperatorID;
processedTableIds = new HashSet<>();
- this.chainingStrategy = ChainingStrategy.ALWAYS;
}
@Override
@@ -79,7 +77,8 @@ public class DataSinkFunctionOperator extends StreamSink<Event> {
@Override
public void initializeState(StateInitializationContext context) throws Exception {
- schemaEvolutionClient.registerSubtask(getRuntimeContext().getIndexOfThisSubtask());
+ schemaEvolutionClient.registerSubtask(
+ getRuntimeContext().getTaskInfo().getIndexOfThisSubtask());
super.initializeState(context);
}
@@ -135,7 +134,8 @@ public class DataSinkFunctionOperator extends StreamSink<Event> {
});
}
schemaEvolutionClient.notifyFlushSuccess(
- getRuntimeContext().getIndexOfThisSubtask(), event.getSourceSubTaskId());
+ getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(),
+ event.getSourceSubTaskId());
}
private void emitLatestSchema(TableId tableId) throws Exception {
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java
index 0432d1f54..bac2929ce 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkWriterOperator.java
@@ -28,6 +28,7 @@ import org.apache.flink.cdc.common.event.FlushEvent;
import org.apache.flink.cdc.common.event.SchemaChangeEventType;
import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.sink.exception.SinkWrapperException;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.state.StateInitializationContext;
@@ -39,6 +40,7 @@ import org.apache.flink.streaming.api.operators.BoundedOneInput;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
+import org.apache.flink.streaming.api.operators.StreamOperatorParameters;
import org.apache.flink.streaming.api.watermark.Watermark;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;
@@ -47,6 +49,7 @@ import org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@@ -62,7 +65,8 @@ import java.util.Set;
* @param <CommT> the type of the committable (to send to downstream operators)
*/
@Internal
-public class DataSinkWriterOperator<CommT> extends AbstractStreamOperator<CommittableMessage<CommT>>
+public class DataSinkWriterOperator<CommT>
+ extends AbstractStreamOperatorAdapter<CommittableMessage<CommT>>
implements OneInputStreamOperator<Event, CommittableMessage<CommT>>, BoundedOneInput {
private SchemaEvolutionClient schemaEvolutionClient;
@@ -107,8 +111,7 @@ public class DataSinkWriterOperator<CommT> extends AbstractStreamOperator<Commit
Output<StreamRecord<CommittableMessage<CommT>>> output) {
super.setup(containingTask, config, output);
flinkWriterOperator = createFlinkWriterOperator();
- this.<AbstractStreamOperator<CommittableMessage<CommT>>>getFlinkWriterOperator()
- .setup(containingTask, config, output);
+ invokeSetup(flinkWriterOperator, containingTask, config, output);
schemaEvolutionClient =
new SchemaEvolutionClient(
containingTask.getEnvironment().getOperatorCoordinatorEventGateway(),
@@ -123,7 +126,8 @@ public class DataSinkWriterOperator<CommT> extends AbstractStreamOperator<Commit
@Override
public void initializeState(StateInitializationContext context) throws Exception {
- schemaEvolutionClient.registerSubtask(getRuntimeContext().getIndexOfThisSubtask());
+ schemaEvolutionClient.registerSubtask(
+ getRuntimeContext().getTaskInfo().getIndexOfThisSubtask());
this.<AbstractStreamOperator<CommittableMessage<CommT>>>getFlinkWriterOperator()
.initializeState(context);
}
@@ -223,7 +227,8 @@ public class DataSinkWriterOperator<CommT> extends AbstractStreamOperator<Commit
});
}
schemaEvolutionClient.notifyFlushSuccess(
- getRuntimeContext().getIndexOfThisSubtask(), event.getSourceSubTaskId());
+ getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(),
+ event.getSourceSubTaskId());
}
private void emitLatestSchema(TableId tableId) throws Exception {
@@ -243,19 +248,56 @@ public class DataSinkWriterOperator<CommT> extends AbstractStreamOperator<Commit
// -------------------------- Reflection helper functions --------------------------
+ private void invokeSetup(
+ Object operator,
+ StreamTask<?, ?> containingTask,
+ StreamConfig config,
+ Output<?> output) {
+ try {
+ Method setupMethod =
+ AbstractStreamOperator.class.getDeclaredMethod(
+ "setup", StreamTask.class, StreamConfig.class, Output.class);
+ setupMethod.setAccessible(true);
+ setupMethod.invoke(operator, containingTask, config, output);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to invoke setup on flinkWriterOperator", e);
+ }
+ }
+
private Object createFlinkWriterOperator() {
+ Class<?> flinkWriterClass;
try {
- Class<?> flinkWriterClass =
+ flinkWriterClass =
getRuntimeContext()
.getUserCodeClassLoader()
.loadClass(
"org.apache.flink.streaming.runtime.operators.sink.SinkWriterOperator");
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException("Failed to load SinkWriterOperator class", e);
+ }
+
+ try {
Constructor<?> constructor =
flinkWriterClass.getDeclaredConstructor(
Sink.class, ProcessingTimeService.class, MailboxExecutor.class);
constructor.setAccessible(true);
return constructor.newInstance(sink, processingTimeService, mailboxExecutor);
+ } catch (NoSuchMethodException e) {
+ // Constructor with 3 parameters not found, try the 4-parameter version
+ try {
+ Constructor<?> constructor =
+ flinkWriterClass.getDeclaredConstructor(
+ StreamOperatorParameters.class,
+ Sink.class,
+ ProcessingTimeService.class,
+ MailboxExecutor.class);
+ constructor.setAccessible(true);
+ return constructor.newInstance(null, sink, processingTimeService, mailboxExecutor);
+ } catch (Exception ex) {
+ throw new RuntimeException("Failed to create SinkWriterOperator in Flink", ex);
+ }
} catch (Exception e) {
+ // Other exceptions (e.g., InvocationTargetException) indicate real failures
throw new RuntimeException("Failed to create SinkWriterOperator in Flink", e);
}
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java
index ae0b48d57..36e348e09 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PostTransformOperator.java
@@ -34,12 +34,12 @@ import org.apache.flink.cdc.common.schema.Selectors;
import org.apache.flink.cdc.common.udf.UserDefinedFunctionContext;
import org.apache.flink.cdc.common.utils.SchemaMergingUtils;
import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.transform.converter.PostTransformConverters;
import org.apache.flink.cdc.runtime.operators.transform.exceptions.TransformException;
import org.apache.flink.cdc.runtime.parser.TransformParser;
import org.apache.flink.cdc.runtime.typeutils.BinaryInternalObjectConverter;
import org.apache.flink.cdc.runtime.typeutils.BinaryRecordDataGenerator;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -64,7 +64,7 @@ import static org.apache.flink.cdc.common.utils.Preconditions.checkNotNull;
* A data process function that performs column filtering, calculated column evaluation & final
* projection.
*/
-public class PostTransformOperator extends AbstractStreamOperator<Event>
+public class PostTransformOperator extends AbstractStreamOperatorAdapter<Event>
implements OneInputStreamOperator<Event, Event>, Serializable {
private static final long serialVersionUID = 1L;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java
index ff6cc512f..03909e3a7 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/PreTransformOperator.java
@@ -33,12 +33,12 @@ import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.schema.Selectors;
import org.apache.flink.cdc.common.utils.Preconditions;
import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.transform.exceptions.TransformException;
import org.apache.flink.cdc.runtime.parser.TransformParser;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateSnapshotContext;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.Output;
@@ -60,7 +60,7 @@ import java.util.stream.Collectors;
/**
* A data process function that filters out columns which aren't (directly & indirectly) referenced.
*/
-public class PreTransformOperator extends AbstractStreamOperator<Event>
+public class PreTransformOperator extends AbstractStreamOperatorAdapter<Event>
implements OneInputStreamOperator<Event, Event>, Serializable {
private static final long serialVersionUID = 1L;
@@ -161,6 +161,12 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
try {
processEvent(event);
} catch (Exception e) {
+ // If this exception originates from a chained downstream operator, rethrow as-is
+ // to avoid misleadingly wrapping it as a "pre-transform" error.
+ if (isChainedOperatorException(e)) {
+ throw e;
+ }
+
TableId tableId = null;
Schema schemaBefore = null;
Schema schemaAfter = null;
@@ -398,4 +404,21 @@ public class PreTransformOperator extends AbstractStreamOperator<Event>
this.transforms = null;
this.preTransformProcessorMap = null;
}
+
+ /**
+ * Checks if the given exception originates from a chained downstream operator. In Flink 2.x,
+ * all operators default to {@code ChainingStrategy.ALWAYS}, which means downstream operators
+ * like PostTransformOperator may be chained with PreTransformOperator. When a chained operator
+ * fails, the exception propagates back through the chain and should not be re-wrapped as a
+ * "pre-transform" error.
+ */
+ private static boolean isChainedOperatorException(Exception e) {
+ try {
+ return Class.forName(
+ "org.apache.flink.streaming.runtime.tasks.ExceptionInChainedOperatorException")
+ .isInstance(e);
+ } catch (ClassNotFoundException ignored) {
+ return false;
+ }
+ }
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/BatchRegularPrePartitionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/BatchRegularPrePartitionOperator.java
index d6142868f..1a2a69798 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/BatchRegularPrePartitionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/BatchRegularPrePartitionOperator.java
@@ -25,10 +25,10 @@ import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.function.HashFunction;
import org.apache.flink.cdc.common.function.HashFunctionProvider;
import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.cdc.runtime.serializer.event.EventSerializer;
import org.apache.flink.runtime.state.StateSnapshotContext;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -44,7 +44,8 @@ import java.util.Optional;
* regular topology in batch mode.
*/
@Internal
-public class BatchRegularPrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
+public class BatchRegularPrePartitionOperator
+ extends AbstractStreamOperatorAdapter<PartitioningEvent>
implements OneInputStreamOperator<Event, PartitioningEvent>, Serializable {
private static final long serialVersionUID = 1L;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java
index bedbe8e8e..4750c685a 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/DistributedPrePartitionOperator.java
@@ -26,10 +26,10 @@ import org.apache.flink.cdc.common.function.HashFunction;
import org.apache.flink.cdc.common.function.HashFunctionProvider;
import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.utils.SchemaUtils;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.cdc.runtime.serializer.event.EventSerializer;
import org.apache.flink.runtime.state.StateSnapshotContext;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -40,7 +40,8 @@ import java.util.Map;
/** Operator for processing events from upstream before flowing to {@link SchemaOperator}. */
@Internal
-public class DistributedPrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
+public class DistributedPrePartitionOperator
+ extends AbstractStreamOperatorAdapter<PartitioningEvent>
implements OneInputStreamOperator<Event, PartitioningEvent>, Serializable {
private static final long serialVersionUID = 1L;
@@ -63,7 +64,7 @@ public class DistributedPrePartitionOperator extends AbstractStreamOperator<Part
@Override
public void open() throws Exception {
super.open();
- subTaskId = getRuntimeContext().getIndexOfThisSubtask();
+ subTaskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
schemaMap = new HashMap<>();
hashFunctionMap = new HashMap<>();
}
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java
index 38f177472..a5d81bfb1 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/partitioning/RegularPrePartitionOperator.java
@@ -26,13 +26,13 @@ import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.function.HashFunction;
import org.apache.flink.cdc.common.function.HashFunctionProvider;
import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.regular.SchemaOperator;
import org.apache.flink.cdc.runtime.operators.sink.SchemaEvolutionClient;
import org.apache.flink.cdc.runtime.serializer.event.EventSerializer;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.runtime.state.StateSnapshotContext;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
@@ -50,7 +50,7 @@ import java.util.Optional;
* regular topology.
*/
@Internal
-public class RegularPrePartitionOperator extends AbstractStreamOperator<PartitioningEvent>
+public class RegularPrePartitionOperator extends AbstractStreamOperatorAdapter<PartitioningEvent>
implements OneInputStreamOperator<Event, PartitioningEvent>, Serializable {
private static final long serialVersionUID = 1L;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/EnumSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/EnumSerializer.java
index 5b93c320d..9987e06aa 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/EnumSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/EnumSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.cdc.common.annotation.VisibleForTesting;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
@@ -176,7 +177,7 @@ public final class EnumSerializer<T extends Enum<T>> extends TypeSerializer<T> {
/** {@link TypeSerializerSnapshot} for {@link EnumSerializer}. */
public static final class EnumSerializerSnapshot<T extends Enum<T>>
- implements TypeSerializerSnapshot<T> {
+ implements TypeSerializerSnapshotAdapter<T> {
private static final int CURRENT_VERSION = 3;
private T[] previousEnums;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NestedSerializersSnapshotDelegate.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NestedSerializersSnapshotDelegate.java
index 66c62630d..c9a053bbf 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NestedSerializersSnapshotDelegate.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NestedSerializersSnapshotDelegate.java
@@ -192,7 +192,7 @@ public class NestedSerializersSnapshotDelegate {
TypeSerializer<E> typedSerializer = (TypeSerializer<E>) serializer;
TypeSerializerSnapshot<E> typedSnapshot = (TypeSerializerSnapshot<E>) snapshot;
- return typedSnapshot.resolveSchemaCompatibility(typedSerializer);
+ return typedSnapshot.resolveSchemaCompatibility(typedSerializer.snapshotConfiguration());
}
private static TypeSerializer<?>[] snapshotsToRestoreSerializers(
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NullableSerializerWrapper.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NullableSerializerWrapper.java
index c230c8c9a..853754b30 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NullableSerializerWrapper.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/NullableSerializerWrapper.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.cdc.common.types.utils.runtime.DataInputViewStream;
import org.apache.flink.cdc.common.types.utils.runtime.DataOutputViewStream;
import org.apache.flink.cdc.common.utils.InstantiationUtil;
@@ -130,7 +131,7 @@ public class NullableSerializerWrapper<T> extends TypeSerializer<T> {
/** Serializer configuration snapshot for compatibility and format evolution. */
@SuppressWarnings("WeakerAccess")
public static final class NullableSerializerWrapperSnapshot<T>
- implements TypeSerializerSnapshot<T> {
+ implements TypeSerializerSnapshotAdapter<T> {
private static final int CURRENT_VERSION = 1;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ArrayDataSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ArrayDataSerializer.java
index 39939375d..d9f6987b6 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ArrayDataSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ArrayDataSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer.data;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
import org.apache.flink.cdc.common.annotation.VisibleForTesting;
@@ -52,12 +53,12 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
private transient BinaryArrayWriter reuseWriter;
public ArrayDataSerializer(DataType eleType) {
- this(eleType, InternalSerializers.create(eleType));
+ this(eleType, new NullableSerializerWrapper<>(InternalSerializers.create(eleType)));
}
private ArrayDataSerializer(DataType eleType, TypeSerializer<Object> eleSer) {
this.eleType = eleType;
- this.eleSer = new NullableSerializerWrapper<>(eleSer);
+ this.eleSer = eleSer;
this.elementGetter = ArrayData.createElementGetter(eleType);
}
@@ -218,7 +219,7 @@ public class ArrayDataSerializer extends TypeSerializer<ArrayData> {
/** {@link TypeSerializerSnapshot} for {@link ArrayDataSerializer}. */
public static final class ArrayDataSerializerSnapshot
- implements TypeSerializerSnapshot<ArrayData> {
+ implements TypeSerializerSnapshotAdapter<ArrayData> {
private static final int CURRENT_VERSION = 3;
private DataType previousType;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/DecimalDataSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/DecimalDataSerializer.java
index 4872330f3..7b799240c 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/DecimalDataSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/DecimalDataSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer.data;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.cdc.common.data.DecimalData;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
@@ -138,7 +139,7 @@ public final class DecimalDataSerializer extends TypeSerializer<DecimalData> {
/** {@link TypeSerializerSnapshot} for {@link DecimalDataSerializer}. */
public static final class DecimalSerializerSnapshot
- implements TypeSerializerSnapshot<DecimalData> {
+ implements TypeSerializerSnapshotAdapter<DecimalData> {
private static final int CURRENT_VERSION = 3;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/LocalZonedTimestampDataSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/LocalZonedTimestampDataSerializer.java
index c3622db23..2743d157d 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/LocalZonedTimestampDataSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/LocalZonedTimestampDataSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer.data;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.cdc.common.data.LocalZonedTimestampData;
import org.apache.flink.cdc.common.data.TimestampData;
import org.apache.flink.core.memory.DataInputView;
@@ -142,7 +143,7 @@ public class LocalZonedTimestampDataSerializer extends TypeSerializer<LocalZoned
/** {@link TypeSerializerSnapshot} for {@link TimestampDataSerializer}. */
public static final class LocalZonedTimestampDataSerializerSnapshot
- implements TypeSerializerSnapshot<LocalZonedTimestampData> {
+ implements TypeSerializerSnapshotAdapter<LocalZonedTimestampData> {
private static final int CURRENT_VERSION = 1;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/MapDataSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/MapDataSerializer.java
index d8f92ee29..0c5050962 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/MapDataSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/MapDataSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer.data;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
import org.apache.flink.cdc.common.annotation.Internal;
@@ -248,7 +249,8 @@ public class MapDataSerializer extends TypeSerializer<MapData> {
}
/** {@link TypeSerializerSnapshot} for {@link MapDataSerializer}. */
- public static final class MapDataSerializerSnapshot implements TypeSerializerSnapshot<MapData> {
+ public static final class MapDataSerializerSnapshot
+ implements TypeSerializerSnapshotAdapter<MapData> {
private static final int CURRENT_VERSION = 0;
private DataType keyType;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/TimestampDataSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/TimestampDataSerializer.java
index babc6345b..25d63f3a7 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/TimestampDataSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/TimestampDataSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer.data;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.cdc.common.data.TimestampData;
import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataOutputView;
@@ -136,7 +137,7 @@ public class TimestampDataSerializer extends TypeSerializer<TimestampData> {
/** {@link TypeSerializerSnapshot} for {@link TimestampDataSerializer}. */
public static final class TimestampDataSerializerSnapshot
- implements TypeSerializerSnapshot<TimestampData> {
+ implements TypeSerializerSnapshotAdapter<TimestampData> {
private static final int CURRENT_VERSION = 1;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ZonedTimestampDataSerializer.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ZonedTimestampDataSerializer.java
index ca9861761..dc833e52a 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ZonedTimestampDataSerializer.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/serializer/data/ZonedTimestampDataSerializer.java
@@ -20,6 +20,7 @@ package org.apache.flink.cdc.runtime.serializer.data;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotAdapter;
import org.apache.flink.cdc.common.data.TimestampData;
import org.apache.flink.cdc.common.data.ZonedTimestampData;
import org.apache.flink.cdc.runtime.serializer.StringSerializer;
@@ -146,7 +147,7 @@ public class ZonedTimestampDataSerializer extends TypeSerializer<ZonedTimestampD
/** {@link TypeSerializerSnapshot} for {@link ZonedTimestampDataSerializer}. */
public static final class ZonedTimestampDataSerializerSnapshot
- implements TypeSerializerSnapshot<ZonedTimestampData> {
+ implements TypeSerializerSnapshotAdapter<ZonedTimestampData> {
private static final int CURRENT_VERSION = 1;
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/EventTypeInfo.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/EventTypeInfo.java
index 8087746cd..c126abca6 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/EventTypeInfo.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/EventTypeInfo.java
@@ -18,6 +18,7 @@
package org.apache.flink.cdc.runtime.typeutils;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.serialization.SerializerConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.cdc.common.event.Event;
@@ -58,11 +59,18 @@ public class EventTypeInfo extends TypeInformation<Event> {
return false;
}
- @Override
+ // Flink 1.x abstract method - not abstract in Flink 2.x so @Override omitted for cross-version
+ // compatibility
public TypeSerializer<Event> createSerializer(ExecutionConfig config) {
return EventSerializer.INSTANCE;
}
+ /** For Flink 2.2+ (TypeInformation.createSerializer now takes SerializerConfig). */
+ @Override
+ public TypeSerializer<Event> createSerializer(SerializerConfig config) {
+ return EventSerializer.INSTANCE;
+ }
+
@Override
public String toString() {
return "Event";
diff --git a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/PartitioningEventTypeInfo.java b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/PartitioningEventTypeInfo.java
index 4a3018fb9..1de112319 100644
--- a/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/PartitioningEventTypeInfo.java
+++ b/flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/typeutils/PartitioningEventTypeInfo.java
@@ -18,6 +18,7 @@
package org.apache.flink.cdc.runtime.typeutils;
import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.serialization.SerializerConfig;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.cdc.common.annotation.Internal;
@@ -58,11 +59,18 @@ public class PartitioningEventTypeInfo extends TypeInformation<PartitioningEvent
return false;
}
- @Override
+ // Flink 1.x abstract method - not abstract in Flink 2.x so @Override omitted for cross-version
+ // compatibility
public TypeSerializer<PartitioningEvent> createSerializer(ExecutionConfig config) {
return PartitioningEventSerializer.INSTANCE;
}
+ /** For Flink 2.2+ (TypeInformation.createSerializer now takes SerializerConfig). */
+ @Override
+ public TypeSerializer<PartitioningEvent> createSerializer(SerializerConfig config) {
+ return PartitioningEventSerializer.INSTANCE;
+ }
+
@Override
public String toString() {
return "PartitioningEvent";
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaEvolveTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaEvolveTest.java
index 1b4de204c..7ece5d20c 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaEvolveTest.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/schema/distributed/SchemaEvolveTest.java
@@ -32,9 +32,9 @@ import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
import org.apache.flink.cdc.common.schema.Column;
import org.apache.flink.cdc.common.schema.Schema;
import org.apache.flink.cdc.common.types.DataTypes;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.common.SchemaTestBase;
import org.apache.flink.cdc.runtime.testutils.operators.DistributedEventOperatorTestHarness;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.util.function.BiConsumerWithException;
@@ -378,7 +378,10 @@ public class SchemaEvolveTest extends SchemaTestBase {
"Unexpected schema change events occurred in EXCEPTION mode. Job will fail now.");
}
- protected static <OP extends AbstractStreamOperator<E>, E extends Event, T extends Throwable>
+ protected static <
+ OP extends AbstractStreamOperatorAdapter<E>,
+ E extends Event,
+ T extends Throwable>
LinkedList<StreamRecord<E>> runInHarness(
Supplier<OP> opCreator,
Function<OP, DistributedEventOperatorTestHarness<OP, E>> harnessCreator,
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkOperatorAdapter.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkOperatorAdapter.java
index 8fd60f632..61c561348 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkOperatorAdapter.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/sink/DataSinkOperatorAdapter.java
@@ -24,11 +24,11 @@ import org.apache.flink.cdc.common.event.FlushEvent;
import org.apache.flink.cdc.common.event.SchemaChangeEventType;
import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.state.StateInitializationContext;
import org.apache.flink.runtime.state.StateSnapshotContext;
import org.apache.flink.streaming.api.graph.StreamConfig;
-import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.BoundedOneInput;
import org.apache.flink.streaming.api.operators.ChainingStrategy;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
@@ -46,7 +46,7 @@ import java.util.Set;
* The DataSinkOperatorAdapter class acts as an adapter for testing the core schema evolution
* process in both {@link DataSinkWriterOperator} and {@link DataSinkFunctionOperator}.
*/
-public class DataSinkOperatorAdapter extends AbstractStreamOperator<Event>
+public class DataSinkOperatorAdapter extends AbstractStreamOperatorAdapter<Event>
implements OneInputStreamOperator<Event, Event>, BoundedOneInput {
private SchemaEvolutionClient schemaEvolutionClient;
@@ -79,7 +79,8 @@ public class DataSinkOperatorAdapter extends AbstractStreamOperator<Event>
@Override
public void initializeState(StateInitializationContext context) throws Exception {
- schemaEvolutionClient.registerSubtask(getRuntimeContext().getIndexOfThisSubtask());
+ schemaEvolutionClient.registerSubtask(
+ getRuntimeContext().getTaskInfo().getIndexOfThisSubtask());
}
@Override
@@ -149,7 +150,8 @@ public class DataSinkOperatorAdapter extends AbstractStreamOperator<Event>
});
}
schemaEvolutionClient.notifyFlushSuccess(
- getRuntimeContext().getIndexOfThisSubtask(), event.getSourceSubTaskId());
+ getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(),
+ event.getSourceSubTaskId());
}
private void emitLatestSchema(TableId tableId) throws Exception {
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/EnumSerializerTest.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/EnumSerializerTest.java
index e101492cd..f18fd0173 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/EnumSerializerTest.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/EnumSerializerTest.java
@@ -131,7 +131,7 @@ class EnumSerializerTest extends TestLogger {
}
TypeSerializerSchemaCompatibility<PublicEnum> compatResult =
- restoredConfig.resolveSchemaCompatibility(serializer);
+ restoredConfig.resolveSchemaCompatibility(serializer.snapshotConfiguration());
Assertions.assertThat(compatResult.isCompatibleAsIs()).isTrue();
Assertions.assertThat(serializer.getValueToOrdinal().get(PublicEnum.FOO).intValue())
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/SerializerTestBase.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/SerializerTestBase.java
index 79566763b..a3ebd9056 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/SerializerTestBase.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/serializer/SerializerTestBase.java
@@ -152,7 +152,7 @@ public abstract class SerializerTestBase<T> {
}
TypeSerializerSchemaCompatibility<T> strategy =
- restoredConfig.resolveSchemaCompatibility(getSerializer());
+ restoredConfig.resolveSchemaCompatibility(getSerializer().snapshotConfiguration());
final TypeSerializer<T> restoreSerializer;
if (strategy.isCompatibleAsIs()) {
restoreSerializer = restoredConfig.restoreSerializer();
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/DistributedEventOperatorTestHarness.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/DistributedEventOperatorTestHarness.java
index 2b9030bc4..cbc1f407d 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/DistributedEventOperatorTestHarness.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/DistributedEventOperatorTestHarness.java
@@ -22,6 +22,7 @@ import org.apache.flink.cdc.common.event.FlushEvent;
import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.common.event.FlushSuccessEvent;
import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaRequest;
import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaResponse;
@@ -31,6 +32,7 @@ import org.apache.flink.cdc.runtime.operators.sink.SchemaEvolutionClient;
import org.apache.flink.cdc.runtime.testutils.schema.CollectingMetadataApplier;
import org.apache.flink.cdc.runtime.testutils.schema.TestingSchemaRegistryGateway;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.event.WatermarkEvent;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
@@ -66,7 +68,7 @@ import static org.apache.flink.cdc.runtime.operators.schema.common.CoordinationR
* @param <E> Type of the event emitted by the operator
*/
public class DistributedEventOperatorTestHarness<
- OP extends AbstractStreamOperator<E>, E extends Event>
+ OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
implements AutoCloseable {
public static final OperatorID SCHEMA_OPERATOR_ID = new OperatorID(15213L, 15513L);
@@ -196,6 +198,10 @@ public class DistributedEventOperatorTestHarness<
throw new UnsupportedOperationException();
}
+ public void emitWatermark(WatermarkEvent mark) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {
throw new UnsupportedOperationException();
diff --git a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/RegularEventOperatorTestHarness.java b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/RegularEventOperatorTestHarness.java
index ea4de7a14..8a3a21ba6 100644
--- a/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/RegularEventOperatorTestHarness.java
+++ b/flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/RegularEventOperatorTestHarness.java
@@ -26,6 +26,7 @@ import org.apache.flink.cdc.common.event.TableId;
import org.apache.flink.cdc.common.pipeline.PipelineOptions;
import org.apache.flink.cdc.common.pipeline.SchemaChangeBehavior;
import org.apache.flink.cdc.common.schema.Schema;
+import org.apache.flink.cdc.runtime.operators.AbstractStreamOperatorAdapter;
import org.apache.flink.cdc.runtime.operators.schema.common.CoordinationResponseUtils;
import org.apache.flink.cdc.runtime.operators.schema.common.event.FlushSuccessEvent;
import org.apache.flink.cdc.runtime.operators.schema.common.event.GetEvolvedSchemaRequest;
@@ -40,6 +41,7 @@ import org.apache.flink.cdc.runtime.operators.sink.SchemaEvolutionClient;
import org.apache.flink.cdc.runtime.testutils.schema.CollectingMetadataApplier;
import org.apache.flink.cdc.runtime.testutils.schema.TestingSchemaRegistryGateway;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.event.WatermarkEvent;
import org.apache.flink.runtime.jobgraph.OperatorID;
import org.apache.flink.runtime.jobgraph.tasks.TaskOperatorEventGateway;
import org.apache.flink.runtime.operators.testutils.DummyEnvironment;
@@ -82,7 +84,8 @@ import static org.apache.flink.cdc.runtime.operators.schema.common.CoordinationR
* @param <OP> Type of the operator
* @param <E> Type of the event emitted by the operator
*/
-public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E>, E extends Event>
+public class RegularEventOperatorTestHarness<
+ OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
implements AutoCloseable {
private static final Logger LOG =
LoggerFactory.getLogger(RegularEventOperatorTestHarness.class);
@@ -126,7 +129,7 @@ public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E
schemaRegistryGateway = new TestingSchemaRegistryGateway(schemaRegistry);
}
- public static <OP extends AbstractStreamOperator<E>, E extends Event>
+ public static <OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
RegularEventOperatorTestHarness<OP, E> with(OP operator, int numOutputs) {
return new RegularEventOperatorTestHarness<>(
operator,
@@ -138,7 +141,7 @@ public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E
Collections.emptySet());
}
- public static <OP extends AbstractStreamOperator<E>, E extends Event>
+ public static <OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
RegularEventOperatorTestHarness<OP, E> withDuration(
OP operator, int numOutputs, Duration evolveDuration) {
return new RegularEventOperatorTestHarness<>(
@@ -151,7 +154,7 @@ public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E
Collections.emptySet());
}
- public static <OP extends AbstractStreamOperator<E>, E extends Event>
+ public static <OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
RegularEventOperatorTestHarness<OP, E> withDurationAndBehavior(
OP operator,
int numOutputs,
@@ -167,7 +170,7 @@ public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E
Collections.emptySet());
}
- public static <OP extends AbstractStreamOperator<E>, E extends Event>
+ public static <OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
RegularEventOperatorTestHarness<OP, E> withDurationAndFineGrainedBehavior(
OP operator,
int numOutputs,
@@ -184,7 +187,7 @@ public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E
Collections.emptySet());
}
- public static <OP extends AbstractStreamOperator<E>, E extends Event>
+ public static <OP extends AbstractStreamOperatorAdapter<E>, E extends Event>
RegularEventOperatorTestHarness<OP, E> withDurationAndFineGrainedBehaviorWithError(
OP operator,
int numOutputs,
@@ -327,6 +330,10 @@ public class RegularEventOperatorTestHarness<OP extends AbstractStreamOperator<E
throw new UnsupportedOperationException();
}
+ public void emitWatermark(WatermarkEvent mark) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public void emitWatermarkStatus(WatermarkStatus watermarkStatus) {
throw new UnsupportedOperationException();
diff --git a/pom.xml b/pom.xml
index 3d87b6905..53220c9ef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,6 +33,8 @@ limitations under the License.
<packaging>pom</packaging>
<modules>
+ <module>flink-cdc-flink1-compat</module>
+ <module>flink-cdc-flink2-compat</module>
<module>flink-cdc-cli</module>
<module>flink-cdc-common</module>
<module>flink-cdc-composer</module>
@@ -67,8 +69,13 @@ limitations under the License.
<flink.reuseForks>true</flink.reuseForks>
<!-- dependencies versions -->
- <flink.version>1.20.3</flink.version>
+ <flink.1.x.version>1.20.3</flink.1.x.version>
+ <flink.2.x.version>2.2.0</flink.2.x.version>
+ <flink.version>${flink.1.x.version}</flink.version>
<flink.major.version>1.20</flink.major.version>
+ <flink.1.x.shaded.guava.version>31.1-jre-17.0</flink.1.x.shaded.guava.version>
+ <flink.2.x.shaded.guava.version>33.4.0-jre-20.0</flink.2.x.shaded.guava.version>
+ <shaded.guava.version>${flink.1.x.shaded.guava.version}</shaded.guava.version>
<flink.shaded.version>17.0</flink.shaded.version>
<debezium.version>1.9.8.Final</debezium.version>
<tikv.version>3.2.0</tikv.version>
@@ -289,7 +296,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-guava</artifactId>
- <version>31.1-jre-${flink.shaded.version}</version>
+ <version>${shaded.guava.version}</version>
</dependency>
<dependency>
@@ -842,6 +849,47 @@ limitations under the License.
</plugins>
</build>
</profile>
+ <profile>
+ <id>flink2</id>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-flink2-guava</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-shaded-guava</artifactId>
+ <version>${flink.2.x.shaded.guava.version}</version>
+ <outputDirectory>${project.build.directory}/flink2-extra-libs</outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <additionalClasspathElements>
+ <additionalClasspathElement>${project.build.directory}/flink2-extra-libs/flink-shaded-guava-${flink.2.x.shaded.guava.version}.jar</additionalClasspathElement>
+ </additionalClasspathElements>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
<profile>
<id>java-17-target</id>
<activation>