You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by go...@apache.org on 2024/10/15 13:55:02 UTC
(flink-cdc) branch master updated (a1781f432 -> e8251310e)
This is an automated email from the ASF dual-hosted git repository.
gongzhongqiang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
from a1781f432 [FLINK-36151][docs] Add schema evolution related docs
new daf27fa7a [hotfix][test] Reorganize test cases
new e8251310e [hotfix][ci] Optimize CI performance by implementing parallel execution
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/flink_cdc.yml | 349 ++-------------------
.../{flink_cdc.yml => flink_cdc_base.yml} | 265 ++++------------
.github/workflows/flink_cdc_migration_test.yml | 114 +++++++
.../mongodb/source/MongoDBFullChangelogITCase.java | 2 +-
.../source/MongoDBParallelSourceExampleTest.java | 2 +-
.../source/MongoDBParallelSourceITCase.java | 2 +-
.../mongodb/source/MongoDBSourceTestBase.java | 9 +-
.../mongodb/source/NewlyAddedTableITCase.java | 2 +-
.../reader/MongoDBSnapshotSplitReaderTest.java | 2 +-
.../reader/MongoDBStreamSplitReaderTest.java | 2 +-
.../mongodb/table/MongoDBTimeZoneITCase.java | 2 +-
.../tests/utils/PipelineTestEnvironment.java | 8 +-
.../flink/cdc/connectors/tests/MongoE2eITCase.java | 16 +-
.../tests/utils/FlinkContainerTestEnvironment.java | 8 +-
tools/mig-test/misc/patch_flink_conf.rb | 6 +-
tools/mig-test/run_migration_test.rb | 10 +-
16 files changed, 263 insertions(+), 536 deletions(-)
copy .github/workflows/{flink_cdc.yml => flink_cdc_base.yml} (51%)
create mode 100644 .github/workflows/flink_cdc_migration_test.yml
(flink-cdc) 01/02: [hotfix][test] Reorganize test cases
Posted by go...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gongzhongqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
commit daf27fa7a8165c1a851d38285e70eb1fcb8b62af
Author: yuxiqian <34...@users.noreply.github.com>
AuthorDate: Fri Oct 11 23:42:07 2024 +0800
[hotfix][test] Reorganize test cases
---
.../mongodb/source/MongoDBFullChangelogITCase.java | 2 +-
.../mongodb/source/MongoDBParallelSourceExampleTest.java | 2 +-
.../mongodb/source/MongoDBParallelSourceITCase.java | 2 +-
.../connectors/mongodb/source/MongoDBSourceTestBase.java | 9 ++++++++-
.../connectors/mongodb/source/NewlyAddedTableITCase.java | 2 +-
.../source/reader/MongoDBSnapshotSplitReaderTest.java | 2 +-
.../source/reader/MongoDBStreamSplitReaderTest.java | 2 +-
.../connectors/mongodb/table/MongoDBTimeZoneITCase.java | 2 +-
.../pipeline/tests/utils/PipelineTestEnvironment.java | 8 +++++++-
.../flink/cdc/connectors/tests/MongoE2eITCase.java | 16 ++++++++++++----
.../tests/utils/FlinkContainerTestEnvironment.java | 8 +++++++-
tools/mig-test/misc/patch_flink_conf.rb | 6 +++++-
tools/mig-test/run_migration_test.rb | 10 ++++++++--
13 files changed, 54 insertions(+), 17 deletions(-)
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBFullChangelogITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBFullChangelogITCase.java
index 8d8047fa7..eebaacafc 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBFullChangelogITCase.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBFullChangelogITCase.java
@@ -90,7 +90,7 @@ public class MongoDBFullChangelogITCase extends MongoDBSourceTestBase {
@Parameterized.Parameters(name = "mongoVersion: {0} parallelismSnapshot: {1}")
public static Object[] parameters() {
List<Object[]> parameterTuples = new ArrayList<>();
- for (String mongoVersion : MONGO_VERSIONS) {
+ for (String mongoVersion : getMongoVersions()) {
parameterTuples.add(new Object[] {mongoVersion, true});
parameterTuples.add(new Object[] {mongoVersion, false});
}
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceExampleTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceExampleTest.java
index a285fe4a3..350f96fb5 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceExampleTest.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceExampleTest.java
@@ -41,7 +41,7 @@ public class MongoDBParallelSourceExampleTest extends MongoDBSourceTestBase {
@Parameterized.Parameters(name = "mongoVersion: {0} parallelismSnapshot: {1}")
public static Object[] parameters() {
List<Object[]> parameterTuples = new ArrayList<>();
- for (String mongoVersion : MONGO_VERSIONS) {
+ for (String mongoVersion : getMongoVersions()) {
parameterTuples.add(new Object[] {mongoVersion, true});
parameterTuples.add(new Object[] {mongoVersion, false});
}
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceITCase.java
index 0595f6b5a..461bf76af 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceITCase.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBParallelSourceITCase.java
@@ -80,7 +80,7 @@ public class MongoDBParallelSourceITCase extends MongoDBSourceTestBase {
@Parameterized.Parameters(name = "mongoVersion: {0}")
public static Object[] parameters() {
- return Stream.of(MONGO_VERSIONS).map(e -> new Object[] {e}).toArray();
+ return Stream.of(getMongoVersions()).map(e -> new Object[] {e}).toArray();
}
@Rule public final Timeout timeoutPerTest = Timeout.seconds(300);
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBSourceTestBase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBSourceTestBase.java
index 49a362969..3da217b8b 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBSourceTestBase.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/MongoDBSourceTestBase.java
@@ -45,7 +45,14 @@ public class MongoDBSourceTestBase {
.withLogConsumer(new Slf4jLogConsumer(LOG));
}
- public static final String[] MONGO_VERSIONS = {"6.0.16", "7.0.12"};
+ public static String[] getMongoVersions() {
+ String specifiedMongoVersion = System.getProperty("specifiedMongoVersion");
+ if (specifiedMongoVersion != null) {
+ return new String[] {specifiedMongoVersion};
+ } else {
+ return new String[] {"6.0.16", "7.0.12"};
+ }
+ }
protected static final int DEFAULT_PARALLELISM = 4;
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/NewlyAddedTableITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/NewlyAddedTableITCase.java
index 69900b4e9..1a5943492 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/NewlyAddedTableITCase.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/NewlyAddedTableITCase.java
@@ -79,7 +79,7 @@ public class NewlyAddedTableITCase extends MongoDBSourceTestBase {
@Parameterized.Parameters(name = "mongoVersion: {0}")
public static Object[] parameters() {
- return Stream.of(MONGO_VERSIONS).map(e -> new Object[] {e}).toArray();
+ return Stream.of(getMongoVersions()).map(e -> new Object[] {e}).toArray();
}
private final ScheduledExecutorService mockChangelogExecutor =
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBSnapshotSplitReaderTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBSnapshotSplitReaderTest.java
index b683f303d..8634f96cf 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBSnapshotSplitReaderTest.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBSnapshotSplitReaderTest.java
@@ -81,7 +81,7 @@ public class MongoDBSnapshotSplitReaderTest extends MongoDBSourceTestBase {
@Parameterized.Parameters(name = "mongoVersion: {0}")
public static Object[] parameters() {
- return Stream.of(MONGO_VERSIONS).map(e -> new Object[] {e}).toArray();
+ return Stream.of(getMongoVersions()).map(e -> new Object[] {e}).toArray();
}
@Before
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBStreamSplitReaderTest.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBStreamSplitReaderTest.java
index bf781904b..f71dc6307 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBStreamSplitReaderTest.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/source/reader/MongoDBStreamSplitReaderTest.java
@@ -95,7 +95,7 @@ public class MongoDBStreamSplitReaderTest extends MongoDBSourceTestBase {
@Parameterized.Parameters(name = "mongoVersion: {0}")
public static Object[] parameters() {
- return Stream.of(MONGO_VERSIONS).map(e -> new Object[] {e}).toArray();
+ return Stream.of(getMongoVersions()).map(e -> new Object[] {e}).toArray();
}
@Before
diff --git a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/table/MongoDBTimeZoneITCase.java b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/table/MongoDBTimeZoneITCase.java
index f9edc73fa..96ab145a9 100644
--- a/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/table/MongoDBTimeZoneITCase.java
+++ b/flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc/src/test/java/org/apache/flink/cdc/connectors/mongodb/table/MongoDBTimeZoneITCase.java
@@ -66,7 +66,7 @@ public class MongoDBTimeZoneITCase extends MongoDBSourceTestBase {
name = "mongoVersion: {0}, localTimeZone: {1}, parallelismSnapshot: {2}")
public static Object[] parameters() {
List<Object[]> parameterTuples = new ArrayList<>();
- for (String mongoVersion : MONGO_VERSIONS) {
+ for (String mongoVersion : getMongoVersions()) {
for (String timezone : new String[] {"Asia/Shanghai", "Europe/Berlin", "UTC"}) {
for (boolean parallelismSnapshot : new boolean[] {true, false}) {
parameterTuples.add(new Object[] {mongoVersion, timezone, parallelismSnapshot});
diff --git a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
index 048e23bd7..093359642 100644
--- a/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
+++ b/flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/utils/PipelineTestEnvironment.java
@@ -53,6 +53,7 @@ import java.nio.file.Path;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -90,7 +91,12 @@ public abstract class PipelineTestEnvironment extends TestLogger {
@Parameterized.Parameters(name = "flinkVersion: {0}")
public static List<String> getFlinkVersion() {
- return Arrays.asList("1.17.2", "1.18.1", "1.19.1", "1.20.0");
+ String flinkVersion = System.getProperty("specifiedFlinkVersion");
+ if (flinkVersion != null) {
+ return Collections.singletonList(flinkVersion);
+ } else {
+ return Arrays.asList("1.17.2", "1.18.1", "1.19.1", "1.20.0");
+ }
}
@Before
diff --git a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/MongoE2eITCase.java b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/MongoE2eITCase.java
index e9cf648f9..352e3e28a 100644
--- a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/MongoE2eITCase.java
+++ b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/MongoE2eITCase.java
@@ -44,6 +44,7 @@ import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.stream.Stream;
@@ -66,8 +67,6 @@ public class MongoE2eITCase extends FlinkContainerTestEnvironment {
private MongoClient mongoClient;
- public static final String[] MONGO_VERSIONS = {"6.0.16", "7.0.12"};
-
@Parameterized.Parameter(1)
public String mongoVersion;
@@ -77,6 +76,15 @@ public class MongoE2eITCase extends FlinkContainerTestEnvironment {
@Parameterized.Parameter(3)
public boolean scanFullChangelog;
+ public static List<String> getMongoVersions() {
+ String specifiedMongoVersion = System.getProperty("specifiedMongoVersion");
+ if (specifiedMongoVersion != null) {
+ return Collections.singletonList(specifiedMongoVersion);
+ } else {
+ return Arrays.asList("6.0.16", "7.0.12");
+ }
+ }
+
@Parameterized.Parameters(
name =
"flinkVersion: {0}, mongoVersion: {1}, parallelismSnapshot: {2}, scanFullChangelog: {3}")
@@ -84,7 +92,7 @@ public class MongoE2eITCase extends FlinkContainerTestEnvironment {
final List<String> flinkVersions = getFlinkVersion();
List<Object[]> params = new ArrayList<>();
for (String flinkVersion : flinkVersions) {
- for (String mongoVersion : MONGO_VERSIONS) {
+ for (String mongoVersion : getMongoVersions()) {
params.add(new Object[] {flinkVersion, mongoVersion, true, true});
params.add(new Object[] {flinkVersion, mongoVersion, true, false});
params.add(new Object[] {flinkVersion, mongoVersion, false, true});
@@ -99,7 +107,7 @@ public class MongoE2eITCase extends FlinkContainerTestEnvironment {
super.before();
container =
- new MongoDBContainer("mongo:6.0.9")
+ new MongoDBContainer("mongo:" + mongoVersion)
.withSharding()
.withNetwork(NETWORK)
.withNetworkAliases(INTER_CONTAINER_MONGO_ALIAS)
diff --git a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
index 49ef039e0..77aabc2df 100644
--- a/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
+++ b/flink-cdc-e2e-tests/flink-cdc-source-e2e-tests/src/test/java/org/apache/flink/cdc/connectors/tests/utils/FlinkContainerTestEnvironment.java
@@ -61,6 +61,7 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -120,7 +121,12 @@ public abstract class FlinkContainerTestEnvironment extends TestLogger {
@Parameterized.Parameters(name = "flinkVersion: {0}")
public static List<String> getFlinkVersion() {
- return Arrays.asList("1.16.3", "1.17.2", "1.18.1", "1.19.1", "1.20.0");
+ String flinkVersion = System.getProperty("specifiedFlinkVersion");
+ if (flinkVersion != null) {
+ return Collections.singletonList(flinkVersion);
+ } else {
+ return Arrays.asList("1.16.3", "1.17.2", "1.18.1", "1.19.1", "1.20.0");
+ }
}
@Before
diff --git a/tools/mig-test/misc/patch_flink_conf.rb b/tools/mig-test/misc/patch_flink_conf.rb
index fe6030188..6241a52c3 100644
--- a/tools/mig-test/misc/patch_flink_conf.rb
+++ b/tools/mig-test/misc/patch_flink_conf.rb
@@ -26,7 +26,11 @@ parallelism.default: 4
execution.checkpointing.interval: 300
EXTRACONF
-File.write("#{FLINK_HOME}/conf/flink-conf.yaml", EXTRA_CONF, mode: 'a+')
+if File.file?("#{FLINK_HOME}/conf/flink-conf.yaml")
+ File.write("#{FLINK_HOME}/conf/flink-conf.yaml", EXTRA_CONF, mode: 'a+')
+else
+ File.write("#{FLINK_HOME}/conf/config.yaml", EXTRA_CONF, mode: 'a+')
+end
# MySQL connector is not provided
`wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar -O #{FLINK_HOME}/lib/mysql-connector-java-8.0.27.jar`
\ No newline at end of file
diff --git a/tools/mig-test/run_migration_test.rb b/tools/mig-test/run_migration_test.rb
index d7c2fff28..1a454efe8 100644
--- a/tools/mig-test/run_migration_test.rb
+++ b/tools/mig-test/run_migration_test.rb
@@ -114,7 +114,13 @@ def test_migration(from_version, to_version)
end
end
-version_list = %w[3.0.0 3.0.1 3.1.0 3.1.1 3.3-SNAPSHOT]
+version_list = case ARGV[0]
+ when '1.18.1' then %w[3.0.0 3.0.1 3.1.1 3.3-SNAPSHOT]
+ when '1.19.1' then %w[3.1.1 3.3-SNAPSHOT]
+ when '1.20.0' then %w[3.3-SNAPSHOT]
+ else []
+ end
+
no_savepoint_versions = %w[3.0.0 3.0.1]
version_result = Hash.new('❓')
@failures = []
@@ -157,6 +163,6 @@ rescue LoadError
end
puts "✅ - Compatible, ❌ - Not compatible, ❓ - Target version doesn't support `--from-savepoint`"
-if @failures.filter { |old_version, new_version| new_version == version_list.last && old_version != '3.1.0' }.any?
+if @failures.any?
abort 'Some migration to snapshot version tests failed.'
end
(flink-cdc) 02/02: [hotfix][ci] Optimize CI performance by implementing parallel execution
Posted by go...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gongzhongqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
commit e8251310edbf01626ced5147994ceba4d7b98e4c
Author: gongzhongqiang <76...@qq.com>
AuthorDate: Fri Oct 11 23:44:33 2024 +0800
[hotfix][ci] Optimize CI performance by implementing parallel execution
---
.github/workflows/flink_cdc.yml | 349 ++-------------------
.../{flink_cdc.yml => flink_cdc_base.yml} | 265 ++++------------
.github/workflows/flink_cdc_migration_test.yml | 114 +++++++
3 files changed, 209 insertions(+), 519 deletions(-)
diff --git a/.github/workflows/flink_cdc.yml b/.github/workflows/flink_cdc.yml
index ac0dfdc82..a58452c96 100644
--- a/.github/workflows/flink_cdc.yml
+++ b/.github/workflows/flink_cdc.yml
@@ -30,78 +30,13 @@ on:
- 'docs/**'
- 'README.md'
-# Concurrency strategy:
-# github.workflow: distinguish this workflow from others
-# github.event_name: distinguish `push` event from `pull_request` event
-# github.event.number: set to the number of the pull request if `pull_request` event
-# github.run_id: otherwise, it's a `push` or `schedule` event, only cancel if we rerun the workflow
-#
-# Reference:
-# https://docs.github.com/en/actions/using-jobs/using-concurrency
-# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true
-env:
- MODULES_CORE: "\
- flink-cdc-cli,\
- flink-cdc-common,\
- flink-cdc-composer,\
- flink-cdc-runtime"
-
- MODULES_PIPELINE_CONNECTORS: "\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
-
- MODULES_MYSQL: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
-
- MODULES_POSTGRES: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
-
- MODULES_ORACLE: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
-
- MODULES_MONGODB: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"
-
- MODULES_SQLSERVER: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"
-
- MODULES_TIDB: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
-
- MODULES_OCEANBASE: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
-
- MODULES_DB2: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
-
- MODULES_VITESS: "\
- flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
-
- MODULES_PIPELINE_E2E: "\
- flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
-
- MODULES_SOURCE_E2E: "\
- flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
-
jobs:
license_check:
+ name: License Check
runs-on: ubuntu-latest
steps:
- name: Check out repository code
@@ -116,260 +51,40 @@ jobs:
run: mvn --no-snapshot-updates -B package -DskipTests
- name: Run license check
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
-
- compile_and_test:
- needs: license_check
- # Only run the CI pipeline for the flink-cdc-connectors repository
-# if: github.repository == 'apache/flink-cdc-connectors'
- runs-on: ubuntu-latest
+ ut:
strategy:
matrix:
- jdk: [ 8 ]
- module: [ "core",
- "pipeline_connectors",
- "mysql",
- "postgres",
- "oracle",
- "mongodb",
- "sqlserver",
- "tidb",
- "oceanbase",
- "db2",
- "vitess",
- "pipeline_e2e",
- "source_e2e"
- ]
- timeout-minutes: 120
- env:
- MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
- -Dorg.slf4j.simpleLogger.showDateTime=true \
- -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
- --no-snapshot-updates -B \
- --settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \
- -Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
- -Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
- steps:
- - run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"
-
- - name: Clean up disk space
- run: |
- set -euo pipefail
-
- echo "Disk space before cleanup"
- df -h
-
- echo "Cleaning up disk space"
- sudo rm -rf /usr/share/dotnet
- sudo rm -rf /usr/local/lib/android
- sudo rm -rf /opt/ghc
- sudo rm -rf /opt/hostedtoolcache/CodeQL
- sudo docker image prune --all --force
-
- echo "Disk space after cleanup"
- df -h
-
- - name: Check out repository code
- uses: actions/checkout@v4
- with:
- submodules: true
-
- - name: Set JDK
- uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.jdk }}
- distribution: 'temurin'
- cache: 'maven'
-
- - name: Set Maven 3.8.6
- uses: stCarolas/setup-maven@v5
- with:
- maven-version: 3.8.6
-
- - name: Compile and test ${{ matrix.module }}
- timeout-minutes: 90
- run: |
- set -o pipefail
-
- case ${{ matrix.module }} in
- ("core")
- modules=${{ env.MODULES_CORE }}
- ;;
- ("pipeline_connectors")
- modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
- ;;
- ("mysql")
- modules=${{ env.MODULES_MYSQL }}
- ;;
- ("postgres")
- modules=${{ env.MODULES_POSTGRES }}
- ;;
- ("oracle")
- modules=${{ env.MODULES_ORACLE }}
- ;;
- ("mongodb")
- modules=${{ env.MODULES_MONGODB }}
- ;;
- ("sqlserver")
- modules=${{ env.MODULES_SQLSERVER }}
- ;;
- ("tidb")
- modules=${{ env.MODULES_TIDB }}
- ;;
- ("oceanbase")
- modules=${{ env.MODULES_OCEANBASE }}
- ;;
- ("db2")
- modules=${{ env.MODULES_DB2 }}
- ;;
- ("vitess")
- modules=${{ env.MODULES_VITESS }}
- ;;
- ("pipeline_e2e")
- compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
- modules=${{ env.MODULES_PIPELINE_E2E }}
- ;;
- ("source_e2e")
- compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
- modules=${{ env.MODULES_SOURCE_E2E }}
- ;;
- esac
-
- if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
- compile_modules=$modules
- fi
-
- mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify
-
- - name: Print JVM thread dumps when cancelled
- if: ${{ failure() }}
- run: |
- # ----------------------------------------------------------------------------
- # Copyright 2023 The Netty Project
- #
- # ----------------------------------------------------------------------------
- # Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
- echo "$OSTYPE"
- if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
- echo "Setting up JVM thread dumps"
- # use jattach so that Java processes in docker containers are also covered
- # download jattach
- curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
- if command -v sha256sum &> /dev/null; then
- # verify hash of jattach binary
- sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1
- fi
- chmod +x /tmp/jattach
- for java_pid in $(sudo pgrep java); do
- echo "----------------------- pid $java_pid -----------------------"
- echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)"
- sudo /tmp/jattach $java_pid jcmd VM.command_line || true
- sudo /tmp/jattach $java_pid jcmd "Thread.print -l"
- sudo /tmp/jattach $java_pid jcmd GC.heap_info || true
- done
- else
- for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
- echo "----------------------- pid $java_pid -----------------------"
- jcmd $java_pid VM.command_line || true
- jcmd $java_pid Thread.print -l
- jcmd $java_pid GC.heap_info || true
- done
- fi
- exit 0
-
-
- migration_test_ut:
- needs: license_check
- runs-on: ubuntu-latest
- steps:
- - name: Check out repository code
- uses: actions/checkout@v4
- with:
- submodules: true
- - name: Compile snapshot CDC version
- run: mvn --no-snapshot-updates -B install -DskipTests
- - name: Run migration tests
- run: cd flink-cdc-migration-tests && mvn clean verify
-
- pipeline_migration_test:
- needs: migration_test_ut
- runs-on: ubuntu-latest
+ java-version: [ '8' ]
+ module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
+ name: Unit Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-version: ${{ matrix.java-version }}
+ module: ${{ matrix.module }}
+ pipeline_e2e:
strategy:
matrix:
- java-version: [ '8', '11' ]
-
- steps:
- - uses: actions/checkout@v4
- - name: Set up Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: 3.0
- bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- - uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: temurin
- cache: maven
- - name: Install dependencies
- run: gem install terminal-table
- - name: Prepare CDC versions
- run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- - name: Prepare Flink distro
- run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
- working-directory: ./tools/mig-test
- - name: Patch Flink configs
- run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
- working-directory: ./tools/mig-test
- - name: Start containers
- run: cd conf && docker compose up -d
- working-directory: ./tools/mig-test
- - name: Run migration tests
- run: FLINK_HOME=./flink-1.18.1/ ruby run_migration_test.rb
- working-directory: ./tools/mig-test
- - name: Stop containers
- if: always()
- run: cd conf && docker compose down
- working-directory: ./tools/mig-test
-
- data_stream_migration_test:
- needs: migration_test_ut
- runs-on: ubuntu-latest
+ java-version: [ '8' ]
+ flink-version: ['1.17.2', '1.18.1', '1.19.1', '1.20.0']
+ module: [ 'pipeline_e2e' ]
+ name: Pipeline E2E Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-version: ${{ matrix.java-version }}
+ flink-version: ${{ matrix.flink-version }}
+ module: ${{ matrix.module }}
+ source_e2e:
strategy:
matrix:
- java-version: [ '8', '11' ]
-
- steps:
- - uses: actions/checkout@v4
- - name: Set up Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: 3.0
- bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- - uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: temurin
- cache: maven
- - name: Install dependencies
- run: gem install terminal-table
- - name: Prepare CDC versions
- run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- - name: Prepare Flink distro
- run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
- working-directory: ./tools/mig-test
- - name: Patch Flink configs
- run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
- working-directory: ./tools/mig-test
- - name: Compile Dummy DataStream Jobs
- run: cd datastream && ruby compile_jobs.rb
- working-directory: ./tools/mig-test
- - name: Start containers
- run: cd conf && docker compose up -d
- working-directory: ./tools/mig-test
- - name: Run migration tests
- run: cd datastream && FLINK_HOME=../flink-1.18.1/ ruby run_migration_test.rb
- working-directory: ./tools/mig-test
- - name: Stop containers
- if: always()
- run: cd conf && docker compose down
- working-directory: ./tools/mig-test
+ java-version: [ '8' ]
+ flink-version: ['1.16.3', '1.17.2', '1.18.1', '1.19.1', '1.20.0']
+ module: [ 'source_e2e' ]
+ name: Source E2E Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-version: ${{ matrix.java-version }}
+ flink-version: ${{ matrix.flink-version }}
+ module: ${{ matrix.module }}
+ migration_test:
+ name: Migration Tests
+ uses: ./.github/workflows/flink_cdc_migration_test.yml
\ No newline at end of file
diff --git a/.github/workflows/flink_cdc.yml b/.github/workflows/flink_cdc_base.yml
similarity index 51%
copy from .github/workflows/flink_cdc.yml
copy to .github/workflows/flink_cdc_base.yml
index ac0dfdc82..74260ef14 100644
--- a/.github/workflows/flink_cdc.yml
+++ b/.github/workflows/flink_cdc_base.yml
@@ -13,35 +13,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-name: Flink CDC CI
-on:
- push:
- branches:
- - master
- - release-*
- paths-ignore:
- - 'docs/**'
- - 'README.md'
- pull_request:
- branches:
- - master
- - release-*
- paths-ignore:
- - 'docs/**'
- - 'README.md'
+name: Flink CDC Base Workflow
-# Concurrency strategy:
-# github.workflow: distinguish this workflow from others
-# github.event_name: distinguish `push` event from `pull_request` event
-# github.event.number: set to the number of the pull request if `pull_request` event
-# github.run_id: otherwise, it's a `push` or `schedule` event, only cancel if we rerun the workflow
-#
-# Reference:
-# https://docs.github.com/en/actions/using-jobs/using-concurrency
-# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
-concurrency:
- group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
- cancel-in-progress: true
+on:
+ workflow_call:
+ inputs:
+ java-version:
+ description: "Jdk version to test against."
+ required: false
+ type: string
+ default: "8"
+ flink-version:
+ description: "Flink version to test against."
+ required: false
+ type: string
+ module:
+ description: "Flink CDC module to test against."
+ required: true
+ type: string
+ custom-maven-parameter:
+ description: "Custom maven parameter."
+ required: false
+ type: string
env:
MODULES_CORE: "\
@@ -49,51 +42,52 @@ env:
flink-cdc-common,\
flink-cdc-composer,\
flink-cdc-runtime"
-
+
MODULES_PIPELINE_CONNECTORS: "\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
MODULES_MYSQL: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
-
+
MODULES_POSTGRES: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
-
+
MODULES_ORACLE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"
-
+
MODULES_MONGODB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"
-
+
MODULES_SQLSERVER: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"
-
+
MODULES_TIDB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
-
+
MODULES_OCEANBASE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
-
+
MODULES_DB2: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
-
+
MODULES_VITESS: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
-
+
MODULES_PIPELINE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
@@ -101,71 +95,26 @@ env:
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
jobs:
- license_check:
- runs-on: ubuntu-latest
- steps:
- - name: Check out repository code
- uses: actions/checkout@v4
- with:
- submodules: true
- - name: Set up Ruby environment
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: '3.3'
- - name: Compiling jar packages
- run: mvn --no-snapshot-updates -B package -DskipTests
- - name: Run license check
- run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
-
compile_and_test:
- needs: license_check
- # Only run the CI pipeline for the flink-cdc-connectors repository
-# if: github.repository == 'apache/flink-cdc-connectors'
runs-on: ubuntu-latest
- strategy:
- matrix:
- jdk: [ 8 ]
- module: [ "core",
- "pipeline_connectors",
- "mysql",
- "postgres",
- "oracle",
- "mongodb",
- "sqlserver",
- "tidb",
- "oceanbase",
- "db2",
- "vitess",
- "pipeline_e2e",
- "source_e2e"
- ]
timeout-minutes: 120
- env:
- MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
- -Dorg.slf4j.simpleLogger.showDateTime=true \
- -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \
- -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
- --no-snapshot-updates -B \
- --settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \
- -Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
- -Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
steps:
- - run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"
+ - run: echo "Running CI pipeline for JDK version ${{ inputs.java-version }}"
- name: Clean up disk space
run: |
set -euo pipefail
-
+
echo "Disk space before cleanup"
df -h
-
+
echo "Cleaning up disk space"
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
-
+
echo "Disk space after cleanup"
df -h
@@ -177,7 +126,7 @@ jobs:
- name: Set JDK
uses: actions/setup-java@v4
with:
- java-version: ${{ matrix.jdk }}
+ java-version: ${{ inputs.java-version }}
distribution: 'temurin'
cache: 'maven'
@@ -186,12 +135,12 @@ jobs:
with:
maven-version: 3.8.6
- - name: Compile and test ${{ matrix.module }}
- timeout-minutes: 90
+ - name: Compile and test
+ timeout-minutes: 60
run: |
set -o pipefail
-
- case ${{ matrix.module }} in
+
+ case ${{ inputs.module }} in
("core")
modules=${{ env.MODULES_CORE }}
;;
@@ -207,7 +156,10 @@ jobs:
("oracle")
modules=${{ env.MODULES_ORACLE }}
;;
- ("mongodb")
+ ("mongodb6")
+ modules=${{ env.MODULES_MONGODB }}
+ ;;
+ ("mongodb7")
modules=${{ env.MODULES_MONGODB }}
;;
("sqlserver")
@@ -234,29 +186,35 @@ jobs:
modules=${{ env.MODULES_SOURCE_E2E }}
;;
esac
-
- if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
+
+ if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
compile_modules=$modules
fi
+
+ build_maven_parameter=""
- mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify
+ if [ ${{ inputs.module }} == "mongodb6" ]; then
+ build_maven_parameter="-DspecifiedMongoVersion=6.0.16"
+ elif [ ${{ inputs.module }} == "mongodb7" ]; then
+ build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
+ fi
+
+ if [ ! -z "${{ inputs.flink-version }}" ]; then
+ build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }-DspecifiedFlinkVersion=${{ inputs.flink-version }}"
+ fi
+
+ build_maven_parameter="${build_maven_parameter:+$build_maven_parameter }${{ inputs.custom-maven-parameter }}"
+
+ mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B $build_maven_parameter -pl $modules verify
- name: Print JVM thread dumps when cancelled
if: ${{ failure() }}
run: |
- # ----------------------------------------------------------------------------
- # Copyright 2023 The Netty Project
- #
- # ----------------------------------------------------------------------------
- # Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
echo "$OSTYPE"
if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
echo "Setting up JVM thread dumps"
- # use jattach so that Java processes in docker containers are also covered
- # download jattach
curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
if command -v sha256sum &> /dev/null; then
- # verify hash of jattach binary
sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1
fi
chmod +x /tmp/jattach
@@ -276,100 +234,3 @@ jobs:
done
fi
exit 0
-
-
- migration_test_ut:
- needs: license_check
- runs-on: ubuntu-latest
- steps:
- - name: Check out repository code
- uses: actions/checkout@v4
- with:
- submodules: true
- - name: Compile snapshot CDC version
- run: mvn --no-snapshot-updates -B install -DskipTests
- - name: Run migration tests
- run: cd flink-cdc-migration-tests && mvn clean verify
-
- pipeline_migration_test:
- needs: migration_test_ut
- runs-on: ubuntu-latest
- strategy:
- matrix:
- java-version: [ '8', '11' ]
-
- steps:
- - uses: actions/checkout@v4
- - name: Set up Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: 3.0
- bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- - uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: temurin
- cache: maven
- - name: Install dependencies
- run: gem install terminal-table
- - name: Prepare CDC versions
- run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- - name: Prepare Flink distro
- run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
- working-directory: ./tools/mig-test
- - name: Patch Flink configs
- run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
- working-directory: ./tools/mig-test
- - name: Start containers
- run: cd conf && docker compose up -d
- working-directory: ./tools/mig-test
- - name: Run migration tests
- run: FLINK_HOME=./flink-1.18.1/ ruby run_migration_test.rb
- working-directory: ./tools/mig-test
- - name: Stop containers
- if: always()
- run: cd conf && docker compose down
- working-directory: ./tools/mig-test
-
- data_stream_migration_test:
- needs: migration_test_ut
- runs-on: ubuntu-latest
- strategy:
- matrix:
- java-version: [ '8', '11' ]
-
- steps:
- - uses: actions/checkout@v4
- - name: Set up Ruby
- uses: ruby/setup-ruby@v1
- with:
- ruby-version: 3.0
- bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- - uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: temurin
- cache: maven
- - name: Install dependencies
- run: gem install terminal-table
- - name: Prepare CDC versions
- run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- - name: Prepare Flink distro
- run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
- working-directory: ./tools/mig-test
- - name: Patch Flink configs
- run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
- working-directory: ./tools/mig-test
- - name: Compile Dummy DataStream Jobs
- run: cd datastream && ruby compile_jobs.rb
- working-directory: ./tools/mig-test
- - name: Start containers
- run: cd conf && docker compose up -d
- working-directory: ./tools/mig-test
- - name: Run migration tests
- run: cd datastream && FLINK_HOME=../flink-1.18.1/ ruby run_migration_test.rb
- working-directory: ./tools/mig-test
- - name: Stop containers
- if: always()
- run: cd conf && docker compose down
- working-directory: ./tools/mig-test
diff --git a/.github/workflows/flink_cdc_migration_test.yml b/.github/workflows/flink_cdc_migration_test.yml
new file mode 100644
index 000000000..38baec7f6
--- /dev/null
+++ b/.github/workflows/flink_cdc_migration_test.yml
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Migration Tests
+
+on:
+ workflow_call:
+
+jobs:
+ migration_test_ut:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check out repository code
+ uses: actions/checkout@v4
+ with:
+ submodules: true
+ - name: Compile snapshot CDC version
+ run: mvn --no-snapshot-updates -B install -DskipTests
+ - name: Run migration tests
+ run: cd flink-cdc-migration-tests && mvn clean verify
+
+ pipeline_migration_test:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ # '1.20.0' is excluded since FLINK-36105 has not been merged.
+ flink-version: [ '1.18.1', '1.19.1' ]
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Ruby
+ uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: 3.0
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+ - uses: actions/setup-java@v4
+ with:
+ java-version: 8
+ distribution: temurin
+ cache: maven
+ - name: Install dependencies
+ run: gem install terminal-table
+ - name: Prepare CDC versions
+ run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
+ - name: Prepare Flink distro
+ run: wget https://dlcdn.apache.org/flink/flink-${{ matrix.flink-version }}/flink-${{ matrix.flink-version }}-bin-scala_2.12.tgz && tar -xzvf flink-${{ matrix.flink-version }}-bin-scala_2.12.tgz
+ working-directory: ./tools/mig-test
+ - name: Patch Flink configs
+ run: FLINK_HOME=./flink-${{ matrix.flink-version }}/ ruby misc/patch_flink_conf.rb
+ working-directory: ./tools/mig-test
+ - name: Start containers
+ run: cd conf && docker compose up -d
+ working-directory: ./tools/mig-test
+ - name: Run migration tests
+ run: FLINK_HOME=./flink-${{ matrix.flink-version }}/ ruby run_migration_test.rb ${{ matrix.flink-version }}
+ working-directory: ./tools/mig-test
+ - name: Stop containers
+ if: always()
+ run: cd conf && docker compose down
+ working-directory: ./tools/mig-test
+
+ data_stream_migration_test:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ flink-version: [ '1.18.1', '1.19.1', '1.20.0' ]
+
+ steps:
+ - uses: actions/checkout@v4
+ - name: Set up Ruby
+ uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: 3.0
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+ - uses: actions/setup-java@v4
+ with:
+ java-version: 8
+ distribution: temurin
+ cache: maven
+ - name: Install dependencies
+ run: gem install terminal-table
+ - name: Prepare CDC versions
+ run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
+ - name: Prepare Flink distro
+ run: wget https://dlcdn.apache.org/flink/flink-${{ matrix.flink-version }}/flink-${{ matrix.flink-version }}-bin-scala_2.12.tgz && tar -xzvf flink-${{ matrix.flink-version }}-bin-scala_2.12.tgz
+ working-directory: ./tools/mig-test
+ - name: Patch Flink configs
+ run: FLINK_HOME=./flink-${{ matrix.flink-version }}/ ruby misc/patch_flink_conf.rb
+ working-directory: ./tools/mig-test
+ - name: Compile Dummy DataStream Jobs
+ run: cd datastream && ruby compile_jobs.rb
+ working-directory: ./tools/mig-test
+ - name: Start containers
+ run: cd conf && docker compose up -d
+ working-directory: ./tools/mig-test
+ - name: Run migration tests
+ run: cd datastream && FLINK_HOME=../flink-${{ matrix.flink-version }}/ ruby run_migration_test.rb
+ working-directory: ./tools/mig-test
+ - name: Stop containers
+ if: always()
+ run: cd conf && docker compose down
+ working-directory: ./tools/mig-test