You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by go...@apache.org on 2025/06/26 06:08:13 UTC

(flink-cdc) branch master updated: [hotfix][ci] Add missing Iceberg unit tests & split pipeline connector UT into stages (#4046)

This is an automated email from the ASF dual-hosted git repository.

gongzhongqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git


The following commit(s) were added to refs/heads/master by this push:
     new 23a1c2efb [hotfix][ci] Add missing Iceberg unit tests & split pipeline connector UT into stages (#4046)
23a1c2efb is described below

commit 23a1c2efb6fa9ce1c9f17b3836f6aaa995bb0660
Author: yuxiqian <34...@users.noreply.github.com>
AuthorDate: Thu Jun 26 14:08:09 2025 +0800

    [hotfix][ci] Add missing Iceberg unit tests & split pipeline connector UT into stages (#4046)
---
 .github/workflows/flink_cdc_base.yml       | 112 +++++++++++++++++++++--------
 .github/workflows/flink_cdc_ci.yml         |  36 ++++++----
 .github/workflows/flink_cdc_ci_nightly.yml |  40 +++++++----
 3 files changed, 130 insertions(+), 58 deletions(-)

diff --git a/.github/workflows/flink_cdc_base.yml b/.github/workflows/flink_cdc_base.yml
index 00c3475f6..1b0cdabb2 100644
--- a/.github/workflows/flink_cdc_base.yml
+++ b/.github/workflows/flink_cdc_base.yml
@@ -18,18 +18,18 @@ name: Flink CDC Base Workflow
 on:
   workflow_call:
     inputs:
-      java-version:
+      java-versions:
         description: "Jdk versions to test against."
         required: false
         type: string
         default: "['8']"
-      flink-version:
+      flink-versions:
         description: "Flink versions to test against."
         required: false
         type: string
         default: "['generic']"
-      module:
-        description: "Flink CDC module to test against."
+      modules:
+        description: "Flink CDC modules to test against."
         required: true
         type: string
       parallelism:
@@ -48,23 +48,28 @@ env:
   flink-cdc-common,\
   flink-cdc-composer,\
   flink-cdc-runtime,\
-  flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base"
+  flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
 
   MODULES_PIPELINE_CONNECTORS: "\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
   flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
-  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
 
-  MODULES_MYSQL: "\
+  MODULES_MYSQL_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
 
+  MODULES_MYSQL_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
+
   MODULES_POSTGRES: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
@@ -85,10 +90,13 @@ env:
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
 
-  MODULES_OCEANBASE: "\
+  MODULES_OCEANBASE_SOURCE: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
 
+  MODULES_OCEANBASE_PIPELINE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
+
   MODULES_DB2: "\
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
@@ -97,6 +105,27 @@ env:
   flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
   flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
 
+  MODULES_DORIS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris"
+
+  MODULES_STARROCKS: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks"
+
+  MODULES_ICEBERG: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg"
+
+  MODULES_KAFKA: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka"
+
+  MODULES_PAIMON: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
+
+  MODULES_ELASTICSEARCH: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
+
+  MODULES_MAXCOMPUTE: "\
+  flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+
   MODULES_PIPELINE_E2E: "\
   flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
 
@@ -104,14 +133,15 @@ env:
   flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
 
 jobs:
-  compile_and_test:
+  test:
     runs-on: ubuntu-latest
     timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
-        java-version: ${{ fromJSON(inputs.java-version) }}
-        flink-version: ${{ fromJSON(inputs.flink-version) }}
+        java-version: ${{ fromJSON(inputs.java-versions) }}
+        flink-version: ${{ fromJSON(inputs.flink-versions) }}
+        module: ${{ fromJSON(inputs.modules) }}
     steps:
       - run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
       - name: Clean up disk space
@@ -149,22 +179,22 @@ jobs:
           maven-version: 3.8.6
 
       - name: Compile and test
-        timeout-minutes: 60
+        timeout-minutes: 90
         run: |
           . .github/workflows/utils.sh
           jvm_timezone=$(random_timezone)
           echo "JVM timezone is set to $jvm_timezone"
           set -o pipefail
 
-          case ${{ inputs.module }} in
+          case ${{ matrix.module }} in
               ("core")
                modules=${{ env.MODULES_CORE }}
               ;;
-              ("pipeline_connectors")
-               modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
+              ("mysql-source")
+               modules=${{ env.MODULES_MYSQL_SOURCE }}
               ;;
-              ("mysql")
-               modules=${{ env.MODULES_MYSQL }}
+              ("mysql-pipeline")
+               modules=${{ env.MODULES_MYSQL_PIPELINE }}
               ;;
               ("postgres")
                modules=${{ env.MODULES_POSTGRES }}
@@ -184,8 +214,11 @@ jobs:
               ("tidb")
                modules=${{ env.MODULES_TIDB }}
               ;;
-              ("oceanbase")
-               modules=${{ env.MODULES_OCEANBASE }}
+              ("oceanbase-source")
+               modules=${{ env.MODULES_OCEANBASE_SOURCE }}
+              ;;
+              ("oceanbase-pipeline")
+               modules=${{ env.MODULES_OCEANBASE_PIPELINE }}
               ;;
               ("db2")
                modules=${{ env.MODULES_DB2 }}
@@ -193,25 +226,46 @@ jobs:
               ("vitess")
                modules=${{ env.MODULES_VITESS }}
               ;;
+              ("doris")
+               modules=${{ env.MODULES_DORIS }}
+              ;;
+              ("elasticsearch")
+               modules=${{ env.MODULES_ELASTICSEARCH }}
+              ;;
+              ("iceberg")
+               modules=${{ env.MODULES_ICEBERG }}
+              ;;
+              ("kafka")
+               modules=${{ env.MODULES_KAFKA }}
+              ;;
+              ("maxcompute")
+               modules=${{ env.MODULES_MAXCOMPUTE }}
+              ;;
+              ("paimon")
+               modules=${{ env.MODULES_PAIMON }}
+              ;;
+              ("starrocks")
+               modules=${{ env.MODULES_STARROCKS }}
+              ;;
               ("pipeline_e2e")
-               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
                modules=${{ env.MODULES_PIPELINE_E2E }}
               ;;
               ("source_e2e")
-               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
+               compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
                modules=${{ env.MODULES_SOURCE_E2E }}
               ;;
           esac
           
-          if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
+          if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
             compile_modules=$modules
           fi
           
           build_maven_parameter=""
 
-          if [ ${{ inputs.module }} == "mongodb6" ]; then
+          if [ ${{ matrix.module }} == "mongodb6" ]; then
             build_maven_parameter="-DspecifiedMongoVersion=6.0.16"
-          elif [ ${{ inputs.module }} == "mongodb7" ]; then
+          elif [ ${{ matrix.module }} == "mongodb7" ]; then
             build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
           fi
           
diff --git a/.github/workflows/flink_cdc_ci.yml b/.github/workflows/flink_cdc_ci.yml
index 789313b1f..d4a7fefcc 100644
--- a/.github/workflows/flink_cdc_ci.yml
+++ b/.github/workflows/flink_cdc_ci.yml
@@ -57,16 +57,24 @@ jobs:
         run: mvn --no-snapshot-updates -B package -DskipTests
       - name: Run license check
         run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
-  ut:
-    strategy:
-      fail-fast: false
-      matrix:
-        module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
-    name: Unit Tests
+  common:
+    name: Common Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-versions: "[8]"
+      modules: "['core']"
+  pipeline-ut:
+    name: Pipeline Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-versions: "[8]"
+      modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks']"
+  source-ut:
+    name: Source Unit Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[8]"
-      module: ${{ matrix.module }}
+      java-versions: "[8]"
+      modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
   pipeline_e2e:
     strategy:
       fail-fast: false
@@ -75,14 +83,14 @@ jobs:
     name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[8]"
-      flink-version: "['1.19.2', '1.20.1']"
-      module: pipeline_e2e
+      java-versions: "[8]"
+      flink-versions: "['1.19.2', '1.20.1']"
+      modules: "['pipeline_e2e']"
       parallelism: ${{ matrix.parallelism }}
   source_e2e:
     name: Source E2E Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[8]"
-      flink-version: "['1.19.2', '1.20.1']"
-      module: source_e2e
+      java-versions: "[8]"
+      flink-versions: "['1.19.2', '1.20.1']"
+      modules: "['source_e2e']"
diff --git a/.github/workflows/flink_cdc_ci_nightly.yml b/.github/workflows/flink_cdc_ci_nightly.yml
index 5f54ecd1a..c17058413 100644
--- a/.github/workflows/flink_cdc_ci_nightly.yml
+++ b/.github/workflows/flink_cdc_ci_nightly.yml
@@ -47,35 +47,45 @@ jobs:
         run: mvn --no-snapshot-updates -B package -DskipTests
       - name: Run license check
         run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
-  ut:
+  common:
     if: github.repository == 'apache/flink-cdc'
-    strategy:
-      fail-fast: false
-      matrix:
-        module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
-    name: Unit Tests
+    name: Common Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-versions: "[11]"
+      modules: "['core']"
+  pipeline-ut:
+    if: github.repository == 'apache/flink-cdc'
+    name: Pipeline Unit Tests
+    uses: ./.github/workflows/flink_cdc_base.yml
+    with:
+      java-versions: "[11]"
+      modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks']"
+  source-ut:
+    if: github.repository == 'apache/flink-cdc'
+    name: Source Unit Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[11]"
-      module: ${{ matrix.module }}
+      java-versions: "[11]"
+      modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
   pipeline_e2e:
     if: github.repository == 'apache/flink-cdc'
     strategy:
       fail-fast: false
       matrix:
         parallelism: [ 1, 4 ]
-    name: Pipeline E2E Tests (${{ matrix.parallelism }} Parallelism)
+    name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[11]"
-      flink-version: "['1.19.2', '1.20.1']"
-      module: pipeline_e2e
+      java-versions: "[11]"
+      flink-versions: "['1.19.2', '1.20.1']"
+      modules: "['pipeline_e2e']"
       parallelism: ${{ matrix.parallelism }}
   source_e2e:
     if: github.repository == 'apache/flink-cdc'
     name: Source E2E Tests
     uses: ./.github/workflows/flink_cdc_base.yml
     with:
-      java-version: "[11]"
-      flink-version: "['1.19.2', '1.20.1']"
-      module: source_e2e
+      java-versions: "[11]"
+      flink-versions: "['1.19.2', '1.20.1']"
+      modules: "['source_e2e']"