You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by go...@apache.org on 2025/06/26 06:08:13 UTC
(flink-cdc) branch master updated: [hotfix][ci] Add missing Iceberg unit tests & split pipeline connector UT into stages (#4046)
This is an automated email from the ASF dual-hosted git repository.
gongzhongqiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
The following commit(s) were added to refs/heads/master by this push:
new 23a1c2efb [hotfix][ci] Add missing Iceberg unit tests & split pipeline connector UT into stages (#4046)
23a1c2efb is described below
commit 23a1c2efb6fa9ce1c9f17b3836f6aaa995bb0660
Author: yuxiqian <34...@users.noreply.github.com>
AuthorDate: Thu Jun 26 14:08:09 2025 +0800
[hotfix][ci] Add missing Iceberg unit tests & split pipeline connector UT into stages (#4046)
---
.github/workflows/flink_cdc_base.yml | 112 +++++++++++++++++++++--------
.github/workflows/flink_cdc_ci.yml | 36 ++++++----
.github/workflows/flink_cdc_ci_nightly.yml | 40 +++++++----
3 files changed, 130 insertions(+), 58 deletions(-)
diff --git a/.github/workflows/flink_cdc_base.yml b/.github/workflows/flink_cdc_base.yml
index 00c3475f6..1b0cdabb2 100644
--- a/.github/workflows/flink_cdc_base.yml
+++ b/.github/workflows/flink_cdc_base.yml
@@ -18,18 +18,18 @@ name: Flink CDC Base Workflow
on:
workflow_call:
inputs:
- java-version:
+ java-versions:
description: "Jdk versions to test against."
required: false
type: string
default: "['8']"
- flink-version:
+ flink-versions:
description: "Flink versions to test against."
required: false
type: string
default: "['generic']"
- module:
- description: "Flink CDC module to test against."
+ modules:
+ description: "Flink CDC modules to test against."
required: true
type: string
parallelism:
@@ -48,23 +48,28 @@ env:
flink-cdc-common,\
flink-cdc-composer,\
flink-cdc-runtime,\
- flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base"
+ flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
MODULES_PIPELINE_CONNECTORS: "\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase,\
- flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values"
- MODULES_MYSQL: "\
+ MODULES_MYSQL_SOURCE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"
+ MODULES_MYSQL_PIPELINE: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql"
+
MODULES_POSTGRES: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"
@@ -85,10 +90,13 @@ env:
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"
- MODULES_OCEANBASE: "\
+ MODULES_OCEANBASE_SOURCE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"
+ MODULES_OCEANBASE_PIPELINE: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-oceanbase"
+
MODULES_DB2: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"
@@ -97,6 +105,27 @@ env:
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"
+ MODULES_DORIS: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris"
+
+ MODULES_STARROCKS: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks"
+
+ MODULES_ICEBERG: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-iceberg"
+
+ MODULES_KAFKA: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka"
+
+ MODULES_PAIMON: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"
+
+ MODULES_ELASTICSEARCH: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-elasticsearch"
+
+ MODULES_MAXCOMPUTE: "\
+ flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-maxcompute"
+
MODULES_PIPELINE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"
@@ -104,14 +133,15 @@ env:
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"
jobs:
- compile_and_test:
+ test:
runs-on: ubuntu-latest
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
- java-version: ${{ fromJSON(inputs.java-version) }}
- flink-version: ${{ fromJSON(inputs.flink-version) }}
+ java-version: ${{ fromJSON(inputs.java-versions) }}
+ flink-version: ${{ fromJSON(inputs.flink-versions) }}
+ module: ${{ fromJSON(inputs.modules) }}
steps:
- run: echo "Running CI pipeline for JDK version ${{ matrix.java-version }}"
- name: Clean up disk space
@@ -149,22 +179,22 @@ jobs:
maven-version: 3.8.6
- name: Compile and test
- timeout-minutes: 60
+ timeout-minutes: 90
run: |
. .github/workflows/utils.sh
jvm_timezone=$(random_timezone)
echo "JVM timezone is set to $jvm_timezone"
set -o pipefail
- case ${{ inputs.module }} in
+ case ${{ matrix.module }} in
("core")
modules=${{ env.MODULES_CORE }}
;;
- ("pipeline_connectors")
- modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
+ ("mysql-source")
+ modules=${{ env.MODULES_MYSQL_SOURCE }}
;;
- ("mysql")
- modules=${{ env.MODULES_MYSQL }}
+ ("mysql-pipeline")
+ modules=${{ env.MODULES_MYSQL_PIPELINE }}
;;
("postgres")
modules=${{ env.MODULES_POSTGRES }}
@@ -184,8 +214,11 @@ jobs:
("tidb")
modules=${{ env.MODULES_TIDB }}
;;
- ("oceanbase")
- modules=${{ env.MODULES_OCEANBASE }}
+ ("oceanbase-source")
+ modules=${{ env.MODULES_OCEANBASE_SOURCE }}
+ ;;
+ ("oceanbase-pipeline")
+ modules=${{ env.MODULES_OCEANBASE_PIPELINE }}
;;
("db2")
modules=${{ env.MODULES_DB2 }}
@@ -193,25 +226,46 @@ jobs:
("vitess")
modules=${{ env.MODULES_VITESS }}
;;
+ ("doris")
+ modules=${{ env.MODULES_DORIS }}
+ ;;
+ ("elasticsearch")
+ modules=${{ env.MODULES_ELASTICSEARCH }}
+ ;;
+ ("iceberg")
+ modules=${{ env.MODULES_ICEBERG }}
+ ;;
+ ("kafka")
+ modules=${{ env.MODULES_KAFKA }}
+ ;;
+ ("maxcompute")
+ modules=${{ env.MODULES_MAXCOMPUTE }}
+ ;;
+ ("paimon")
+ modules=${{ env.MODULES_PAIMON }}
+ ;;
+ ("starrocks")
+ modules=${{ env.MODULES_STARROCKS }}
+ ;;
("pipeline_e2e")
- compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
+ compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
modules=${{ env.MODULES_PIPELINE_E2E }}
;;
("source_e2e")
- compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
+ compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL_SOURCE }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE_SOURCE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
modules=${{ env.MODULES_SOURCE_E2E }}
;;
esac
- if [ ${{ inputs.module }} != "pipeline_e2e" ] && [ ${{ inputs.module }} != "source_e2e" ]; then
+ if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
compile_modules=$modules
fi
build_maven_parameter=""
- if [ ${{ inputs.module }} == "mongodb6" ]; then
+ if [ ${{ matrix.module }} == "mongodb6" ]; then
build_maven_parameter="-DspecifiedMongoVersion=6.0.16"
- elif [ ${{ inputs.module }} == "mongodb7" ]; then
+ elif [ ${{ matrix.module }} == "mongodb7" ]; then
build_maven_parameter="-DspecifiedMongoVersion=7.0.12"
fi
diff --git a/.github/workflows/flink_cdc_ci.yml b/.github/workflows/flink_cdc_ci.yml
index 789313b1f..d4a7fefcc 100644
--- a/.github/workflows/flink_cdc_ci.yml
+++ b/.github/workflows/flink_cdc_ci.yml
@@ -57,16 +57,24 @@ jobs:
run: mvn --no-snapshot-updates -B package -DskipTests
- name: Run license check
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
- ut:
- strategy:
- fail-fast: false
- matrix:
- module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
- name: Unit Tests
+ common:
+ name: Common Unit Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[8]"
+ modules: "['core']"
+ pipeline-ut:
+ name: Pipeline Unit Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[8]"
+ modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks']"
+ source-ut:
+ name: Source Unit Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
- java-version: "[8]"
- module: ${{ matrix.module }}
+ java-versions: "[8]"
+ modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
pipeline_e2e:
strategy:
fail-fast: false
@@ -75,14 +83,14 @@ jobs:
name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
uses: ./.github/workflows/flink_cdc_base.yml
with:
- java-version: "[8]"
- flink-version: "['1.19.2', '1.20.1']"
- module: pipeline_e2e
+ java-versions: "[8]"
+ flink-versions: "['1.19.2', '1.20.1']"
+ modules: "['pipeline_e2e']"
parallelism: ${{ matrix.parallelism }}
source_e2e:
name: Source E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
- java-version: "[8]"
- flink-version: "['1.19.2', '1.20.1']"
- module: source_e2e
+ java-versions: "[8]"
+ flink-versions: "['1.19.2', '1.20.1']"
+ modules: "['source_e2e']"
diff --git a/.github/workflows/flink_cdc_ci_nightly.yml b/.github/workflows/flink_cdc_ci_nightly.yml
index 5f54ecd1a..c17058413 100644
--- a/.github/workflows/flink_cdc_ci_nightly.yml
+++ b/.github/workflows/flink_cdc_ci_nightly.yml
@@ -47,35 +47,45 @@ jobs:
run: mvn --no-snapshot-updates -B package -DskipTests
- name: Run license check
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb
- ut:
+ common:
if: github.repository == 'apache/flink-cdc'
- strategy:
- fail-fast: false
- matrix:
- module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
- name: Unit Tests
+ name: Common Unit Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[11]"
+ modules: "['core']"
+ pipeline-ut:
+ if: github.repository == 'apache/flink-cdc'
+ name: Pipeline Unit Tests
+ uses: ./.github/workflows/flink_cdc_base.yml
+ with:
+ java-versions: "[11]"
+ modules: "['mysql-pipeline', 'oceanbase-pipeline', 'doris', 'elasticsearch', 'iceberg', 'kafka', 'maxcompute', 'paimon', 'starrocks']"
+ source-ut:
+ if: github.repository == 'apache/flink-cdc'
+ name: Source Unit Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
- java-version: "[11]"
- module: ${{ matrix.module }}
+ java-versions: "[11]"
+ modules: "['mysql-source', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase-source', 'db2', 'vitess']"
pipeline_e2e:
if: github.repository == 'apache/flink-cdc'
strategy:
fail-fast: false
matrix:
parallelism: [ 1, 4 ]
- name: Pipeline E2E Tests (${{ matrix.parallelism }} Parallelism)
+ name: Pipeline E2E Tests (${{ matrix.parallelism }}-Parallelism)
uses: ./.github/workflows/flink_cdc_base.yml
with:
- java-version: "[11]"
- flink-version: "['1.19.2', '1.20.1']"
- module: pipeline_e2e
+ java-versions: "[11]"
+ flink-versions: "['1.19.2', '1.20.1']"
+ modules: "['pipeline_e2e']"
parallelism: ${{ matrix.parallelism }}
source_e2e:
if: github.repository == 'apache/flink-cdc'
name: Source E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
- java-version: "[11]"
- flink-version: "['1.19.2', '1.20.1']"
- module: source_e2e
+ java-versions: "[11]"
+ flink-versions: "['1.19.2', '1.20.1']"
+ modules: "['source_e2e']"