You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ab...@apache.org on 2026/10/17 22:53:25 UTC
(fluss) branch main updated: [build] Extract Spark CI into a separate workflow (#4654)
This is an automated email from the ASF dual-hosted git repository.
fresh-borzoni pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new bbf2f28d2 [build] Extract Spark CI into a separate workflow (#4654)
bbf2f28d2 is described below
commit bbf2f28d2c2d59a844f5bc14aced7436fc0d947e
Author: Yang Zhang <ya...@alibaba-inc.com>
AuthorDate: Sun Oct 18 06:53:20 2026 +0800
[build] Extract Spark CI into a separate workflow (#4654)
* [build] Run only Spark CI for Spark-only pull requests
Co-Authored-By: Codex <no...@openai.com>
AI-Model: gpt-6
AI-Contributed/Feature: 49/49
AI-Contributed/UT: 0/0
* [test] Verify Spark-only CI job selection
Co-Authored-By: Codex <no...@openai.com>
AI-Model: gpt-6
AI-Contributed/Feature: 1/1
AI-Contributed/UT: 0/0
* [test] Verify full CI for mixed Spark and client changes
Co-Authored-By: Codex <no...@openai.com>
AI-Model: gpt-6
AI-Contributed/Feature: 1/1
AI-Contributed/UT: 0/0
* [test] Remove temporary CI trigger verification comments
Co-Authored-By: Codex <no...@openai.com>
AI-Model: gpt-6
AI-Contributed/Feature: 2/2
AI-Contributed/UT: 0/0
* [build] Extract Spark CI into a separate workflow
Co-Authored-By: Codex <no...@openai.com>
AI-Model: gpt-6
AI-Contributed/Feature: 137/137
AI-Contributed/UT: 0/0
* [build] Include Spark CI in unstable test reporting
Co-Authored-By: Codex <no...@openai.com>
AI-Model: gpt-6
AI-Contributed/Feature: 2/2
AI-Contributed/UT: 0/0
---------
Co-authored-by: Codex <no...@openai.com>
---
.github/workflows/ci-spark.yaml | 93 ++++++++++++++++++++++++++++++++++++++
.github/workflows/ci-template.yaml | 16 +++----
.github/workflows/ci.yaml | 5 +-
tools/ci/unstable_test_reporter.py | 2 +-
4 files changed, 105 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/ci-spark.yaml b/.github/workflows/ci-spark.yaml
new file mode 100644
index 000000000..471661d84
--- /dev/null
+++ b/.github/workflows/ci-spark.yaml
@@ -0,0 +1,93 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+name: Spark Connector CI
+
+permissions:
+ contents: read
+
+on:
+ push:
+ branches:
+ - main
+ - release-**
+ - ci-**
+ paths:
+ - 'fluss-spark/**'
+ - 'fluss-client/**'
+ - 'fluss-common/**'
+ - 'fluss-rpc/**'
+ - 'fluss-server/**'
+ - 'fluss-test-utils/**'
+ - 'fluss-metrics/**'
+ - 'fluss-protogen/**'
+ - 'fluss-lake/**'
+ - 'fluss-flink/**'
+ - 'fluss-filesystems/**'
+ - 'fluss-test-coverage/**'
+ - 'fluss-docgen/**'
+ - 'pom.xml'
+ - '.mvn/**'
+ - 'mvnw'
+ - 'tools/maven/**'
+ - 'tools/ci/**'
+ - '.scalafmt.conf'
+ - 'copyright.txt'
+ - '.github/workflows/ci-spark.yaml'
+ - '.github/workflows/ci-template.yaml'
+ - '.github/workflows/stage.sh'
+ - '!**/*.md'
+
+ pull_request:
+ paths:
+ - 'fluss-spark/**'
+ - 'fluss-client/**'
+ - 'fluss-common/**'
+ - 'fluss-rpc/**'
+ - 'fluss-server/**'
+ - 'fluss-test-utils/**'
+ - 'fluss-metrics/**'
+ - 'fluss-protogen/**'
+ - 'fluss-lake/**'
+ - 'fluss-flink/**'
+ - 'fluss-filesystems/**'
+ - 'fluss-test-coverage/**'
+ - 'fluss-docgen/**'
+ - 'pom.xml'
+ - '.mvn/**'
+ - 'mvnw'
+ - 'tools/maven/**'
+ - 'tools/ci/**'
+ - '.scalafmt.conf'
+ - 'copyright.txt'
+ - '.github/workflows/ci-spark.yaml'
+ - '.github/workflows/ci-template.yaml'
+ - '.github/workflows/stage.sh'
+ - '!**/*.md'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
+ cancel-in-progress: true
+
+jobs:
+ build-on-jdk11:
+ name: "Java 11"
+ uses: ./.github/workflows/ci-template.yaml
+ with:
+ java-version: "11"
+ modules: '["spark3","spark3-lake","spark3-scala213"]'
diff --git a/.github/workflows/ci-template.yaml b/.github/workflows/ci-template.yaml
index c02406a33..2583e0f88 100644
--- a/.github/workflows/ci-template.yaml
+++ b/.github/workflows/ci-template.yaml
@@ -30,6 +30,11 @@ on:
required: false
type: string
default: ""
+ modules:
+ description: "JSON array of CI stages to run."
+ required: false
+ type: string
+ default: '["core","flink1","flink2","spark3","spark3-lake","spark3-scala213","lake"]'
run-spark-scala213:
description: "Whether to run the Spark tests with Scala 2.13."
required: false
@@ -41,15 +46,8 @@ jobs:
strategy:
fail-fast: false
matrix:
- module: [ core, flink1, flink2, spark3, spark3-lake, spark3-scala213, lake ]
+ module: ${{ fromJSON(inputs.modules) }}
exclude: ${{ fromJSON(inputs.run-spark-scala213 && '[]' || '[{"module":"spark3-scala213"}]') }}
- include:
- - module: flink1
- test-profile: "test-flink1"
- - module: flink2
- test-profile: "test-flink2"
- - module: spark3-lake
- test-profile: "test-spark3"
name: "${{ matrix.module }}"
steps:
- name: Checkout code
@@ -90,7 +88,7 @@ jobs:
TEST_MODULES=$(./.github/workflows/stage.sh ${{ matrix.module }})
echo "github ref: ${{ github.ref }}"
echo "Start testing modules: $TEST_MODULES"
- mvn -B verify $TEST_MODULES $MAVEN_TOOLCHAIN_ARGS -Ptest-coverage -P${{ matrix.module == 'spark3-scala213' && 'test-spark3' || matrix.test-profile || format('test-{0}', matrix.module) }} -Dlog.dir=${{ runner.temp }}/fluss-logs -Dlog4j.configurationFile=${{ github.workspace }}/tools/ci/log4j.properties ${{ inputs.maven-parameters }}
+ mvn -B verify $TEST_MODULES $MAVEN_TOOLCHAIN_ARGS -Ptest-coverage -P${{ startsWith(matrix.module, 'spark3') && 'test-spark3' || format('test-{0}', matrix.module) }} -Dlog.dir=${{ runner.temp }}/fluss-logs -Dlog4j.configurationFile=${{ github.workspace }}/tools/ci/log4j.properties ${{ inputs.maven-parameters }}
env:
MAVEN_OPTS: -Xmx4096m
ARTIFACTS_OSS_ENDPOINT: ${{ secrets.ARTIFACTS_OSS_ENDPOINT }}
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 4cc357388..d7ff54f13 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -27,6 +27,7 @@ on:
- 'helm/**'
- 'fluss-rust/**'
- 'fluss-gateway/**'
+ - 'fluss-spark/**'
- '**/*.md'
pull_request:
paths-ignore:
@@ -34,6 +35,7 @@ on:
- 'helm/**'
- 'fluss-rust/**'
- 'fluss-gateway/**'
+ - 'fluss-spark/**'
- '**/*.md'
concurrency:
@@ -60,4 +62,5 @@ jobs:
name: "Java 11"
uses: ./.github/workflows/ci-template.yaml
with:
- java-version: "11"
\ No newline at end of file
+ java-version: "11"
+ modules: '["core","flink1","flink2","lake"]'
diff --git a/tools/ci/unstable_test_reporter.py b/tools/ci/unstable_test_reporter.py
index 410f7bf6b..84810be8a 100644
--- a/tools/ci/unstable_test_reporter.py
+++ b/tools/ci/unstable_test_reporter.py
@@ -53,7 +53,7 @@ log = logging.getLogger(__name__)
REPO_OWNER = "apache"
REPO_NAME = "fluss"
GITHUB_API = "https://api.github.com"
-WORKFLOW_FILES = ["ci.yaml", "nightly.yaml"]
+WORKFLOW_FILES = ["ci.yaml", "ci-spark.yaml", "nightly.yaml"]
ISSUE_TITLE_PREFIX = "[test] Unstable test "
MAX_STACK_TRACE_LINES = 30
DEFAULT_LOOKBACK_HOURS = 28