You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@auron.apache.org by cs...@apache.org on 2026/05/19 04:45:44 UTC
(auron) branch master updated: [AURON #2260] Add support for Spark 4.0 and 4.1 in build workflows (#2261)
This is an automated email from the ASF dual-hosted git repository.
cxzl25 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 98539287f [AURON #2260] Add support for Spark 4.0 and 4.1 in build workflows (#2261)
98539287f is described below
commit 98539287f2e7ca83d85fe500bdbaaeea874a7b37
Author: cxzl25 <38...@users.noreply.github.com>
AuthorDate: Tue May 19 12:45:39 2026 +0800
[AURON #2260] Add support for Spark 4.0 and 4.1 in build workflows (#2261)
# Which issue does this PR close?
Closes #2260
# Rationale for this change
# What changes are included in this PR?
Add Spark 4.0 and Spark 4.1 to the release build matrix across all
platforms (amd64, arm, macOS), with proper version constraints and
Docker image support for JDK 17+.
# Are there any user-facing changes?
# How was this patch tested?
GHA
```bash
AURON_JAVA_VERSION=17 ./auron-build.sh --docker true --image rockylinux8 --release --sparkver 4.0 --scalaver 2.13
```
---
.github/workflows/build-amd64-releases.yml | 38 +++++++++++++++++++++++++++---
.github/workflows/build-arm-releases.yml | 11 ++++++++-
.github/workflows/build-macos-releases.yml | 11 ++++++++-
auron-build.sh | 4 ++++
dev/docker-build/docker-compose.yml | 1 +
dev/docker-build/rockylinux8/Dockerfile | 22 +++++++++++++----
dev/docker-build/rockylinux8/entrypoint.sh | 28 ++++++++++++++++++++++
7 files changed, 106 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/build-amd64-releases.yml b/.github/workflows/build-amd64-releases.yml
index 20cafe1fb..b34fa763b 100644
--- a/.github/workflows/build-amd64-releases.yml
+++ b/.github/workflows/build-amd64-releases.yml
@@ -37,13 +37,13 @@ jobs:
runs-on: ${{ matrix.runner }}
strategy:
matrix:
- sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5]
+ sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1]
scalaver: [ 2.12, 2.13 ]
javaver: [ 8, 21 ]
auronver: [8.0.0-SNAPSHOT]
runner: [ ubuntu-24.04 ]
exclude:
- # Only build on scala-2.13 for spark-3.5
+ # Only build on scala-2.13 for spark-3.5+
- sparkver: spark-3.0
scalaver: '2.13'
- sparkver: spark-3.1
@@ -64,6 +64,36 @@ jobs:
javaver: '21'
- sparkver: spark-3.4
javaver: '21'
+ # Spark 4.x only supports scala-2.13 and java 17+
+ - sparkver: spark-4.0
+ scalaver: '2.12'
+ - sparkver: spark-4.1
+ scalaver: '2.12'
+ - sparkver: spark-4.0
+ javaver: '8'
+ - sparkver: spark-4.1
+ javaver: '8'
+ # Spark 4.x JDK 21 builds are handled by explicit rockylinux8 include entries below.
+ - sparkver: spark-4.0
+ scalaver: '2.13'
+ javaver: '21'
+ - sparkver: spark-4.1
+ scalaver: '2.13'
+ javaver: '21'
+ include:
+ # Spark 4.x uses rockylinux8 image (with JDK 21)
+ - sparkver: spark-4.0
+ scalaver: '2.13'
+ javaver: '21'
+ auronver: 8.0.0-SNAPSHOT
+ runner: ubuntu-24.04
+ image: rockylinux8
+ - sparkver: spark-4.1
+ scalaver: '2.13'
+ javaver: '21'
+ auronver: 8.0.0-SNAPSHOT
+ runner: ubuntu-24.04
+ image: rockylinux8
steps:
- uses: actions/checkout@v6
@@ -97,11 +127,13 @@ jobs:
key: auron-${{matrix.sparkver}}_${{matrix.scalaver}}-release-${{ env.osclassfier }}-${{ matrix.auronver }}.jar
- name: Build auron-${{ matrix.sparkver }}_${{ matrix.scalaver }}
+ env:
+ AURON_JAVA_VERSION: ${{ matrix.javaver }}
run: |
sed -i 's/docker-compose -f/docker compose -f/g' ./auron-build.sh
./auron-build.sh \
--docker true \
- --image centos7 \
+ --image ${{ matrix.image || 'centos7' }} \
--release \
--sparkver ${{ env.sparkver_short }} \
--scalaver ${{ matrix.scalaver }}
diff --git a/.github/workflows/build-arm-releases.yml b/.github/workflows/build-arm-releases.yml
index fb278144e..817193ae6 100644
--- a/.github/workflows/build-arm-releases.yml
+++ b/.github/workflows/build-arm-releases.yml
@@ -37,7 +37,7 @@ jobs:
runs-on: ${{ matrix.runner }}
strategy:
matrix:
- sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5]
+ sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1]
auronver: [8.0.0-SNAPSHOT]
scalaver: [2.12, 2.13]
javaver: [ 8, 21 ]
@@ -64,6 +64,15 @@ jobs:
javaver: '21'
- sparkver: spark-3.4
javaver: '21'
+ # Spark 4.x only supports scala-2.13 and java 17+
+ - sparkver: spark-4.0
+ scalaver: '2.12'
+ - sparkver: spark-4.1
+ scalaver: '2.12'
+ - sparkver: spark-4.0
+ javaver: '8'
+ - sparkver: spark-4.1
+ javaver: '8'
steps:
- uses: actions/checkout@v6
diff --git a/.github/workflows/build-macos-releases.yml b/.github/workflows/build-macos-releases.yml
index 23f159e79..28763761e 100644
--- a/.github/workflows/build-macos-releases.yml
+++ b/.github/workflows/build-macos-releases.yml
@@ -37,7 +37,7 @@ jobs:
runs-on: ${{ matrix.runner }}
strategy:
matrix:
- sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5]
+ sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1]
auronver: [8.0.0-SNAPSHOT]
scalaver: [2.12, 2.13]
javaver: [8, 21]
@@ -64,6 +64,15 @@ jobs:
javaver: '21'
- sparkver: spark-3.4
javaver: '21'
+ # Spark 4.x only supports scala-2.13 and java 17+
+ - sparkver: spark-4.0
+ scalaver: '2.12'
+ - sparkver: spark-4.1
+ scalaver: '2.12'
+ - sparkver: spark-4.0
+ javaver: '8'
+ - sparkver: spark-4.1
+ javaver: '8'
steps:
- uses: actions/checkout@v6
diff --git a/auron-build.sh b/auron-build.sh
index fc5f75a48..5cf81976c 100755
--- a/auron-build.sh
+++ b/auron-build.sh
@@ -567,6 +567,10 @@ if [[ "$USE_DOCKER" == true ]]; then
echo "[INFO] Compiling inside Docker container..."
export AURON_BUILD_ARGS="${BUILD_ARGS[*]}"
export BUILD_CONTEXT="./${IMAGE_NAME}"
+ # Spark 4.x requires JDK 17+, auto-set if not specified
+ if [[ -z "$AURON_JAVA_VERSION" && "$SPARK_VER" == 4.* ]]; then
+ export AURON_JAVA_VERSION="17"
+ fi
exec docker-compose -f dev/docker-build/docker-compose.yml up --abort-on-container-exit
else
echo "[INFO] Compiling locally with maven args: $MVN_CMD ${MVN_ARGS[@]} ${MVN_D_ARGS} $@"
diff --git a/dev/docker-build/docker-compose.yml b/dev/docker-build/docker-compose.yml
index e9aa9f0dd..7bfb0f816 100644
--- a/dev/docker-build/docker-compose.yml
+++ b/dev/docker-build/docker-compose.yml
@@ -32,6 +32,7 @@ services:
environment:
RUSTFLAGS: "-C target-cpu=skylake"
AURON_BUILD_ARGS: "${AURON_BUILD_ARGS}"
+ AURON_JAVA_VERSION: "${AURON_JAVA_VERSION:-8}"
command: >
bash -c '
source ~/.bashrc &&
diff --git a/dev/docker-build/rockylinux8/Dockerfile b/dev/docker-build/rockylinux8/Dockerfile
index e519c4f4c..d5ee178f1 100644
--- a/dev/docker-build/rockylinux8/Dockerfile
+++ b/dev/docker-build/rockylinux8/Dockerfile
@@ -63,14 +63,28 @@ RUN curl https://sh.rustup.rs -sSf -o rustup-init && \
ENV PATH="/root/.cargo/bin:${PATH}"
# ---------------------------------------------------------------------
-# Install Java (OpenJDK 8)
+# Install Java (OpenJDK 8, 17, and 21)
# ---------------------------------------------------------------------
-RUN dnf -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel && \
+RUN dnf -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel \
+ java-17-openjdk java-17-openjdk-devel \
+ java-21-openjdk java-21-openjdk-devel && \
dnf clean all && \
rm -rf /var/cache/dnf
-ENV JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"
+
+# Create stable symlinks for JDK paths
+RUN ln -sfn /usr/lib/jvm/java-1.8.0-openjdk /usr/lib/jvm/java-8 && \
+ ln -sfn /usr/lib/jvm/java-17-openjdk /usr/lib/jvm/java-17 && \
+ ln -sfn /usr/lib/jvm/java-21-openjdk /usr/lib/jvm/java-21
+
+ENV JAVA_HOME="/usr/lib/jvm/java-8"
ENV PATH="${JAVA_HOME}/bin:${PATH}"
+# Entrypoint script to switch JDK at runtime based on AURON_JAVA_VERSION env var
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
# ---------------------------------------------------------------------
# Set working directory
# ---------------------------------------------------------------------
@@ -79,4 +93,4 @@ WORKDIR /auron
# ---------------------------------------------------------------------
# Default command
# ---------------------------------------------------------------------
-CMD ["/bin/bash"]
\ No newline at end of file
+CMD ["/bin/bash"]
diff --git a/dev/docker-build/rockylinux8/entrypoint.sh b/dev/docker-build/rockylinux8/entrypoint.sh
new file mode 100644
index 000000000..d3d991f17
--- /dev/null
+++ b/dev/docker-build/rockylinux8/entrypoint.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Switch JDK based on AURON_JAVA_VERSION env var (default: 8)
+case "${AURON_JAVA_VERSION:-8}" in
+ 17) export JAVA_HOME="/usr/lib/jvm/java-17" ;;
+ 21) export JAVA_HOME="/usr/lib/jvm/java-21" ;;
+ *) export JAVA_HOME="/usr/lib/jvm/java-8" ;;
+esac
+export PATH="$JAVA_HOME/bin:$PATH"
+
+exec "$@"