You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/08/20 08:04:50 UTC
(spark) branch branch-4.x updated: [SPARK-58880][4.X][INFRA] Make precompile a hard requirement
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 09509b6773c5 [SPARK-58880][4.X][INFRA] Make precompile a hard requirement
09509b6773c5 is described below
commit 09509b6773c57bd7bb070ea6ea28e9a4554f8ea7
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Aug 20 16:04:39 2026 +0800
[SPARK-58880][4.X][INFRA] Make precompile a hard requirement
### What changes were proposed in this pull request?
This PR backports #58129 to branch-4.x by cherry-picking
3ac19a09fbcf8f7ab0973e26239565caaa74d1e6.
It makes the `precompile` job a hard prerequisite for downstream build and test
jobs that consume its artifact.
Specifically, it:
- removes `continue-on-error` from `precompile`;
- requires `needs.precompile.result == 'success'` in downstream job conditions;
- makes precompiled artifact download and extraction mandatory; and
- removes local SBT build fallback paths from jobs that should reuse the
precompiled artifact.
### Why are the changes needed?
Currently, a failed `precompile` job can be treated as an optional optimization:
downstream jobs may continue and rebuild locally. This hides failures in the
shared precompile step and can waste CI resources. Since these jobs are wired to
consume the precompiled artifact, a precompile failure should stop them instead.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Ran:
```
git diff --check HEAD~1..HEAD
conda run -n spark-dev-313 python -c "import yaml; yaml.safe_load(open('.github/workflows/build_and_test.yml'))"
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #58150 from zhengruifeng/spark-58880-make-precompile-required-4.x.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
---
.github/workflows/build_and_test.yml | 78 ++++++------------------------------
1 file changed, 12 insertions(+), 66 deletions(-)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 08451e3d8ff4..64a722ddbb36 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -259,7 +259,7 @@ jobs:
build:
name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }}"
needs: [precondition, precompile]
- if: (!cancelled()) && fromJson(needs.precondition.outputs.required).build == 'true'
+ if: fromJson(needs.precondition.outputs.required).build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 150
strategy:
@@ -400,16 +400,10 @@ jobs:
python3.12 -m pip install 'numpy>=1.23.2' pyarrow 'pandas==2.3.3' pyyaml scipy unittest-xml-reporting 'lxml==4.9.4' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'zstandard==0.25.0'
python3.12 -m pip list
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v6
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -422,10 +416,8 @@ jobs:
export TERM=vt100
# Hive "other tests" test needs larger metaspace size based on experiment.
if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
export SERIAL_SBT_TESTS=1
./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --included-tags "$INCLUDED_TAGS" --excluded-tags "$EXCLUDED_TAGS"
- name: Upload test results to report
@@ -561,9 +553,6 @@ jobs:
name: "Precompile Spark"
runs-on: ubuntu-latest
timeout-minutes: 60
- # Optional optimization: if this job fails or is cancelled, the pyspark
- # matrix entries fall back to running the SBT build locally as before.
- continue-on-error: true
env:
HADOOP_PROFILE: ${{ inputs.hadoop }}
HIVE_PROFILE: hive2.3
@@ -751,16 +740,10 @@ jobs:
echo ""
done
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v6
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -770,10 +753,8 @@ jobs:
if: ${{ matrix.modules != 'pyspark-connect-old-client' }}
shell: 'script -q -e -c "bash {0}"'
run: |
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
if [[ "$MODULES_TO_TEST" == *"pyspark-install"* ]]; then
export SKIP_PACKAGING=false
echo "Python Packaging Tests Enabled!"
@@ -786,13 +767,6 @@ jobs:
SPARK_CONNECT_TESTING_REMOTE: sc://localhost
if: ${{ matrix.modules == 'pyspark-connect-old-client' && inputs.branch == 'master' }}
run: |
- # Build Spark
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- echo "Reusing precompiled artifact, skipping local SBT build."
- else
- ./build/sbt -Phive Test/package
- fi
-
# Make less noisy
cp conf/log4j2.properties.template conf/log4j2.properties
sed -i 's/rootLogger.level = info/rootLogger.level = warn/g' conf/log4j2.properties
@@ -920,16 +894,10 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v6
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -940,10 +908,8 @@ jobs:
# R issues at docker environment
export TZ=UTC
export _R_CHECK_SYSTEM_CLOCK_=FALSE
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
./dev/run-tests --parallelism 1 --modules sparkr
- name: Upload test results to report
if: always()
@@ -1302,7 +1268,7 @@ jobs:
# Any TPC-DS related updates on this job need to be applied to tpcds-1g-gen job of benchmark.yml as well
tpcds-1g:
needs: [precondition, precompile]
- if: (!cancelled()) && fromJson(needs.precondition.outputs.required).tpcds-1g == 'true'
+ if: fromJson(needs.precondition.outputs.required).tpcds-1g == 'true'
name: Run TPC-DS queries with SF=1
runs-on: ubuntu-latest
timeout-minutes: 120
@@ -1344,16 +1310,10 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
@@ -1426,7 +1386,7 @@ jobs:
docker-integration-tests:
needs: [precondition, precompile]
- if: (!cancelled()) && fromJson(needs.precondition.outputs.required).docker-integration-tests == 'true'
+ if: fromJson(needs.precondition.outputs.required).docker-integration-tests == 'true'
name: Run Docker integration tests
runs-on: ubuntu-latest
timeout-minutes: 120
@@ -1475,26 +1435,18 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
- name: Run tests
env: ${{ fromJSON(inputs.envs) }}
run: |
- if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then
- export SKIP_SCALA_BUILD=true
- echo "Reusing precompiled artifact, skipping local SBT build."
- fi
+ export SKIP_SCALA_BUILD=true
+ echo "Reusing precompiled artifact, skipping local SBT build."
./dev/run-tests --parallelism 1 --modules docker-integration-tests --included-tags org.apache.spark.tags.DockerTest
- name: Upload test results to report
if: always()
@@ -1520,7 +1472,7 @@ jobs:
k8s-integration-tests:
needs: [precondition, precompile]
- if: (!cancelled()) && fromJson(needs.precondition.outputs.required).k8s-integration-tests == 'true'
+ if: fromJson(needs.precondition.outputs.required).k8s-integration-tests == 'true'
name: Run Spark on Kubernetes Integration test
runs-on: ubuntu-latest
timeout-minutes: 120
@@ -1568,16 +1520,10 @@ jobs:
distribution: zulu
java-version: ${{ inputs.java }}
- name: Download precompiled artifact
- id: download-precompiled
- if: needs.precompile.result == 'success'
- continue-on-error: true
uses: actions/download-artifact@v8
with:
name: spark-compile-${{ inputs.branch }}-${{ github.run_id }}
- name: Extract precompiled artifact
- id: extract-precompiled
- if: steps.download-precompiled.outcome == 'success'
- continue-on-error: true
run: |
zstd -dc compile-artifact.tar.zst | tar -xf -
rm compile-artifact.tar.zst
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org