You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/12/16 04:19:09 UTC
(spark) branch master updated: [SPARK-50583][INFRA] Apply Python 3.11 image in PR build
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3f8e3959108d [SPARK-50583][INFRA] Apply Python 3.11 image in PR build
3f8e3959108d is described below
commit 3f8e3959108dd734aca531992c9a22f4ddf79ba9
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Dec 16 13:18:57 2024 +0900
[SPARK-50583][INFRA] Apply Python 3.11 image in PR build
### What changes were proposed in this pull request?
Apply Python 3.11 image in PR build
### Why are the changes needed?
for `master`:
- after this PR, all workflows should only use the new images;
- a new workflow should specify `PYSPARK_IMAGE_TO_TEST` and `PYSPARK_TO_TEST` if `pyspark` job is enabled;
for `branch-3.5`:
- it still uses the old image `dev/infra/Dockerfile`
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49193 from zhengruifeng/py_311_pr_builder.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.github/workflows/build_and_test.yml | 38 ++++++++++++++++++++---------
.github/workflows/build_branch35.yml | 1 +
.github/workflows/build_branch35_python.yml | 1 +
3 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 533581ce4b6d..174a8aa6bacb 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -41,7 +41,7 @@ on:
description: Additional environment variables to set when running the tests. Should be in JSON format.
required: false
type: string
- default: '{}'
+ default: '{"PYSPARK_IMAGE_TO_TEST": "python-311", "PYTHON_TO_TEST": "python3.11"}'
jobs:
description: >-
Jobs to run, and should be in JSON format. The values should be matched with the job's key defined
@@ -55,7 +55,6 @@ jobs:
runs-on: ubuntu-latest
env:
GITHUB_PREV_SHA: ${{ github.event.before }}
- PYSPARK_IMAGE_TO_TEST: ''
outputs:
required: ${{ steps.set-outputs.outputs.required }}
image_url: ${{ steps.infra-image-outputs.outputs.image_url }}
@@ -135,6 +134,28 @@ jobs:
precondition="${precondition//$'\n'/}"
echo "required=$precondition" >> $GITHUB_OUTPUT
fi
+ - name: Check envs
+ id: check-envs
+ if: inputs.branch != 'branch-3.5'
+ env: ${{ fromJSON(inputs.envs) }}
+ run: |
+ if [[ "${{ fromJson(steps.set-outputs.outputs.required).pyspark }}" == 'true' ]]; then
+ if [[ "${{ env.PYSPARK_IMAGE_TO_TEST }}" == "" ]]; then
+ echo "PYSPARK_IMAGE_TO_TEST is required when pyspark is enabled."
+ exit 1
+ fi
+ PYSPARK_IMAGE_PATH="dev/spark-test-image/${{ env.PYSPARK_IMAGE_TO_TEST }}/Dockerfile"
+ if [ -f $PYSPARK_IMAGE_PATH ]; then
+ echo "Dockerfile $PYSPARK_IMAGE_PATH exists."
+ else
+ echo "Dockerfile $PYSPARK_IMAGE_PATH does NOT exist."
+ exit 1
+ fi
+ if [[ "${{ env.PYTHON_TO_TEST }}" == "" ]]; then
+ echo "PYTHON_TO_TEST is required when pyspark is enabled."
+ exit 1
+ fi
+ fi
- name: Generate infra image URL
id: infra-image-outputs
run: |
@@ -192,11 +213,7 @@ jobs:
echo "image_docs_url_link=${{ steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT
echo "image_lint_url_link=${{ steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT
echo "image_sparkr_url_link=${{ steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT
- if [[ "${{ env.PYSPARK_IMAGE_TO_TEST }}" != "" ]]; then
- echo "image_pyspark_url_link=${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >> $GITHUB_OUTPUT
- else
- echo "image_pyspark_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT
- fi
+ echo "image_pyspark_url_link=${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >> $GITHUB_OUTPUT
fi
# Build: build Spark and run the tests for specified modules.
@@ -380,8 +397,6 @@ jobs:
runs-on: ubuntu-latest
permissions:
packages: write
- env:
- PYSPARK_IMAGE_TO_TEST: ''
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
@@ -450,8 +465,8 @@ jobs:
${{ needs.precondition.outputs.image_sparkr_url }}
# Use the infra image cache to speed up
cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }}
- - name: Build and push (PySpark ${{ env.PYSPARK_IMAGE_TO_TEST }})
- if: ${{ env.PYSPARK_IMAGE_TO_TEST }}
+ - name: Build and push (PySpark with ${{ env.PYSPARK_IMAGE_TO_TEST }})
+ if: (inputs.branch != 'branch-3.5') && (${{ env.PYSPARK_IMAGE_TO_TEST }} != '')
id: docker_build_pyspark
env: ${{ fromJSON(inputs.envs) }}
uses: docker/build-push-action@v6
@@ -511,7 +526,6 @@ jobs:
- modules: ${{ fromJson(needs.precondition.outputs.required).pyspark-pandas != 'true' && 'pyspark-pandas-connect-part3' }}
env:
MODULES_TO_TEST: ${{ matrix.modules }}
- PYTHON_TO_TEST: 'python3.11'
HADOOP_PROFILE: ${{ inputs.hadoop }}
HIVE_PROFILE: hive2.3
# GitHub Actions' default miniconda to use in pip packaging test.
diff --git a/.github/workflows/build_branch35.yml b/.github/workflows/build_branch35.yml
index 2ec080d5722c..ad46b6ce3238 100644
--- a/.github/workflows/build_branch35.yml
+++ b/.github/workflows/build_branch35.yml
@@ -37,6 +37,7 @@ jobs:
envs: >-
{
"SCALA_PROFILE": "scala2.13",
+ "PYSPARK_IMAGE_TO_TEST": "",
"PYTHON_TO_TEST": "",
"ORACLE_DOCKER_IMAGE_NAME": "gvenzl/oracle-xe:21.3.0"
}
diff --git a/.github/workflows/build_branch35_python.yml b/.github/workflows/build_branch35_python.yml
index 1585534d33ba..ed83f8c4cdf8 100644
--- a/.github/workflows/build_branch35_python.yml
+++ b/.github/workflows/build_branch35_python.yml
@@ -36,6 +36,7 @@ jobs:
hadoop: hadoop3
envs: >-
{
+ "PYSPARK_IMAGE_TO_TEST": "",
"PYTHON_TO_TEST": ""
}
jobs: >-
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org