You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/07/06 15:30:51 UTC
(fory) branch main updated: ci(python): Build manylinux_2_28 wheels using auditwheel (#2381)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new bf9ea015a ci(python): Build manylinux_2_28 wheels using auditwheel (#2381)
bf9ea015a is described below
commit bf9ea015a0da694b0cf0c39222fcb0a6b2e9a73a
Author: Emre Şafak <39...@users.noreply.github.com>
AuthorDate: Sun Jul 6 11:30:47 2025 -0400
ci(python): Build manylinux_2_28 wheels using auditwheel (#2381)
## Why?
The way we were creating wheels was hacky and failing
[PEP600](https://github.com/mayeut/pep600_compliance).
## What does this PR do?
* Removed deprecated Python environment creation and wheel renaming
logic from `ci/deploy.sh`.
* Modified `build_pyfory` to use `auditwheel` for repairing Linux wheels
and embedding shared libraries.
* Added support for `auditwheel` to repair wheels for manylinux2014,
manylinux2010, and manylinux2008 platforms.
* Updated `.github/workflows/release.yaml` to use Docker containers with
manylinux images for building Linux wheels.
* Configured the workflow to build wheels natively on macOS and Windows,
using `delocate` and `delvewheel` respectively for dependency bundling.
* Added `auditwheel`, `delocate`, and `delvewheel` to the build
dependencies.
* Included `manylinux_2_28_x86_64` and `manylinux_2_28_aarch64` images
as environment variables for clarity.
* Ensured the correct `PLAT` environment variable is set for Docker
builds.
## Related issues
Fixes #2363
## Does this PR introduce any user-facing change?
No
## Testing
Only on Mac Silicon in a Linux container, using this script:
```bash
#!/bin/bash
set -e
# This script now uses the same manylinux image as the release.yaml workflow
# to ensure the build environment is as close as possible to the CI.
# The ubuntu-arm64-build.Dockerfile is no longer needed.
DOCKER_IMAGE="quay.io/pypa/manylinux_2_28_aarch64"
PLAT="manylinux_2_28_aarch64"
echo "Pulling latest manylinux image for ARM64..."
docker pull "$DOCKER_IMAGE"
echo "Running build in manylinux container..."
# This command block is aligned with the corrected release.yaml steps
docker run --rm --platform linux/arm64 -v "$(pwd)":/work \
-w /work -e PLAT=$PLAT $DOCKER_IMAGE \
bash -c "
set -e
# Add a specific Python version (e.g., 3.12) to the PATH.
# Manylinux images have multiple versions in /opt/python/
export PATH=/opt/python/cp312-cp312/bin:\$PATH
echo \"Using Python from: \$(which python)\"
echo \"Python version: \$(python -V)\"
# Install system build dependencies, including sudo
yum install -y git sudo wget
git config --global --add safe.directory /work
echo 'Installing bazel...'
bash ci/run_ci.sh install_bazel
echo 'Building pyfory'
# build_pyfory will install python dependencies and then build the wheel
bash ci/deploy.sh build_pyfory
"
echo "Build completed. Check ./dist directory for wheels"
```
---------
Co-authored-by: Emre Şafak <es...@users.noreply.github.com>
---
.github/workflows/release.yaml | 43 ++++++++++++++--
ci/deploy.sh | 110 ++++++++++-------------------------------
2 files changed, 64 insertions(+), 89 deletions(-)
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index c0da8ceac..7061a3436 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -30,6 +30,9 @@ jobs:
matrix:
python-version: [3.8, 3.9, "3.10", 3.11, 3.12, 3.13]
os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, windows-2022] # macos-13: x86, macos-14: arm64
+ env:
+ manylinux_x86_64_image: quay.io/pypa/manylinux_2_28_x86_64
+ manylinux_aarch64_image: quay.io/pypa/manylinux_2_28_aarch64
steps:
- uses: actions/checkout@v4
@@ -37,7 +40,8 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- - name: Install bazel
+ - name: Install bazel (for macOS and Windows)
+ if: "!startsWith(matrix.os, 'ubuntu')"
shell: bash
run: |
if [ "$RUNNER_OS" == "Windows" ]; then
@@ -48,11 +52,42 @@ jobs:
- name: Update version in setup.py
shell: bash
run: ci/deploy.sh bump_py_version
- - name: Build a binary wheel
+ # --------- Use manylinux for Linux wheels ---------
+ - name: Build a binary wheel (Linux, manylinux)
+ if: startsWith(matrix.os, 'ubuntu')
shell: bash
run: |
- ci/deploy.sh build_pyfory
- ci/deploy.sh rename_wheels dist
+ DOCKER_IMAGE=""
+ PLAT=""
+ if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
+ DOCKER_IMAGE="${{ env.manylinux_x86_64_image }}"
+ PLAT="manylinux_2_28_x86_64"
+ elif [[ "${{ matrix.os }}" == "ubuntu-24.04-arm" ]]; then
+ DOCKER_IMAGE="${{ env.manylinux_aarch64_image }}"
+ PLAT="manylinux_2_28_aarch64"
+ fi
+ docker run --rm -e PLAT=$PLAT \
+ -v ${{ github.workspace }}:/work \
+ -w /work "$DOCKER_IMAGE" \
+ bash -c "
+ set -e
+ # Install build dependencies inside the container
+ yum install -y git sudo wget
+ git config --global --add safe.directory /work
+
+ # Install Bazel inside the container
+ bash ci/run_ci.sh install_bazel
+
+ # Build the wheel inside the container
+ bash ci/deploy.sh build_pyfory
+ "
+
+ # --------- Native (not in container) for macOS and Windows ---------
+ - name: Build a binary wheel (native)
+ if: "!startsWith(matrix.os, 'ubuntu')"
+ shell: bash
+ run: |
+ ci/deploy.sh build_pyfory
- name: Upload Wheel Artifact
uses: actions/upload-artifact@v4
with:
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 62f5f61ef..6cce344b0 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -31,57 +31,6 @@ set -e
ROOT="$(git rev-parse --show-toplevel)"
cd "$ROOT"
-WHEEL_DIR="$ROOT/.whl"
-
-PYTHONS=("cp37-cp37m"
- "cp38-cp38"
- "cp39-cp39"
- "cp310-cp310"
- "cp310-cp311"
- "cp312-cp312"
- "cp313-cp313")
-
-VERSIONS=("3.7"
- "3.8"
- "3.9"
- "3.10"
- "3.11"
- "3.12"
- "3.13")
-
-create_py_envs() {
- source $(conda info --base)/etc/profile.d/conda.sh
- for version in "${VERSIONS[@]}"; do
- conda create -y --name "py$version" python="$version"
- done
- conda env list
-}
-
-rename_wheels() {
- for path in "$1"/*.whl; do
- if [ -f "${path}" ]; then
- # Rename linux to manylinux1
- new_path="${path//linux/manylinux1}"
- if [ "${path}" != "${new_path}" ]; then
- mv "${path}" "${new_path}"
- fi
-
- # Copy macosx_14_0_x86_64 to macosx_10_12_x86_64
- if [[ "${path}" == *macosx_14_0_x86_64.whl ]]; then
- copy_path="${path//macosx_14_0_x86_64/macosx_10_12_x86_64}"
- mv "${path}" "${copy_path}"
- fi
- fi
- done
-}
-
-rename_mac_wheels() {
- for path in "$WHEEL_DIR"/*.whl; do
- if [ -f "${path}" ]; then
- cp "${path}" "${path//macosx_12_0_x86_64/macosx_10_13_x86_64}"
- fi
- done
-}
bump_version() {
python "$ROOT/ci/release.py" bump_version -l all -version "$1"
@@ -116,7 +65,7 @@ deploy_jars() {
build_pyfory() {
echo "Python version $(python -V), path $(which python)"
install_pyarrow
- pip install Cython wheel pytest
+ pip install Cython wheel pytest auditwheel
pushd "$ROOT/python"
pip list
echo "Install pyfory"
@@ -132,38 +81,32 @@ build_pyfory() {
fi
python setup.py bdist_wheel --dist-dir=../dist
- popd
-}
-deploy_python() {
- source $(conda info --base)/etc/profile.d/conda.sh
- if command -v pyenv; then
- pyenv local system
+ if [ -n "$PLAT" ]; then
+ # In manylinux container, repair the wheel to embed shared libraries
+ # and rename the wheel with the manylinux tag.
+ PYARROW_LIB_DIR=$(python -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))')
+ export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH"
+ auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" -w ../dist/
+ rm ../dist/pyfory-*-linux_*.whl
+ elif [[ "$OSTYPE" == "darwin"* ]]; then
+ # macOS: use delocate to bundle dependencies and fix wheel tags
+ pip install delocate
+ mkdir -p ../dist_repaired
+ delocate-wheel -w ../dist_repaired/ ../dist/pyfory-*-macosx*.whl
+ rm ../dist/pyfory-*-macosx*.whl
+ mv ../dist_repaired/* ../dist/
+ rmdir ../dist_repaired
+ elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
+ # Windows: use delvewheel to bundle dependencies
+ pip install delvewheel
+ mkdir -p ../dist_repaired
+ delvewheel repair ../dist/pyfory-*-win*.whl -w ../dist_repaired/
+ rm ../dist/pyfory-*-win*.whl
+ mv ../dist_repaired/* ../dist/
+ rmdir ../dist_repaired
fi
- cd "$ROOT/python"
- rm -rf "$WHEEL_DIR"
- mkdir -p "$WHEEL_DIR"
- for ((i=0; i<${#PYTHONS[@]}; ++i)); do
- PYTHON=${PYTHONS[i]}
- ENV="py${VERSIONS[i]}"
- conda activate "$ENV"
- python -V
- git clean -f -f -x -d -e .whl
- # Ensure bazel select the right version of python
- bazel clean --expunge
- install_pyarrow
- pip install --ignore-installed twine setuptools cython numpy
- pyarrow_dir=$(python -c "import importlib.util; import os; print(os.path.dirname(importlib.util.find_spec('pyarrow').origin))")
- # ensure pyarrow is clean
- rm -rf "$pyarrow_dir"
- pip install --ignore-installed pyarrow==$pyarrow_version
- python setup.py clean
- python setup.py bdist_wheel
- mv dist/pyfory*.whl "$WHEEL_DIR"
- done
- rename_wheels "$WHEEL_DIR"
- twine check "$WHEEL_DIR"/pyfory*.whl
- twine upload -r pypi "$WHEEL_DIR"/pyfory*.whl
+ popd
}
install_pyarrow() {
@@ -191,9 +134,6 @@ case "$1" in
java) # Deploy jars to maven repository.
deploy_jars
;;
-python) # Deploy wheel to pypi
- deploy_python
- ;;
*)
echo "Execute command $*"
"$@"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org