You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/07/06 15:30:51 UTC

(fory) branch main updated: ci(python): Build manylinux_2_28 wheels using auditwheel (#2381)

This is an automated email from the ASF dual-hosted git repository.

chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git


The following commit(s) were added to refs/heads/main by this push:
     new bf9ea015a ci(python): Build manylinux_2_28 wheels using auditwheel (#2381)
bf9ea015a is described below

commit bf9ea015a0da694b0cf0c39222fcb0a6b2e9a73a
Author: Emre Şafak <39...@users.noreply.github.com>
AuthorDate: Sun Jul 6 11:30:47 2025 -0400

    ci(python): Build manylinux_2_28 wheels using auditwheel (#2381)
    
    ## Why?
    
    The way we were creating wheels was hacky and failing
    [PEP600](https://github.com/mayeut/pep600_compliance).
    
    ## What does this PR do?
    
    * Removed deprecated Python environment creation and wheel renaming
    logic from `ci/deploy.sh`.
    * Modified `build_pyfory` to use `auditwheel` for repairing Linux wheels
    and embedding shared libraries.
    * Added support for `auditwheel` to repair wheels for manylinux2014,
    manylinux2010, and manylinux2008 platforms.
    * Updated `.github/workflows/release.yaml` to use Docker containers with
    manylinux images for building Linux wheels.
    * Configured the workflow to build wheels natively on macOS and Windows,
    using `delocate` and `delvewheel` respectively for dependency bundling.
    * Added `auditwheel`, `delocate`, and `delvewheel` to the build
    dependencies.
    * Included `manylinux_2_28_x86_64` and `manylinux_2_28_aarch64` images
    as environment variables for clarity.
    * Ensured the correct `PLAT` environment variable is set for Docker
    builds.
    
    ## Related issues
    
    Fixes #2363
    
    ## Does this PR introduce any user-facing change?
    
    No
    
    ## Testing
    
    Only on Mac Silicon in a Linux container, using this script:
    
    ```bash
    #!/bin/bash
    set -e
    
    # This script now uses the same manylinux image as the release.yaml workflow
    # to ensure the build environment is as close as possible to the CI.
    # The ubuntu-arm64-build.Dockerfile is no longer needed.
    
    DOCKER_IMAGE="quay.io/pypa/manylinux_2_28_aarch64"
    PLAT="manylinux_2_28_aarch64"
    
    echo "Pulling latest manylinux image for ARM64..."
    docker pull "$DOCKER_IMAGE"
    
    echo "Running build in manylinux container..."
    # This command block is aligned with the corrected release.yaml steps
    docker run --rm --platform linux/arm64 -v "$(pwd)":/work \
        -w /work -e PLAT=$PLAT $DOCKER_IMAGE \
        bash -c "
          set -e
          # Add a specific Python version (e.g., 3.12) to the PATH.
          # Manylinux images have multiple versions in /opt/python/
          export PATH=/opt/python/cp312-cp312/bin:\$PATH
    
          echo \"Using Python from: \$(which python)\"
          echo \"Python version: \$(python -V)\"
    
    
          # Install system build dependencies, including sudo
          yum install -y git sudo wget
          git config --global --add safe.directory /work
    
          echo 'Installing bazel...'
          bash ci/run_ci.sh install_bazel
    
          echo 'Building pyfory'
          # build_pyfory will install python dependencies and then build the wheel
          bash ci/deploy.sh build_pyfory
        "
    echo "Build completed. Check ./dist directory for wheels"
    ```
    
    ---------
    
    Co-authored-by: Emre Şafak <es...@users.noreply.github.com>
---
 .github/workflows/release.yaml |  43 ++++++++++++++--
 ci/deploy.sh                   | 110 ++++++++++-------------------------------
 2 files changed, 64 insertions(+), 89 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index c0da8ceac..7061a3436 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -30,6 +30,9 @@ jobs:
       matrix:
         python-version: [3.8, 3.9, "3.10", 3.11, 3.12, 3.13]
         os: [ubuntu-latest, ubuntu-24.04-arm, macos-13, macos-14, windows-2022]  # macos-13: x86, macos-14: arm64
+    env:
+      manylinux_x86_64_image: quay.io/pypa/manylinux_2_28_x86_64
+      manylinux_aarch64_image: quay.io/pypa/manylinux_2_28_aarch64
 
     steps:
       - uses: actions/checkout@v4
@@ -37,7 +40,8 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install bazel
+      - name: Install bazel (for macOS and Windows)
+        if: "!startsWith(matrix.os, 'ubuntu')"
         shell: bash
         run: |
           if [ "$RUNNER_OS" == "Windows" ]; then
@@ -48,11 +52,42 @@ jobs:
       - name: Update version in setup.py
         shell: bash
         run: ci/deploy.sh bump_py_version
-      - name: Build a binary wheel
+       # --------- Use manylinux for Linux wheels ---------
+      - name: Build a binary wheel (Linux, manylinux)
+        if: startsWith(matrix.os, 'ubuntu')
         shell: bash
         run: |
-          ci/deploy.sh build_pyfory
-          ci/deploy.sh rename_wheels dist
+           DOCKER_IMAGE=""
+           PLAT=""
+           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
+             DOCKER_IMAGE="${{ env.manylinux_x86_64_image }}"
+             PLAT="manylinux_2_28_x86_64"
+           elif [[ "${{ matrix.os }}" == "ubuntu-24.04-arm" ]]; then
+             DOCKER_IMAGE="${{ env.manylinux_aarch64_image }}"
+             PLAT="manylinux_2_28_aarch64"
+           fi
+           docker run --rm -e PLAT=$PLAT \
+             -v ${{ github.workspace }}:/work \
+             -w /work "$DOCKER_IMAGE" \
+             bash -c "
+               set -e
+               # Install build dependencies inside the container
+               yum install -y git sudo wget
+               git config --global --add safe.directory /work
+
+               # Install Bazel inside the container
+               bash ci/run_ci.sh install_bazel
+
+               # Build the wheel inside the container
+               bash ci/deploy.sh build_pyfory
+             "
+
+       # --------- Native (not in container) for macOS and Windows ---------
+      - name: Build a binary wheel (native)
+        if: "!startsWith(matrix.os, 'ubuntu')"
+        shell: bash
+        run: |
+           ci/deploy.sh build_pyfory
       - name: Upload Wheel Artifact
         uses: actions/upload-artifact@v4
         with:
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 62f5f61ef..6cce344b0 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -31,57 +31,6 @@ set -e
 
 ROOT="$(git rev-parse --show-toplevel)"
 cd "$ROOT"
-WHEEL_DIR="$ROOT/.whl"
-
-PYTHONS=("cp37-cp37m"
-         "cp38-cp38"
-         "cp39-cp39"
-         "cp310-cp310"
-         "cp310-cp311"
-         "cp312-cp312"
-         "cp313-cp313")
-
-VERSIONS=("3.7"
-          "3.8"
-          "3.9"
-          "3.10"
-          "3.11"
-          "3.12"
-          "3.13")
-
-create_py_envs() {
-  source $(conda info --base)/etc/profile.d/conda.sh
-  for version in "${VERSIONS[@]}"; do
-    conda create -y --name "py$version" python="$version"
-  done
-  conda env list
-}
-
-rename_wheels() {
-  for path in "$1"/*.whl; do
-    if [ -f "${path}" ]; then
-      # Rename linux to manylinux1
-      new_path="${path//linux/manylinux1}"
-      if [ "${path}" != "${new_path}" ]; then
-        mv "${path}" "${new_path}"
-      fi
-
-      # Copy macosx_14_0_x86_64 to macosx_10_12_x86_64
-      if [[ "${path}" == *macosx_14_0_x86_64.whl ]]; then
-        copy_path="${path//macosx_14_0_x86_64/macosx_10_12_x86_64}"
-        mv "${path}" "${copy_path}"
-      fi
-    fi
-  done
-}
-
-rename_mac_wheels() {
-  for path in "$WHEEL_DIR"/*.whl; do
-    if [ -f "${path}" ]; then
-      cp "${path}" "${path//macosx_12_0_x86_64/macosx_10_13_x86_64}"
-    fi
-  done
-}
 
 bump_version() {
   python "$ROOT/ci/release.py" bump_version -l all -version "$1"
@@ -116,7 +65,7 @@ deploy_jars() {
 build_pyfory() {
   echo "Python version $(python -V), path $(which python)"
   install_pyarrow
-  pip install Cython wheel pytest
+  pip install Cython wheel pytest auditwheel
   pushd "$ROOT/python"
   pip list
   echo "Install pyfory"
@@ -132,38 +81,32 @@ build_pyfory() {
   fi
 
   python setup.py bdist_wheel --dist-dir=../dist
-  popd
-}
 
-deploy_python() {
-  source $(conda info --base)/etc/profile.d/conda.sh
-  if command -v pyenv; then
-    pyenv local system
+  if [ -n "$PLAT" ]; then
+    # In manylinux container, repair the wheel to embed shared libraries
+    # and rename the wheel with the manylinux tag.
+    PYARROW_LIB_DIR=$(python -c 'import pyarrow; print(":".join(pyarrow.get_library_dirs()))')
+    export LD_LIBRARY_PATH="$PYARROW_LIB_DIR:$LD_LIBRARY_PATH"
+    auditwheel repair ../dist/pyfory-*-linux_*.whl --plat "$PLAT" -w ../dist/
+    rm ../dist/pyfory-*-linux_*.whl
+  elif [[ "$OSTYPE" == "darwin"* ]]; then
+    # macOS: use delocate to bundle dependencies and fix wheel tags
+    pip install delocate
+    mkdir -p ../dist_repaired
+    delocate-wheel -w ../dist_repaired/ ../dist/pyfory-*-macosx*.whl
+    rm ../dist/pyfory-*-macosx*.whl
+    mv ../dist_repaired/* ../dist/
+    rmdir ../dist_repaired
+  elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
+    # Windows: use delvewheel to bundle dependencies
+    pip install delvewheel
+    mkdir -p ../dist_repaired
+    delvewheel repair ../dist/pyfory-*-win*.whl -w ../dist_repaired/
+    rm ../dist/pyfory-*-win*.whl
+    mv ../dist_repaired/* ../dist/
+    rmdir ../dist_repaired
   fi
-  cd "$ROOT/python"
-  rm -rf "$WHEEL_DIR"
-  mkdir -p "$WHEEL_DIR"
-  for ((i=0; i<${#PYTHONS[@]}; ++i)); do
-    PYTHON=${PYTHONS[i]}
-    ENV="py${VERSIONS[i]}"
-    conda activate "$ENV"
-    python -V
-    git clean -f -f -x -d -e .whl
-    # Ensure bazel select the right version of python
-    bazel clean --expunge
-    install_pyarrow
-    pip install --ignore-installed twine setuptools cython numpy
-    pyarrow_dir=$(python -c "import importlib.util; import os; print(os.path.dirname(importlib.util.find_spec('pyarrow').origin))")
-    # ensure pyarrow is clean
-    rm -rf "$pyarrow_dir"
-    pip install --ignore-installed pyarrow==$pyarrow_version
-    python setup.py clean
-    python setup.py bdist_wheel
-    mv dist/pyfory*.whl "$WHEEL_DIR"
-  done
-  rename_wheels "$WHEEL_DIR"
-  twine check "$WHEEL_DIR"/pyfory*.whl
-  twine upload -r pypi "$WHEEL_DIR"/pyfory*.whl
+  popd
 }
 
 install_pyarrow() {
@@ -191,9 +134,6 @@ case "$1" in
 java) # Deploy jars to maven repository.
   deploy_jars
   ;;
-python) # Deploy wheel to pypi
-  deploy_python
-  ;;
 *)
   echo "Execute command $*"
   "$@"


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org