You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/11/15 08:51:07 UTC
[arrow] 11/27: ARROW-17487: [Python][Packaging][CI] Add support for Python 3.11 (#14499)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch maint-10.0.x
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 60bbf14285a45b38f81ecd79769dbc6cf64d453e
Author: Raúl Cumplido <ra...@gmail.com>
AuthorDate: Fri Nov 4 21:32:09 2022 +0100
ARROW-17487: [Python][Packaging][CI] Add support for Python 3.11 (#14499)
This PR adds jobs to build pyarrow wheels for Python 3.11.
Authored-by: Raúl Cumplido <ra...@gmail.com>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
.../python-wheel-windows-test-vs2017.dockerfile | 5 +-
ci/docker/python-wheel-windows-vs2017.dockerfile | 5 +-
ci/scripts/install_gcs_testbench.sh | 11 +----
ci/scripts/install_python.sh | 7 +--
ci/scripts/python_wheel_macos_build.sh | 4 +-
dev/release/verify-release-candidate.sh | 26 ++---------
dev/tasks/python-wheels/github.osx.arm64.yml | 54 ----------------------
dev/tasks/tasks.yml | 17 ++-----
docker-compose.yml | 2 +-
python/pyarrow/_fs.pyx | 11 ++++-
python/requirements-wheel-test.txt | 15 ++++--
python/setup.py | 1 +
12 files changed, 40 insertions(+), 118 deletions(-)
diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile
index 6013efcd46..a4c836ef4f 100644
--- a/ci/docker/python-wheel-windows-test-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile
@@ -37,7 +37,8 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
ARG python=3.8
RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
(if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
- (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
- (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
+ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts")
RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
RUN python -m pip install -U pip setuptools
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index 247f13a15c..f82a47a057 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -80,8 +80,9 @@ RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
ARG python=3.8
RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
(if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
- (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
- (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.8" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts") & \
+ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.0" && setx PATH "%PATH%;C:\Python311;C:\Python311\Scripts")
RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
RUN python -m pip install -U pip setuptools
diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh
index 9a788fdfd4..0109ea607f 100755
--- a/ci/scripts/install_gcs_testbench.sh
+++ b/ci/scripts/install_gcs_testbench.sh
@@ -34,18 +34,9 @@ case "$(uname -m)" in
;;
esac
-case "$(uname -s)-$(uname -m)" in
- Darwin-arm64)
- # Workaround for https://github.com/grpc/grpc/issues/28387 .
- # Build grpcio instead of using wheel.
- # storage-testbench 0.27.0 pins grpcio to 1.46.1.
- ${PYTHON:-python3} -m pip install --no-binary :all: "grpcio==1.46.1"
- ;;
-esac
-
version=$1
if [[ "${version}" -eq "default" ]]; then
- version="v0.27.0"
+ version="v0.32.0"
fi
${PYTHON:-python3} -m pip install \
diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh
index 7a18cd8324..d64318751c 100755
--- a/ci/scripts/install_python.sh
+++ b/ci/scripts/install_python.sh
@@ -27,8 +27,9 @@ platforms=([windows]=Windows
declare -A versions
versions=([3.7]=3.7.9
[3.8]=3.8.10
- [3.9]=3.9.9
- [3.10]=3.10.1)
+ [3.9]=3.9.13
+ [3.10]=3.10.8
+ [3.11]=3.11.0)
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <platform> <version>"
@@ -45,7 +46,7 @@ full_version=${versions[$2]}
if [ $platform = "MacOSX" ]; then
echo "Downloading Python installer..."
- if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ]; then
+ if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ]; then
fname="python-${full_version}-macos11.pkg"
else
fname="python-${full_version}-macosx10.9.pkg"
diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh
index fd24c58d63..7c7ef7745c 100755
--- a/ci/scripts/python_wheel_macos_build.sh
+++ b/ci/scripts/python_wheel_macos_build.sh
@@ -41,8 +41,6 @@ if [ $arch = "arm64" ]; then
export CMAKE_OSX_ARCHITECTURES="arm64"
elif [ $arch = "x86_64" ]; then
export CMAKE_OSX_ARCHITECTURES="x86_64"
-elif [ $arch = "universal2" ]; then
- export CMAKE_OSX_ARCHITECTURES="x86_64;arm64"
else
echo "Unexpected architecture: $arch"
exit 1
@@ -58,7 +56,7 @@ pip install \
--target $PIP_SITE_PACKAGES \
--platform $PIP_TARGET_PLATFORM \
-r ${source_dir}/python/requirements-wheel-build.txt
-pip install "delocate>=0.9"
+pip install "delocate>=0.10.3"
echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
: ${ARROW_DATASET:=ON}
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
index f874500c8f..bcbde89893 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -1014,7 +1014,7 @@ test_linux_wheels() {
local arch="x86_64"
fi
- local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10}"
+ local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10 3.11}"
local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}"
for python in ${python_versions}; do
@@ -1036,11 +1036,11 @@ test_macos_wheels() {
# apple silicon processor
if [ "$(uname -m)" = "arm64" ]; then
- local python_versions="3.8 3.9 3.10"
+ local python_versions="3.8 3.9 3.10 3.11"
local platform_tags="macosx_11_0_arm64"
local check_flight=OFF
else
- local python_versions="3.7m 3.8 3.9 3.10"
+ local python_versions="3.7m 3.8 3.9 3.10 3.11"
local platform_tags="macosx_10_14_x86_64"
fi
@@ -1062,26 +1062,6 @@ test_macos_wheels() {
${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR}
done
done
-
- # verify arm64 and universal2 wheels using an universal2 python binary
- # the interpreter should be installed from python.org:
- # https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg
- if [ "$(uname -m)" = "arm64" ]; then
- for pyver in 3.9 3.10; do
- local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}"
-
- # create and activate a virtualenv for testing as arm64
- for arch in "arm64" "x86_64"; do
- show_header "Testing Python ${pyver} universal2 wheel on ${arch}"
- VENV_ENV=wheel-${pyver}-universal2-${arch} PYTHON=${python} maybe_setup_virtualenv || continue
- # install pyarrow's universal2 wheel
- pip install pyarrow-${VERSION}-cp${pyver/.}-cp${pyver/.}-macosx_11_0_universal2.whl
- # check the imports and execute the unittests
- INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} \
- arch -${arch} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR}
- done
- done
- fi
}
test_wheels() {
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index a555b28713..2c796d162d 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -87,46 +87,6 @@ jobs:
pip install --upgrade pip wheel
arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build
- {% if arch == "universal2" %}
- - name: Install AMD64 Packages
- env:
- VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
- run: |
- vcpkg install \
- --clean-after-build \
- --x-install-root=${VCPKG_ROOT}/installed \
- --x-manifest-root=arrow/ci/vcpkg \
- --x-feature=flight \
- --x-feature=gcs \
- --x-feature=json \
- --x-feature=parquet \
- --x-feature=s3
-
- - name: Build AMD64 Wheel
- env:
- ARROW_SIMD_LEVEL: "NONE"
- VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
- run: |
- $PYTHON -m venv build-amd64-env
- source build-amd64-env/bin/activate
- pip install --upgrade pip wheel
- arch -x86_64 arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build
-
- - name: Fuse AMD64 and ARM64 wheels
- run: |
- source build-amd64-env/bin/activate
- pip install delocate
-
- amd64_wheel=$(ls arrow/python/repaired_wheels/pyarrow*x86_64.whl)
- arm64_wheel=$(ls arrow/python/repaired_wheels/pyarrow*arm64.whl)
- echo "Fusing ${amd64_wheel} and ${arm64_wheel} into an universal2 wheel..."
- delocate-fuse $amd64_wheel $arm64_wheel -w .
-
- fused_wheel=$(ls *x86_64.whl)
- rm arrow/python/repaired_wheels/*.whl
- mv $fused_wheel arrow/python/repaired_wheels/${fused_wheel/x86_64/universal2}
- {% endif %}
-
- uses: actions/upload-artifact@v3
with:
name: wheel
@@ -148,20 +108,6 @@ jobs:
PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default
arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
- {% if arch == "universal2" %}
- - name: Test Wheel on AMD64
- shell: bash
- env:
- PYTEST_ADDOPTS: "-k 'not test_cancellation'"
- run: |
- $PYTHON -m venv test-amd64-env
- source test-amd64-env/bin/activate
- pip install --upgrade pip wheel
- arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt
- PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default
- arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
- {% endif %}
-
- name: Upload artifacts
shell: bash
run: |
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index bdf53ff1da..57878227b2 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -458,7 +458,8 @@ tasks:
{% for python_version, python_tag, abi_tag in [("3.7", "cp37", "cp37m"),
("3.8", "cp38", "cp38"),
("3.9", "cp39", "cp39"),
- ("3.10", "cp310", "cp310")] %}
+ ("3.10", "cp310", "cp310"),
+ ("3.11", "cp311", "cp311")] %}
{############################## Wheel Linux ##################################}
@@ -519,7 +520,7 @@ tasks:
artifacts:
- pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl
-{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %}
+{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311")] %}
wheel-macos-big-sur-{{ python_tag }}-arm64:
ci: github
template: python-wheels/github.osx.arm64.yml
@@ -529,16 +530,6 @@ tasks:
macos_deployment_target: "11.0"
artifacts:
- pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl
-
- wheel-macos-big-sur-{{ python_tag }}-universal2:
- ci: github
- template: python-wheels/github.osx.arm64.yml
- params:
- arch: universal2
- python_version: "{{ python_version }}"
- macos_deployment_target: "10.14"
- artifacts:
- - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_10_14_universal2.whl
{% endfor %}
{############################ Python sdist ####################################}
@@ -1224,7 +1215,7 @@ tasks:
UBUNTU: 20.04
image: ubuntu-cpp-thread-sanitizer
-{% for python_version in ["3.7", "3.8", "3.9", "3.10"] %}
+{% for python_version in ["3.7", "3.8", "3.9", "3.10", "3.11"] %}
test-conda-python-{{ python_version }}:
ci: github
template: docker-tests/github.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index 57cd03d11d..313a1623ba 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -918,7 +918,7 @@ services:
args:
arch: ${ARCH}
arch_short: ${ARCH_SHORT}
- base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
+ base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2022-10-25-fbea779
vcpkg: ${VCPKG}
python: ${PYTHON}
manylinux: 2014
diff --git a/python/pyarrow/_fs.pyx b/python/pyarrow/_fs.pyx
index e7b028a07d..557c081493 100644
--- a/python/pyarrow/_fs.pyx
+++ b/python/pyarrow/_fs.pyx
@@ -78,6 +78,12 @@ cdef CFileType _unwrap_file_type(FileType ty) except *:
assert 0
+def _file_type_to_string(ty):
+ # Python 3.11 changed str(IntEnum) to return the string representation
+ # of the integer value: https://github.com/python/cpython/issues/94763
+ return f"{ty.__class__.__name__}.{ty._name_}"
+
+
cdef class FileInfo(_Weakrefable):
"""
FileSystem entry info.
@@ -185,9 +191,10 @@ cdef class FileInfo(_Weakrefable):
except ValueError:
return ''
- s = '<FileInfo for {!r}: type={}'.format(self.path, str(self.type))
+ s = (f'<FileInfo for {self.path!r}: '
+ f'type={_file_type_to_string(self.type)}')
if self.is_file:
- s += ', size={}'.format(self.size)
+ s += f', size={self.size}'
s += '>'
return s
diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt
index 1644b2f8bc..dd07f0358d 100644
--- a/python/requirements-wheel-test.txt
+++ b/python/requirements-wheel-test.txt
@@ -8,14 +8,19 @@ pytz
tzdata; sys_platform == 'win32'
numpy==1.19.5; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.7"
-numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.7"
+numpy==1.21.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.7" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.11"
numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9"
-numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9"
-numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64"
+numpy==1.21.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.11"
+numpy==1.21.3; platform_system == "Darwin" and platform_machine == "arm64" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Darwin" and platform_machine == "arm64" and python_version >= "3.11"
numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9"
-numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9"
+numpy==1.21.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.11"
numpy==1.19.5; platform_system == "Windows" and python_version < "3.9"
-numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9"
+numpy==1.21.3; platform_system == "Windows" and python_version >= "3.9" and python_version < "3.11"
+numpy==1.23.4; platform_system == "Windows" and python_version >= "3.11"
pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8"
pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8"
diff --git a/python/setup.py b/python/setup.py
index 73bf19225a..6ca87c7940 100755
--- a/python/setup.py
+++ b/python/setup.py
@@ -768,6 +768,7 @@ setup(
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
+ 'Programming Language :: Python :: 3.11',
],
license='Apache License, Version 2.0',
maintainer='Apache Arrow Developers',