You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by mg...@apache.org on 2026/07/03 11:06:15 UTC
(kudu) branch master updated: [Python] modernize packaging and extend version support
This is an automated email from the ASF dual-hosted git repository.
martongreber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new d55aacb62 [Python] modernize packaging and extend version support
d55aacb62 is described below
commit d55aacb623be00756ad227d8f624c345da6e5f08
Author: Marton Greber <gr...@gmail.com>
AuthorDate: Tue Mar 31 11:10:34 2026 +0200
[Python] modernize packaging and extend version support
Add pyproject.toml with PEP 517 build-system table, versioning Cython
per Python version. Drop setup_prerequisites.py. Package metadata
remains in setup.py; KUDU-3764 tracks moving it.
Pin per-version dev dependencies (pytest, setuptools, numpy, pandas),
and add compat aliases NP_OBJECT/NP_BOOL for NumPy >= 1.24 API removals.
Drop PY26/unittest2, extend classifiers to Python 3.12-3.14, and
fix sdist import test to run outside the source tree.
Tested client and packaging tests on:
- Ubuntu 22: Python 2.7, 3.7-3.14
- Ubuntu 18: Python 2.7, 3.7, 3.8
Change-Id: I940cda86ac30d053ba284155a038f9e7b4e363fc
Reviewed-on: http://gerrit.cloudera.org:8080/24200
Reviewed-by: Gabriella Lotz <lo...@gmail.com>
Tested-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Zoltan Chovan <zc...@cloudera.com>
---
RELEASING.adoc | 3 +-
build-support/jenkins/build-and-test.sh | 28 +++++++--------
build-support/python-compat/test-packaging.sh | 9 +++--
examples/python/basic-python-example/README.md | 4 +--
python/MANIFEST.in | 2 +-
python/Makefile | 11 +++---
python/README.md | 9 ++---
python/kudu/compat.py | 27 +++++++++------
python/kudu/tests/test_scanner.py | 23 +++++++------
python/kudu/tests/test_util.py | 2 +-
python/pyproject.toml | 9 +++++
python/requirements_dev.txt | 47 ++++++++++++++++----------
python/setup.py | 17 ++++++----
python/setup_prerequisites.py | 40 ----------------------
python/test-python-examples.sh | 5 +--
15 files changed, 115 insertions(+), 121 deletions(-)
diff --git a/RELEASING.adoc b/RELEASING.adoc
index 88afb2450..30cc6383e 100644
--- a/RELEASING.adoc
+++ b/RELEASING.adoc
@@ -375,7 +375,8 @@ in its summary tab
# Prepare and sign the python source distribution.
cd python
rm -rf dist/*
- python setup.py sdist
+ pip install build
+ python -m build --sdist
gpg --detach-sign -a dist/kudu-python-1.x.y.tar.gz
# Upload the distribution to PyPi using twine.
pip install twine
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index 83309f409..4dff8d990 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -721,11 +721,11 @@ if [ "$BUILD_PYTHON" == "1" ]; then
pip $PIP_FLAGS install $PIP_INSTALL_FLAGS -r requirements_dev.txt
- # Delete old Cython extensions to force them to be rebuilt.
- rm -Rf build kudu_python.egg-info kudu/*.so
-
- # Build the Python bindings. This assumes we run this script from base dir.
- CC=$CLANG CXX=$CLANG++ python setup.py build_ext
+ # Build and install the Python bindings (Cython is installed automatically via pyproject.toml).
+ # -e (editable) is required: plain 'pip install .' copies the source to a temp directory where
+ # python/version.txt (a symlink to ../version.txt) becomes a dangling symlink and setup.py fails.
+ # Editable installs build in-place in the source directory, so the symlink resolves correctly.
+ CC=$CLANG CXX=$CLANG++ pip install -e .
# A testing environment might have HTTP/HTTPS proxy configured to proxy
# requests even for 127.0.0.0/8 network or other quirks. Since some Python
@@ -740,8 +740,8 @@ if [ "$BUILD_PYTHON" == "1" ]; then
# Run the Python tests. This may also involve some compiler work.
set +e
- if ! CC=$CLANG CXX=$CLANG++ python setup.py test \
- --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \
+ if ! CC=$CLANG CXX=$CLANG++ python -m pytest kudu \
+ --junit-xml=$TEST_LOGDIR/python_client.xml \
2> $TEST_LOGDIR/python_client.log ; then
TESTS_FAILED=1
FAILURES="$FAILURES"$'Python tests failed\n'
@@ -776,11 +776,11 @@ if [ "$BUILD_PYTHON3" == "1" ]; then
pip $PIP_FLAGS install $PIP_INSTALL_FLAGS -r requirements_dev.txt
- # Delete old Cython extensions to force them to be rebuilt.
- rm -Rf build kudu_python.egg-info kudu/*.so
-
- # Build the Python bindings. This assumes we run this script from base dir.
- CC=$CLANG CXX=$CLANG++ python setup.py build_ext
+ # Build and install the Python bindings (Cython is installed automatically via pyproject.toml).
+ # -e (editable) is required: plain 'pip install .' copies the source to a temp directory where
+ # python/version.txt (a symlink to ../version.txt) becomes a dangling symlink and setup.py fails.
+ # Editable installs build in-place in the source directory, so the symlink resolves correctly.
+ CC=$CLANG CXX=$CLANG++ pip install -e .
set +e
# A testing environment might have HTTP/HTTPS proxy configured to proxy
@@ -795,8 +795,8 @@ if [ "$BUILD_PYTHON3" == "1" ]; then
unset https_proxy
# Run the Python tests. This may also involve some compiler work.
- if ! CC=$CLANG CXX=$CLANG++ python setup.py test \
- --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \
+ if ! CC=$CLANG CXX=$CLANG++ python -m pytest kudu \
+ --junit-xml=$TEST_LOGDIR/python3_client.xml \
2> $TEST_LOGDIR/python3_client.log ; then
TESTS_FAILED=1
FAILURES="$FAILURES"$'Python 3 tests failed\n'
diff --git a/build-support/python-compat/test-packaging.sh b/build-support/python-compat/test-packaging.sh
index 7a2695753..8ba900b0a 100755
--- a/build-support/python-compat/test-packaging.sh
+++ b/build-support/python-compat/test-packaging.sh
@@ -81,9 +81,12 @@ run_one() {
"${VIRTUALENV_CMD[@]}" -p "${py_bin}" "${build_venv}" >/dev/null
source "${build_venv}/bin/activate"
- make requirements
+ # virtualenv is required as a fallback for stripped Python installs (e.g. Python 3.7 on
+ # Debian/Ubuntu) that lack ensurepip. 'python -m build' uses venv+ensurepip by default to
+ # create an isolated build environment, and falls back to virtualenv when ensurepip is absent.
+ pip install build virtualenv
rm -rf dist/
- python setup.py sdist --quiet
+ python -m build --sdist --outdir dist/
local sdist
sdist="$(ls -1t dist/*.tar.gz 2>/dev/null | head -1)"
@@ -108,7 +111,7 @@ run_one() {
"${VIRTUALENV_CMD[@]}" -p "${py_bin}" "${install_venv}" >/dev/null
source "${install_venv}/bin/activate"
- if pip install "${sdist}" && python -c "import kudu; print('kudu', kudu.__version__)"; then
+ if pip install "${sdist}" && (cd /tmp && python -c "import kudu; print('kudu', kudu.__version__)"); then
log "pip list:"
pip list
ok "Python ${ver} — sdist round-trip OK."
diff --git a/examples/python/basic-python-example/README.md b/examples/python/basic-python-example/README.md
index dca0d05ef..1b405b5c3 100644
--- a/examples/python/basic-python-example/README.md
+++ b/examples/python/basic-python-example/README.md
@@ -39,9 +39,7 @@ export LD_LIBRARY_PATH=$KUDU_HOME/build/latest/lib/exported
# For OS X
export DYLD_LIBRARY_PATH=$KUDU_HOME/build/latest/lib/exported
pushd $KUDU_HOME/python
-pip install -r requirements.txt
-python setup.py build_ext --inplace
-python setup.py install
+pip install -e .
popd
```
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
index 70db1cabc..4f022e057 100644
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@@ -2,7 +2,7 @@ include LICENSE.txt
include MANIFEST.in
include README.md
include setup.py
-include setup_prerequisites.py
+include pyproject.toml
include version.txt
include requirements.txt
diff --git a/python/Makefile b/python/Makefile
index 775f09dc4..4af10f479 100644
--- a/python/Makefile
+++ b/python/Makefile
@@ -12,20 +12,21 @@
.PHONY: all build requirements test install clean test-examples
+# Use editable installs (-e): plain 'pip install .' copies sources to a temp dir
+# where the version.txt symlink becomes dangling under PEP 517 build isolation.
build:
- python setup.py build_ext --inplace
+ pip install -e .
all: build
requirements:
- pip install -r requirements.txt
pip install -r requirements_dev.txt
-test:
+test: build
ifdef TEST
python -m unittest $(TEST)
else
- python setup.py test
+ python -m pytest
endif
# Run Python client examples against a local Kudu cluster.
@@ -53,4 +54,4 @@ clean:
rm -f kudu/version.py
install:
- python setup.py install
\ No newline at end of file
+ pip install -e .
diff --git a/python/README.md b/python/README.md
index 03233aae6..844d329e2 100644
--- a/python/README.md
+++ b/python/README.md
@@ -59,9 +59,7 @@ pip install kudu-python
```bash
cd /path/to/kudu/python # Navigate to the python directory if not already there
-pip install -r requirements.txt
-python setup.py sdist
-pip install dist/kudu-python-*.tar.gz
+pip install -e .
```
## Building for Development
@@ -80,15 +78,14 @@ This variable is required by various scripts and tools in the project. Make sure
```bash
cd $KUDU_HOME/python # Navigate to the python directory if not already there
-pip install -r requirements.txt
pip install -r requirements_dev.txt
-python setup.py build_ext --inplace
+pip install -e .
```
## Run All Tests
```bash
-python setup.py test
+python -m pytest
```
## Run Single Test
diff --git a/python/kudu/compat.py b/python/kudu/compat.py
index 6f8b415ce..fd33ccec6 100644
--- a/python/kudu/compat.py
+++ b/python/kudu/compat.py
@@ -19,6 +19,7 @@
import itertools
import contextlib
+import unittest
try:
import numpy as np
except ImportError:
@@ -30,18 +31,8 @@ import six
from six import BytesIO, StringIO, string_types as py_string
-PY26 = sys.version_info[:2] == (2, 6)
PY2 = sys.version_info[0] == 2
-try:
- if PY26:
- import unittest2 as unittest
- else:
- import unittest
-except ImportError:
- # If we can't import 'unittest', then our tests won't be able
- # to run. But, that's fine.
- pass
if PY2:
import cPickle
@@ -110,3 +101,19 @@ else:
integer_types = six.integer_types
if np is not None:
integer_types += (np.integer,)
+
+# np.object and np.bool were deprecated in NumPy 1.20 and removed in 1.24.
+# See: https://github.com/numpy/numpy/releases/tag/v1.24.0
+# Use try/except so the aliases work on any installed NumPy version.
+if np is not None:
+ try:
+ NP_OBJECT = np.object # NumPy < 1.24
+ except AttributeError:
+ NP_OBJECT = object # NumPy >= 1.24: use the plain Python builtin
+ try:
+ NP_BOOL = np.bool # NumPy < 1.24
+ except AttributeError:
+ NP_BOOL = np.bool_ # NumPy >= 1.24: use the proper NumPy scalar
+else:
+ NP_OBJECT = object
+ NP_BOOL = bool
diff --git a/python/kudu/tests/test_scanner.py b/python/kudu/tests/test_scanner.py
index c4d17839b..4cfff63af 100644
--- a/python/kudu/tests/test_scanner.py
+++ b/python/kudu/tests/test_scanner.py
@@ -20,6 +20,7 @@ from __future__ import division
from kudu.tests.util import TestScanBase
from kudu.tests.common import KuduTestBase, TimeoutError
+from kudu.compat import NP_OBJECT, NP_BOOL
import kudu
import datetime
import time
@@ -347,25 +348,25 @@ class TestScanner(TestScanBase):
if kudu.CLIENT_SUPPORTS_DECIMAL:
self.assertEqual(types[0], np.int64)
self.assertEqual(types[1], 'datetime64[ns, UTC]')
- self.assertEqual(types[2], np.object)
- self.assertEqual(types[3], np.object)
- self.assertEqual(types[4], np.bool)
+ self.assertEqual(types[2], NP_OBJECT)
+ self.assertEqual(types[3], NP_OBJECT)
+ self.assertEqual(types[4], NP_BOOL)
self.assertEqual(types[5], np.float64)
self.assertEqual(types[6], np.int8)
- self.assertEqual(types[7], np.object)
- self.assertEqual(types[8], np.object)
- self.assertEqual(types[9], np.object)
+ self.assertEqual(types[7], NP_OBJECT)
+ self.assertEqual(types[8], NP_OBJECT)
+ self.assertEqual(types[9], NP_OBJECT)
self.assertEqual(types[10], np.float32)
else:
self.assertEqual(types[0], np.int64)
self.assertEqual(types[1], 'datetime64[ns, UTC]')
- self.assertEqual(types[2], np.object)
- self.assertEqual(types[3], np.bool)
+ self.assertEqual(types[2], NP_OBJECT)
+ self.assertEqual(types[3], NP_BOOL)
self.assertEqual(types[4], np.float64)
self.assertEqual(types[5], np.int8)
- self.assertEqual(types[6], np.object)
- self.assertEqual(types[7], np.object)
- self.assertEqual(types[8], np.object)
+ self.assertEqual(types[6], NP_OBJECT)
+ self.assertEqual(types[7], NP_OBJECT)
+ self.assertEqual(types[8], NP_OBJECT)
self.assertEqual(types[9], np.float32)
@pytest.mark.skipif(not (kudu.CLIENT_SUPPORTS_PANDAS),
diff --git a/python/kudu/tests/test_util.py b/python/kudu/tests/test_util.py
index 0d649fc70..b8dbb0834 100644
--- a/python/kudu/tests/test_util.py
+++ b/python/kudu/tests/test_util.py
@@ -18,7 +18,7 @@
from __future__ import division
-from kudu.compat import unittest
+import unittest
from kudu.util import *
diff --git a/python/pyproject.toml b/python/pyproject.toml
new file mode 100644
index 000000000..7bc5ef3b8
--- /dev/null
+++ b/python/pyproject.toml
@@ -0,0 +1,9 @@
+[build-system]
+requires = [
+ "setuptools",
+ "wheel",
+ # Cython 3.x requires Python >= 3.7; use the legacy 0.29.x series for 2.7/3.6.
+ "Cython==0.29.37; python_version < '3.7'",
+ "Cython==3.0.11; python_version >= '3.7'",
+]
+build-backend = "setuptools.build_meta"
diff --git a/python/requirements_dev.txt b/python/requirements_dev.txt
index 61d90f918..cd690298a 100644
--- a/python/requirements_dev.txt
+++ b/python/requirements_dev.txt
@@ -1,30 +1,41 @@
-# For Python 2.7:
-# - Use the highest possible version of each package that still supports Python 2.7.
+# Version pinning strategy:
+# 1. Start with the highest Python version we support and pin the latest available package
+# version for it. If that version also supports older Python versions, it covers a range
+# (e.g. "python_version >= X").
+# 2. Work backwards through older Python versions, pinning the highest package version that
+# is still compatible with each one.
#
-# For Python 3.6 and above:
-# - Use the latest version of each package that still supports Python 3.6,
-# except for numpy and pandas, which have version restrictions to ensure compatibility with
-# macOS during testing.
+# This means that when a new Python version is added, only the last (open-ended range) entry
+# for each package typically needs to be updated.
pytest==4.6.11; python_version == "2.7"
-pytest==6.2.5; python_version >= "3.6"
+pytest==6.2.5; python_version == "3.6"
+pytest==7.4.4; python_version == "3.7"
+pytest==8.3.5; python_version == "3.8"
+pytest==8.4.2; python_version == "3.9"
+pytest==9.0.3; python_version >= "3.10"
pytest-timeout==1.4.2; python_version == "2.7"
-pytest-timeout==2.1.0; python_version >= "3.6"
-
-pytest-runner<5.3.0; python_version == "2.7"
-pytest-runner==5.3.2; python_version >= "3.6"
-
-unittest2==1.1.0
+pytest-timeout==2.1.0; python_version == "3.6"
+pytest-timeout==2.4.0; python_version > "3.6"
setuptools==44.1.1; python_version == "2.7"
-setuptools==58.5.3; python_version >= "3.6"
+setuptools==58.5.3; python_version == "3.6"
+setuptools==59.8.0; python_version == "3.7"
+setuptools==69.5.1; python_version == "3.8"
+setuptools==82.0.1; python_version >= "3.9"
-pandas==0.23.4; python_version == "2.7"
+pandas==0.24.2; python_version == "2.7"
pandas==1.1.5; python_version >= "3.6" and python_version < "3.8"
-pandas==1.5.3; python_version >= "3.8"
+pandas==1.5.3; python_version == "3.8"
+# Last 2.x (below 3.0); avoids pandas 3 defaults that break scanner tests (CoW).
+pandas==2.3.3; python_version >= "3.9"
numpy==1.16.6; python_version == "2.7"
-numpy==1.19.5; python_version >= "3.6" and python_version < "3.8"
-numpy==1.23.5; python_version >= "3.8"
+numpy==1.19.5; python_version == "3.6"
+numpy==1.21.6; python_version == "3.7"
+numpy==1.24.4; python_version == "3.8"
+numpy==2.0.2; python_version == "3.9"
+numpy==2.2.6; python_version == "3.10"
+numpy==2.4.4; python_version >= "3.11"
diff --git a/python/setup.py b/python/setup.py
index f2b7d21f2..7397451c7 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -18,18 +18,16 @@
# specific language governing permissions and limitations
# under the License.
-# setup-prerequisites must be the very first import in this file!
-import setup_prerequisites
+import sys
+import os
from Cython.Distutils import build_ext
from Cython.Build import cythonize
import Cython
-import sys
from setuptools import setup
from distutils.command.clean import clean as _clean
from distutils.extension import Extension
-import os
import re
import subprocess
@@ -165,7 +163,12 @@ for submodule_name in ext_submodules:
runtime_library_dirs=RT_LIBRARY_DIRS)
extensions.append(ext)
-extensions = cythonize(extensions)
+# Match legacy implicit semantics (Cython previously defaulted to language_level 2).
+# language_level 3 rejects patterns used throughout client.pyx (e.g. del on C++ pointers).
+extensions = cythonize(
+ extensions,
+ compiler_directives={'language_level': 2},
+)
write_version_py(VERSION)
@@ -189,6 +192,9 @@ CLASSIFIERS = [
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
+ 'Programming Language :: Python :: 3.12',
+ 'Programming Language :: Python :: 3.13',
+ 'Programming Language :: Python :: 3.14',
'Programming Language :: Cython'
]
@@ -212,5 +218,4 @@ setup(
maintainer="Apache Kudu team",
maintainer_email="dev@kudu.apache.org",
url=URL,
- test_suite="kudu.tests"
)
diff --git a/python/setup_prerequisites.py b/python/setup_prerequisites.py
deleted file mode 100644
index b3e34f079..000000000
--- a/python/setup_prerequisites.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Cython is a pre-requirement to be able to run this setup.py script.
-# By putting this part before the Cython imports there is no need to
-# install Cython manually before the installation.
-
-import sys
-import subprocess
-import pkg_resources
-
-cython_version = "Cython==0.29.37"
-
-# Function to install a package via pip
-def install(package):
- subprocess.check_call([sys.executable, "-m", "pip", "install", package])
-
-# Ensure the correct version of Cython is installed before proceeding
-try:
- pkg_resources.require(cython_version)
-except (pkg_resources.DistributionNotFound, pkg_resources.VersionConflict):
- sys.stdout.write("Installing Cython...")
- install(cython_version)
diff --git a/python/test-python-examples.sh b/python/test-python-examples.sh
index 1acfda4b8..49e290fd4 100755
--- a/python/test-python-examples.sh
+++ b/python/test-python-examples.sh
@@ -127,9 +127,10 @@ test_with_python_version() {
virtualenv -p "$python_version" "$venv_dir"
source "$venv_dir/bin/activate"
- pip install -r "$PYTHON_DIR/requirements.txt"
make clean
- make install
+ # Use editable install: plain 'pip install .' copies sources to a temp dir where
+ # version.txt (a symlink to ../version.txt) becomes dangling and setup.py fails.
+ pip install -e "$PYTHON_DIR"
echo "Running Python examples with $python_version..."