You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/05/07 20:17:30 UTC
[arrow] branch master updated: ARROW-16474: [C++][Packaging] Require Python 3.7 or later
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new d8173f7614 ARROW-16474: [C++][Packaging] Require Python 3.7 or later
d8173f7614 is described below
commit d8173f76145dac07ce1ac10097511ab437dad19f
Author: Sutou Kouhei <ko...@clear-code.com>
AuthorDate: Sun May 8 05:17:05 2022 +0900
ARROW-16474: [C++][Packaging] Require Python 3.7 or later
Closes #13079 from kou/packaging-linux-drop-python-3.6-support
Authored-by: Sutou Kouhei <ko...@clear-code.com>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
cpp/cmake_modules/FindBoostAlt.cmake | 12 +--
cpp/cmake_modules/FindPython3Alt.cmake | 42 ++++----
cpp/src/arrow/python/CMakeLists.txt | 2 +-
dev/release/verify-apt.sh | 12 ++-
dev/release/verify-yum.sh | 1 +
dev/tasks/linux-packages/apache-arrow/Rakefile | 11 +++
.../apache-arrow/apt/ubuntu-bionic/Dockerfile | 1 -
.../linux-packages/apache-arrow/debian/control.in | 108 ++++++++++-----------
dev/tasks/linux-packages/apache-arrow/debian/rules | 13 ++-
.../apache-arrow/yum/almalinux-8/Dockerfile | 8 +-
.../linux-packages/apache-arrow/yum/arrow.spec.in | 16 +--
.../apache-arrow/yum/centos-8-stream/Dockerfile | 8 +-
12 files changed, 127 insertions(+), 107 deletions(-)
diff --git a/cpp/cmake_modules/FindBoostAlt.cmake b/cpp/cmake_modules/FindBoostAlt.cmake
index 1771937125..072e267c1f 100644
--- a/cpp/cmake_modules/FindBoostAlt.cmake
+++ b/cpp/cmake_modules/FindBoostAlt.cmake
@@ -21,15 +21,15 @@ if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
set(Boost_NO_SYSTEM_PATHS ON)
endif()
-set(BoostAlt_FIND_VERSION_OPTIONS)
+set(BoostAlt_FIND_PACKAGE_OPTIONS)
if(BoostAlt_FIND_VERSION)
- list(APPEND BoostAlt_FIND_VERSION_OPTIONS ${BoostAlt_FIND_VERSION})
+ list(APPEND BoostAlt_FIND_PACKAGE_OPTIONS ${BoostAlt_FIND_VERSION})
endif()
if(BoostAlt_FIND_REQUIRED)
- list(APPEND BoostAlt_FIND_VERSION_OPTIONS REQUIRED)
+ list(APPEND BoostAlt_FIND_PACKAGE_OPTIONS REQUIRED)
endif()
if(BoostAlt_FIND_QUIETLY)
- list(APPEND BoostAlt_FIND_VERSION_OPTIONS QUIET)
+ list(APPEND BoostAlt_FIND_PACKAGE_OPTIONS QUIET)
endif()
if(ARROW_BOOST_USE_SHARED)
@@ -38,14 +38,14 @@ if(ARROW_BOOST_USE_SHARED)
set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
set(BUILD_SHARED_LIBS ON)
- find_package(Boost ${BoostAlt_FIND_VERSION_OPTIONS} COMPONENTS system filesystem)
+ find_package(Boost ${BoostAlt_FIND_PACKAGE_OPTIONS} COMPONENTS system filesystem)
set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
unset(BUILD_SHARED_LIBS_KEEP)
else()
# Find static boost headers and libs
# TODO Differentiate here between release and debug builds
set(Boost_USE_STATIC_LIBS ON)
- find_package(Boost ${BoostAlt_FIND_VERSION_OPTIONS} COMPONENTS system filesystem)
+ find_package(Boost ${BoostAlt_FIND_PACKAGE_OPTIONS} COMPONENTS system filesystem)
endif()
if(Boost_FOUND)
diff --git a/cpp/cmake_modules/FindPython3Alt.cmake b/cpp/cmake_modules/FindPython3Alt.cmake
index ab91c7be05..b003bb6a46 100644
--- a/cpp/cmake_modules/FindPython3Alt.cmake
+++ b/cpp/cmake_modules/FindPython3Alt.cmake
@@ -23,16 +23,24 @@
# - PYTHON_OTHER_LIBS
# - NUMPY_INCLUDE_DIRS
+set(Python3Alt_FIND_PACKAGE_OPTIONS)
+set(Python3Alt_NumPy_FIND_PACKAGE_OPTIONS)
+if(Python3Alt_FIND_VERSION)
+ list(APPEND Python3Alt_FIND_PACKAGE_OPTIONS ${Python3Alt_FIND_VERSION})
+endif()
+if(Python3Alt_FIND_REQUIRED)
+ list(APPEND Python3Alt_FIND_PACKAGE_OPTIONS REQUIRED)
+ list(APPEND Python3Alt_NumPy_FIND_PACKAGE_OPTIONS REQUIRED)
+endif()
+if(Python3Alt_FIND_QUIETLY)
+ list(APPEND Python3Alt_FIND_PACKAGE_OPTIONS QUIET)
+ list(APPEND Python3Alt_NumPy_FIND_PACKAGE_OPTIONS QUIET)
+endif()
+
# Need CMake 3.15 or later for Python3_FIND_STRATEGY
if(${CMAKE_VERSION} VERSION_LESS "3.15.0")
- # Use deprecated Python- and NumPy-finding code
- if(Python3Alt_FIND_REQUIRED)
- find_package(PythonLibsNew REQUIRED)
- find_package(NumPy REQUIRED)
- else()
- find_package(PythonLibsNew)
- find_package(NumPy)
- endif()
+ find_package(PythonLibsNew ${Python3Alt_FIND_PACKAGE_OPTIONS})
+ find_package(NumPy ${Python3Alt_NumPy_FIND_PACKAGE_OPTIONS})
find_package_handle_standard_args(
Python3Alt REQUIRED_VARS PYTHON_EXECUTABLE PYTHON_INCLUDE_DIRS NUMPY_INCLUDE_DIRS)
return()
@@ -42,21 +50,11 @@ if(${CMAKE_VERSION} VERSION_LESS "3.18.0" OR ARROW_BUILD_TESTS)
# When building arrow-python-test, we need libpython to be present, so ask for
# the full "Development" component. Also ask for it on CMake < 3.18,
# where "Development.Module" is not available.
- if(Python3Alt_FIND_REQUIRED)
- find_package(Python3
- COMPONENTS Interpreter Development NumPy
- REQUIRED)
- else()
- find_package(Python3 COMPONENTS Interpreter Development NumPy)
- endif()
+ find_package(Python3 ${Python3Alt_FIND_PACKAGE_OPTIONS} COMPONENTS Interpreter
+ Development NumPy)
else()
- if(Python3Alt_FIND_REQUIRED)
- find_package(Python3
- COMPONENTS Interpreter Development.Module NumPy
- REQUIRED)
- else()
- find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
- endif()
+ find_package(Python3 ${Python3Alt_FIND_PACKAGE_OPTIONS}
+ COMPONENTS Interpreter Development.Module NumPy)
endif()
if(NOT Python3_FOUND)
diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
index 7235e2d0fe..34dea64df5 100644
--- a/cpp/src/arrow/python/CMakeLists.txt
+++ b/cpp/src/arrow/python/CMakeLists.txt
@@ -19,7 +19,7 @@
# arrow_python
#
-find_package(Python3Alt REQUIRED)
+find_package(Python3Alt 3.7 REQUIRED)
add_custom_target(arrow_python-all)
add_custom_target(arrow_python)
diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
index 88be807187..184a8f5a94 100755
--- a/dev/release/verify-apt.sh
+++ b/dev/release/verify-apt.sh
@@ -63,6 +63,7 @@ esac
have_flight=yes
have_plasma=yes
+have_python=yes
workaround_missing_packages=()
case "${distribution}-${code_name}" in
debian-*)
@@ -71,6 +72,9 @@ case "${distribution}-${code_name}" in
-e "s/ main$/ main contrib non-free/g" \
/etc/apt/sources.list
;;
+ ubuntu-bionic)
+ have_python=no
+ ;;
esac
if [ "$(arch)" = "aarch64" ]; then
have_plasma=no
@@ -173,9 +177,11 @@ if [ "${have_flight}" = "yes" ]; then
fi
-echo "::group::Test libarrow-python"
-${APT_INSTALL} libarrow-python-dev=${package_version}
-echo "::endgroup::"
+if [ "${have_python}" = "yes" ]; then
+ echo "::group::Test libarrow-python"
+ ${APT_INSTALL} libarrow-python-dev=${package_version}
+ echo "::endgroup::"
+fi
if [ "${have_plasma}" = "yes" ]; then
diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
index d3a0c614db..17c36e4be9 100755
--- a/dev/release/verify-yum.sh
+++ b/dev/release/verify-yum.sh
@@ -73,6 +73,7 @@ case "${distribution}-${distribution_version}" in
cmake_command=cmake3
have_flight=no
have_gandiva=no
+ have_python=no
install_command="yum install -y"
;;
centos-*)
diff --git a/dev/tasks/linux-packages/apache-arrow/Rakefile b/dev/tasks/linux-packages/apache-arrow/Rakefile
index 9dfae955e2..6d99ad63ec 100644
--- a/dev/tasks/linux-packages/apache-arrow/Rakefile
+++ b/dev/tasks/linux-packages/apache-arrow/Rakefile
@@ -116,6 +116,16 @@ class ApacheArrowPackageTask < PackageTask
control.gsub(/@USE_SYSTEM_GRPC@/, use_system_grpc)
end
+ def apt_prepare_debian_control_python(control, target)
+ case target
+ when /\Aubuntu-bionic/
+ enable_python = "#"
+ else
+ enable_python = ""
+ end
+ control.gsub(/@ENABLE_PYTHON@/, enable_python)
+ end
+
def apt_prepare_debian_control_thrift(control, target)
case target
when /\Aubuntu-bionic/
@@ -151,6 +161,7 @@ class ApacheArrowPackageTask < PackageTask
control = apt_prepare_debian_control_cuda_architecture(control, target)
control = apt_prepare_debian_control_c_ares(control, target)
control = apt_prepare_debian_control_grpc(control, target)
+ control = apt_prepare_debian_control_python(control, target)
control = apt_prepare_debian_control_thrift(control, target)
control = apt_prepare_debian_control_utf8proc(control, target)
control = apt_prepare_debian_control_zstd(control, target)
diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
index bbc8a1acb4..e132d6e4eb 100644
--- a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -58,7 +58,6 @@ RUN \
nlohmann-json-dev \
pkg-config \
python3-dev \
- python3-numpy \
python3-pip \
python3-setuptools \
python3-wheel \
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in
index 2cf8a25da4..79af9bbaee 100644
--- a/dev/tasks/linux-packages/apache-arrow/debian/control.in
+++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -32,8 +32,8 @@ Build-Depends:
nvidia-cuda-toolkit [!arm64],
pkg-config,
@USE_SYSTEM_GRPC@ protobuf-compiler-grpc,
- python3-dev,
- python3-numpy,
+@ENABLE_PYTHON@ python3-dev,
+@ENABLE_PYTHON@ python3-numpy,
tzdata,
zlib1g-dev
Build-Depends-Indep: libglib2.0-doc
@@ -92,34 +92,34 @@ Description: Apache Arrow is a data processing library for analysis
.
This package provides C++ library files for Flight RPC system.
-Package: libarrow-python900
-Section: libs
-Architecture: any
-Multi-Arch: same
-Pre-Depends: ${misc:Pre-Depends}
-Depends:
- ${misc:Depends},
- ${shlibs:Depends},
- libarrow900 (= ${binary:Version}),
- python3,
- python3-numpy
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ library files for Python support.
-
-Package: libarrow-python-flight900
-Section: libs
-Architecture: any
-Multi-Arch: same
-Pre-Depends: ${misc:Pre-Depends}
-Depends:
- ${misc:Depends},
- ${shlibs:Depends},
- libarrow-flight900 (= ${binary:Version}),
- libarrow-python900 (= ${binary:Version})
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ library files for Flight and Python support.
+@ENABLE_PYTHON@Package: libarrow-python900
+@ENABLE_PYTHON@Section: libs
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Pre-Depends: ${misc:Pre-Depends}
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@ ${misc:Depends},
+@ENABLE_PYTHON@ ${shlibs:Depends},
+@ENABLE_PYTHON@ libarrow900 (= ${binary:Version}),
+@ENABLE_PYTHON@ python3,
+@ENABLE_PYTHON@ python3-numpy
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ library files for Python support.
+
+@ENABLE_PYTHON@Package: libarrow-python-flight900
+@ENABLE_PYTHON@Section: libs
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Pre-Depends: ${misc:Pre-Depends}
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@ ${misc:Depends},
+@ENABLE_PYTHON@ ${shlibs:Depends},
+@ENABLE_PYTHON@ libarrow-flight900 (= ${binary:Version}),
+@ENABLE_PYTHON@ libarrow-python900 (= ${binary:Version})
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ library files for Flight and Python support.
Package: libarrow-dev
Section: libdevel
@@ -183,30 +183,30 @@ Description: Apache Arrow is a data processing library for analysis
.
This package provides C++ header files for Flight RPC system.
-Package: libarrow-python-dev
-Section: libdevel
-Architecture: any
-Multi-Arch: same
-Depends:
- ${misc:Depends},
- libarrow-dev (= ${binary:Version}),
- libarrow-python900 (= ${binary:Version})
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ header files for Python support.
-
-Package: libarrow-python-flight-dev
-Section: libdevel
-Architecture: any
-Multi-Arch: same
-Depends:
- ${misc:Depends},
- libarrow-flight-dev (= ${binary:Version}),
- libarrow-python-dev (= ${binary:Version}),
- libarrow-python-flight900 (= ${binary:Version})
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ header files for Flight and Python support.
+@ENABLE_PYTHON@Package: libarrow-python-dev
+@ENABLE_PYTHON@Section: libdevel
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@ ${misc:Depends},
+@ENABLE_PYTHON@ libarrow-dev (= ${binary:Version}),
+@ENABLE_PYTHON@ libarrow-python900 (= ${binary:Version})
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ header files for Python support.
+
+@ENABLE_PYTHON@Package: libarrow-python-flight-dev
+@ENABLE_PYTHON@Section: libdevel
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@ ${misc:Depends},
+@ENABLE_PYTHON@ libarrow-flight-dev (= ${binary:Version}),
+@ENABLE_PYTHON@ libarrow-python-dev (= ${binary:Version}),
+@ENABLE_PYTHON@ libarrow-python-flight900 (= ${binary:Version})
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ header files for Flight and Python support.
Package: libgandiva900
Section: libs
diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
index 9f686994d5..8b4b67f15a 100755
--- a/dev/tasks/linux-packages/apache-arrow/debian/rules
+++ b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -21,22 +21,33 @@ override_dh_auto_configure:
ARROW_CUDA=OFF; \
ARROW_PLASMA=OFF; \
fi; \
+ if python3 -c 'import numpy' > /dev/null 2>&1; then \
+ ARROW_PYTHON=ON; \
+ else \
+ ARROW_PYTHON=OFF; \
+ fi; \
dh_auto_configure \
--sourcedirectory=cpp \
--builddirectory=cpp_build \
--buildsystem=cmake+ninja \
-- \
+ -DARROW_COMPUTE=ON \
+ -DARROW_CSV=ON \
-DARROW_CUDA=$${ARROW_CUDA} \
+ -DARROW_DATASET=ON \
+ -DARROW_FILESYSTEM=ON \
-DARROW_FLIGHT=ON \
-DARROW_GANDIVA=ON \
-DARROW_GANDIVA_JAVA=OFF \
-DARROW_GCS=ON \
+ -DARROW_HDFS=ON \
+ -DARROW_JSON=ON \
-DARROW_MIMALLOC=ON \
-DARROW_ORC=ON \
-DARROW_PACKAGE_KIND=deb \
-DARROW_PARQUET=ON \
-DARROW_PLASMA=$${ARROW_PLASMA} \
- -DARROW_PYTHON=ON \
+ -DARROW_PYTHON=$${ARROW_PYTHON} \
-DARROW_S3=ON \
-DARROW_USE_CCACHE=OFF \
-DARROW_WITH_BROTLI=ON \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
index f45398a6e1..01c9662488 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
@@ -51,10 +51,10 @@ RUN \
ninja-build \
openssl-devel \
pkg-config \
- python3 \
- python3-devel \
- python3-numpy \
- python3-pip \
+ python39 \
+ python39-devel \
+ python39-numpy \
+ python39-pip \
re2-devel \
# rapidjson-devel \
rpmdevtools \
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
index e3ae52bf5c..7884af7bb9 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
+++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in
@@ -30,12 +30,6 @@
if [ %{rhel} -lt 8 ]; then \
echo 3; \
fi)
-%define python_version %( \
- if [ %{rhel} -eq 7 ]; then \
- echo 36; \
- else \
- echo 3; \
- fi)
%define lz4_requirement %( \
if [ %{_amzn} -eq 0 ]; then \
@@ -48,7 +42,7 @@
%define use_gcs (%{rhel} >= 8 || %{is_amazon_linux})
%define use_gflags (!%{is_amazon_linux})
%define use_mimalloc (%{rhel} >= 8)
-%define use_python (!%{is_amazon_linux})
+%define use_python (%{rhel} >= 8)
# TODO: Enable this. This works on local but is fragile on GitHub Actions and
# Travis CI.
# %define use_s3 (%{rhel} >= 8)
@@ -99,8 +93,8 @@ BuildRequires: ninja-build
BuildRequires: openssl-devel
BuildRequires: pkgconfig
%if %{use_python}
-BuildRequires: python%{python_version}-devel
-BuildRequires: python%{python_version}-numpy
+BuildRequires: python39-devel
+BuildRequires: python39-numpy
%endif
%if %{have_rapidjson}
BuildRequires: rapidjson-devel
@@ -418,7 +412,7 @@ Libraries and header files for Gandiva.
Summary: Python integration library for Apache Arrow
License: Apache-2.0
Requires: %{name}%{major_version}-libs = %{version}-%{release}
-Requires: python%{python_version}-numpy
+Requires: python39-numpy
%description -n %{name}%{major_version}-python-libs
This package contains the Python integration library for Apache Arrow.
@@ -433,7 +427,7 @@ Summary: Libraries and header files for Python integration library for Apache Ar
License: Apache-2.0
Requires: %{name}%{major_version}-python-libs = %{version}-%{release}
Requires: %{name}-devel = %{version}-%{release}
-Requires: python%{python_version}-devel
+Requires: python39-devel
%description python-devel
Libraries and header files for Python integration library for Apache Arrow.
diff --git a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
index c84a1ea43a..c9e8109f78 100644
--- a/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
+++ b/dev/tasks/linux-packages/apache-arrow/yum/centos-8-stream/Dockerfile
@@ -51,10 +51,10 @@ RUN \
ninja-build \
openssl-devel \
pkg-config \
- python3 \
- python3-devel \
- python3-numpy \
- python3-pip \
+ python39 \
+ python39-devel \
+ python39-numpy \
+ python39-pip \
re2-devel \
rapidjson-devel \
rpmdevtools \