You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2024/09/09 16:29:02 UTC
(arrow) branch main updated: GH-43536: [Python][CI] Add a Crossbow job with the free-threaded build (#43671)
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new d28e542ef4 GH-43536: [Python][CI] Add a Crossbow job with the free-threaded build (#43671)
d28e542ef4 is described below
commit d28e542ef4dafcbb37abc5501154ee45b1b16b6f
Author: Lysandros Nikolaou <li...@gmail.com>
AuthorDate: Mon Sep 9 19:28:57 2024 +0300
GH-43536: [Python][CI] Add a Crossbow job with the free-threaded build (#43671)
### Rationale for this change
Testing with the free-threaded build is required for adding support for it. (see #43536)
### What changes are included in this PR?
- Add a Docker build with the CPython free-threaded build from deadsnakes.
- Add a Crossbow job to run said Docker build with Python 3.13t
### Are there any user-facing changes?
No.
* GitHub Issue: #43536
Lead-authored-by: Lysandros Nikolaou <li...@gmail.com>
Co-authored-by: Antoine Pitrou <an...@python.org>
Co-authored-by: Sutou Kouhei <ko...@cozmixng.org>
Signed-off-by: Antoine Pitrou <an...@python.org>
---
.../linux-apt-python-313-freethreading.dockerfile | 59 ++++++++++++++++++++++
dev/tasks/tasks.yml | 8 +++
docker-compose.yml | 27 ++++++++++
python/CMakeLists.txt | 7 +++
4 files changed, 101 insertions(+)
diff --git a/ci/docker/linux-apt-python-313-freethreading.dockerfile b/ci/docker/linux-apt-python-313-freethreading.dockerfile
new file mode 100644
index 0000000000..f5505e67f0
--- /dev/null
+++ b/ci/docker/linux-apt-python-313-freethreading.dockerfile
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+RUN apt-get update -y -q && \
+ apt install -y -q --no-install-recommends software-properties-common gpg-agent && \
+ add-apt-repository -y ppa:deadsnakes/ppa && \
+ apt-get update -y -q && \
+ apt install -y -q --no-install-recommends python3.13-dev python3.13-nogil python3.13-venv && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists*
+
+COPY python/requirements-build.txt \
+ python/requirements-test.txt \
+ /arrow/python/
+
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install -U pip setuptools wheel
+RUN ${ARROW_PYTHON_VENV}/bin/python -m pip install \
+ --pre \
+ --prefer-binary \
+ --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \
+ -r arrow/python/requirements-build.txt \
+ -r arrow/python/requirements-test.txt
+
+# We want to run the PyArrow test suite with the GIL disabled, but cffi
+# (more precisely, the `_cffi_backend` module) currently doesn't declare
+# itself safe to run without the GIL.
+# Therefore set PYTHON_GIL to 0.
+ENV ARROW_ACERO=ON \
+ ARROW_BUILD_STATIC=OFF \
+ ARROW_BUILD_TESTS=OFF \
+ ARROW_BUILD_UTILITIES=OFF \
+ ARROW_COMPUTE=ON \
+ ARROW_CSV=ON \
+ ARROW_DATASET=ON \
+ ARROW_FILESYSTEM=ON \
+ ARROW_GDB=ON \
+ ARROW_HDFS=ON \
+ ARROW_JSON=ON \
+ ARROW_USE_GLOG=OFF \
+ PYTHON_GIL=0
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 9ded6ee41a..c1c15a3ff7 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1257,6 +1257,14 @@ tasks:
PYTHON: "3.10"
image: conda-python-cython2
+ test-ubuntu-22.04-python-313-freethreading:
+ ci: github
+ template: docker-tests/github.linux.yml
+ params:
+ env:
+ UBUNTU: 22.04
+ image: ubuntu-python-313-freethreading
+
test-debian-12-python-3-amd64:
ci: github
template: docker-tests/github.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index 36cf150f25..8721eef524 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -149,6 +149,7 @@ x-hierarchy:
- ubuntu-lint
- ubuntu-python
- ubuntu-python-sdist-test
+ - ubuntu-python-313-freethreading
- ubuntu-r
- ubuntu-r-only-r
- ubuntu-cpp-bundled
@@ -1085,6 +1086,32 @@ services:
/arrow/ci/scripts/cpp_build.sh /arrow /build &&
/arrow/ci/scripts/python_sdist_test.sh /arrow"
+ ############################ Python free-threading ##########################
+
+ ubuntu-python-313-freethreading:
+ # Usage:
+ # docker-compose build ubuntu-cpp
+ # docker-compose build ubuntu-python-313-freethreading
+ # docker-compose run --rm ubuntu-python-313-freethreading
+ # Parameters:
+ # ARCH: amd64, arm64v8, ...
+ # UBUNTU: 20.04, 22.04, 24.04
+ image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading
+ build:
+ context: .
+ dockerfile: ci/docker/linux-apt-python-313-freethreading.dockerfile
+ cache_from:
+ - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading
+ args:
+ base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
+ shm_size: *shm-size
+ environment:
+ <<: [*common, *ccache]
+ # Bundled build of OpenTelemetry needs a git client
+ ARROW_WITH_OPENTELEMETRY: "OFF"
+ volumes: *ubuntu-volumes
+ command: *python-command
+
############################ Python wheels ##################################
# See available versions at:
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index eda4ff4ca5..912719b20f 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -21,6 +21,13 @@
cmake_minimum_required(VERSION 3.16)
project(pyarrow)
+# This is needed for 3.13 free-threading. CMake used to add Python
+# include directories with `-isystem`, which led to some Python-internal
+# includes to resolve to normal 3.13 includes (cause -isystem includes
+# are searched after system directories), instead of 3.13-freethreading,
+# which in turn meant that Py_GIL_DISABLED was not set.
+set(CMAKE_NO_SYSTEM_FROM_IMPORTED ON)
+
set(PYARROW_VERSION "18.0.0-SNAPSHOT")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" PYARROW_BASE_VERSION "${PYARROW_VERSION}")