You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/11/15 08:51:04 UTC
[arrow] 08/27: ARROW-18054: [Python][CI] Enable Cython tests on windows wheels (#13552)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch maint-10.0.x
in repository https://gitbox.apache.org/repos/asf/arrow.git
commit 59809769f59d115847cc3fa5025dc9110d6cbb6b
Author: Raúl Cumplido <ra...@gmail.com>
AuthorDate: Tue Oct 25 09:39:25 2022 +0200
ARROW-18054: [Python][CI] Enable Cython tests on windows wheels (#13552)
Authored-by: Raúl Cumplido <ra...@gmail.com>
Signed-off-by: Antoine Pitrou <an...@python.org>
---
.../python-wheel-windows-test-vs2017.dockerfile | 43 ++++++++++++++++++++++
ci/scripts/python_wheel_windows_test.bat | 2 +-
docker-compose.yml | 9 ++++-
python/pyarrow/tests/test_cython.py | 26 ++++++++++---
4 files changed, 72 insertions(+), 8 deletions(-)
diff --git a/ci/docker/python-wheel-windows-test-vs2017.dockerfile b/ci/docker/python-wheel-windows-test-vs2017.dockerfile
new file mode 100644
index 0000000000..6013efcd46
--- /dev/null
+++ b/ci/docker/python-wheel-windows-test-vs2017.dockerfile
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
+# when you update this file.
+
+# based on mcr.microsoft.com/windows/servercore:ltsc2019
+# contains choco and vs2017 preinstalled
+FROM abrarov/msvc-2017:2.11.0
+
+# Add unix tools to path
+RUN setx path "%path%;C:\Program Files\Git\usr\bin"
+
+# Remove previous installations of python from the base image
+# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
+# and the msi installers are failing to remove pip and tcl/tk "products" making
+# the subsequent choco python installation step failing for installing python
+# version 3.9.* due to existing python version
+RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \
+ rm -rf Python*
+
+# Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0)
+ARG python=3.8
+RUN (if "%python%"=="3.7" setx PYTHON_VERSION "3.7.9" && setx PATH "%PATH%;C:\Python37;C:\Python37\Scripts") & \
+ (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH "%PATH%;C:\Python38;C:\Python38\Scripts") & \
+ (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.7" && setx PATH "%PATH%;C:\Python39;C:\Python39\Scripts") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.2" && setx PATH "%PATH%;C:\Python310;C:\Python310\Scripts")
+RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
+RUN python -m pip install -U pip setuptools
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
index 2b7aad3abe..2abf8ca50f 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -17,7 +17,7 @@
@echo on
-set PYARROW_TEST_CYTHON=OFF
+set PYARROW_TEST_CYTHON=ON
set PYARROW_TEST_DATASET=ON
set PYARROW_TEST_FLIGHT=ON
set PYARROW_TEST_GANDIVA=OFF
diff --git a/docker-compose.yml b/docker-compose.yml
index 1c3813757f..57cd03d11d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -981,10 +981,15 @@ services:
target: "C:/arrow"
command: arrow\\ci\\scripts\\python_wheel_windows_build.bat
- # doesn't exit properly on fail
python-wheel-windows-test:
- image: python:${PYTHON}-windowsservercore-1809
+ image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2017-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ build:
+ args:
+ python: ${PYTHON}
+ context: .
+ dockerfile: ci/docker/python-wheel-windows-test-vs2017.dockerfile
volumes:
+ - "${DOCKER_VOLUME_PREFIX}python-wheel-windows-clcache:C:/clcache"
- type: bind
source: .
target: "C:/arrow"
diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py
index 7152f3f1d4..42d8302afa 100644
--- a/python/pyarrow/tests/test_cython.py
+++ b/python/pyarrow/tests/test_cython.py
@@ -123,21 +123,37 @@ def test_cython_api(tmpdir):
# pyarrow imported first.
code = """if 1:
import sys
+ import os
+
+ try:
+ # Add dll directory was added on python 3.8
+ # and is required in order to find extra DLLs
+ # only for win32
+ for dir in {library_dirs}:
+ os.add_dll_directory(dir)
+ except AttributeError:
+ pass
mod = __import__({mod_name!r})
arr = mod.make_null_array(5)
assert mod.get_array_length(arr) == 5
assert arr.null_count == 5
- """.format(mod_name='pyarrow_cython_example')
+ """.format(mod_name='pyarrow_cython_example',
+ library_dirs=pa.get_library_dirs())
+ var = None
if sys.platform == 'win32':
- delim, var = ';', 'PATH'
+ if not hasattr(os, 'add_dll_directory'):
+ # Python 3.8 onwards don't check extension module DLLs on path
+ # we have to use os.add_dll_directory instead.
+ delim, var = ';', 'PATH'
else:
delim, var = ':', 'LD_LIBRARY_PATH'
- subprocess_env[var] = delim.join(
- pa.get_library_dirs() + [subprocess_env.get(var, '')]
- )
+ if var:
+ subprocess_env[var] = delim.join(
+ pa.get_library_dirs() + [subprocess_env.get(var, '')]
+ )
subprocess.check_call([sys.executable, '-c', code],
stdout=subprocess.PIPE,
env=subprocess_env)