You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2018/04/09 08:59:34 UTC
[arrow] branch master updated: ARROW-2353: [CI] Check correctness
of built wheel on AppVeyor
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 55c1075 ARROW-2353: [CI] Check correctness of built wheel on AppVeyor
55c1075 is described below
commit 55c1075accbb3f48ed8387033b3c50a99ecabf57
Author: Antoine Pitrou <an...@python.org>
AuthorDate: Mon Apr 9 10:59:12 2018 +0200
ARROW-2353: [CI] Check correctness of built wheel on AppVeyor
Also allows building PyArrow with another generator than Visual Studio, and assorted fixes on Windows.
Author: Antoine Pitrou <an...@python.org>
Closes #1793 from pitrou/ARROW-2353 and squashes the following commits:
1272018 <Antoine Pitrou> ARROW-2353: Check correctness of built wheel on AppVeyor
---
ci/msvc-build.bat | 39 ++++++++++++++++++++++++++++++------
python/CMakeLists.txt | 22 ++++++++++----------
python/pyarrow/tests/test_feather.py | 8 ++++----
python/setup.py | 23 ++++++++++++---------
4 files changed, 62 insertions(+), 30 deletions(-)
diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat
index cec1429..678e29d 100644
--- a/ci/msvc-build.bat
+++ b/ci/msvc-build.bat
@@ -104,12 +104,12 @@ cmake -G "%GENERATOR%" ^
cmake --build . --target install --config %CONFIGURATION% || exit /B
@rem Needed so python-test.exe works
-set OLD_PYTHONPATH=%PYTHONPATH%
-set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
+set OLD_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=%CONDA_PREFIX%
ctest -VV || exit /B
-set PYTHONPATH=%OLD_PYTHONPATH%
+set PYTHONHOME=%OLD_PYTHONHOME%
popd
@rem Build parquet-cpp
@@ -124,7 +124,8 @@ cmake -G "%GENERATOR%" ^
-DCMAKE_INSTALL_PREFIX=%PARQUET_HOME% ^
-DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
-DPARQUET_BOOST_USE_SHARED=OFF ^
- -DPARQUET_BUILD_TESTS=off .. || exit /B
+ -DPARQUET_BUILD_TESTS=OFF ^
+ .. || exit /B
cmake --build . --target install --config %CONFIGURATION% || exit /B
popd
@@ -135,13 +136,39 @@ popd
pushd python
set PYARROW_CXXFLAGS=/WX
-python setup.py build_ext --with-parquet --bundle-arrow-cpp --with-static-boost ^
+set PYARROW_CMAKE_GENERATOR=%GENERATOR%
+set PYARROW_BUNDLE_ARROW_CPP=ON
+set PYARROW_BUNDLE_BOOST=OFF
+set PYARROW_WITH_STATIC_BOOST=ON
+set PYARROW_WITH_PARQUET=ON
+
+python setup.py build_ext ^
install -q --single-version-externally-managed --record=record.text ^
- bdist_wheel || exit /B
+ bdist_wheel -q || exit /B
+
+for /F %%i in ('dir /B /S dist\*.whl') do set WHEEL_PATH=%%i
@rem Test directly from installed location
+@rem Needed for test_cython
SET PYARROW_PATH=%CONDA_PREFIX%\Lib\site-packages\pyarrow
py.test -r sxX --durations=15 -v %PYARROW_PATH% --parquet || exit /B
popd
+
+@rem Test pyarrow wheel from pristine environment
+
+call deactivate
+
+conda create -n wheel_test -q -y python=%PYTHON%
+
+call activate wheel_test
+
+pip install %WHEEL_PATH% || exit /B
+
+python -c "import pyarrow" || exit /B
+python -c "import pyarrow.parquet" || exit /B
+
+pip install pandas pytest pytest-faulthandler
+
+py.test -r sxX --durations=15 --pyargs pyarrow.tests || exit /B
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index cb3cd70..fcc1d3c 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -141,13 +141,18 @@ endif()
# For any C code, use the same flags.
set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}")
-# set compile output directory
-string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
+if (MSVC)
+ # MSVC makes its own output directories based on the build configuration
+ set(BUILD_SUBDIR_NAME "")
+else()
+ # Set compile output directory
+ string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME)
+endif()
# If build in-source, create the latest symlink. If build out-of-source, which is
# preferred, simply output the binaries in the build folder
if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}/")
+ set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/build/${BUILD_SUBDIR_NAME}")
# Link build/latest to the current build directory, to avoid developers
# accidentally running the latest debug build when in fact they're building
# release builds.
@@ -155,15 +160,10 @@ if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})
if (NOT APPLE)
set(MORE_ARGS "-T")
endif()
-EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
- ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
+ EXECUTE_PROCESS(COMMAND ln ${MORE_ARGS} -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
+ ${CMAKE_CURRENT_BINARY_DIR}/build/latest)
else()
- if (MSVC)
- # MSVC makes its own output directories based on the build configuration
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/")
- else()
- set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}/")
- endif()
+ set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${BUILD_SUBDIR_NAME}")
endif()
message(STATUS "Build output directory: ${BUILD_OUTPUT_ROOT_DIRECTORY}")
diff --git a/python/pyarrow/tests/test_feather.py b/python/pyarrow/tests/test_feather.py
index a14673f..171f28d 100644
--- a/python/pyarrow/tests/test_feather.py
+++ b/python/pyarrow/tests/test_feather.py
@@ -17,6 +17,7 @@
import os
import sys
+import tempfile
import unittest
import pytest
@@ -27,14 +28,13 @@ from pandas.util.testing import assert_frame_equal
import pandas as pd
import pyarrow as pa
-from pyarrow.compat import guid
from pyarrow.feather import (read_feather, write_feather,
FeatherReader)
from pyarrow.lib import FeatherWriter
-def random_path():
- return 'feather_{}'.format(guid())
+def random_path(prefix='feather_'):
+ return tempfile.mktemp(prefix=prefix)
class TestFeatherReader(unittest.TestCase):
@@ -400,7 +400,7 @@ class TestFeatherReader(unittest.TestCase):
# GH #209
name = (b'Besa_Kavaj\xc3\xab.feather').decode('utf-8')
df = pd.DataFrame({'foo': [1, 2, 3, 4]})
- self._check_pandas_roundtrip(df, path=name)
+ self._check_pandas_roundtrip(df, path=random_path(prefix=name))
def test_read_columns(self):
data = {'foo': [1, 2, 3, 4],
diff --git a/python/setup.py b/python/setup.py
index 61c9a75..7b0f175 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -92,7 +92,8 @@ class build_ext(_build_ext):
# github.com/libdynd/dynd-python
description = "Build the C-extensions for arrow"
- user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'),
+ user_options = ([('cmake-generator=', None, 'CMake generator'),
+ ('extra-cmake-args=', None, 'extra arguments for CMake'),
('build-type=', None, 'build type (debug or release)'),
('boost-namespace=', None,
'namespace of boost (default: boost)'),
@@ -109,6 +110,9 @@ class build_ext(_build_ext):
def initialize_options(self):
_build_ext.initialize_options(self)
+ self.cmake_generator = os.environ.get('PYARROW_CMAKE_GENERATOR')
+ if not self.cmake_generator and sys.platform == 'win32':
+ self.cmake_generator = 'Visual Studio 14 2015 Win64'
self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '')
self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower()
self.boost_namespace = os.environ.get('PYARROW_BOOST_NAMESPACE', 'boost')
@@ -133,8 +137,6 @@ class build_ext(_build_ext):
os.environ.get('PYARROW_WITH_ORC', '0'))
self.bundle_arrow_cpp = strtobool(
os.environ.get('PYARROW_BUNDLE_ARROW_CPP', '0'))
- # Default is True but this only is actually bundled when
- # we also bundle arrow-cpp.
self.bundle_boost = strtobool(
os.environ.get('PYARROW_BUNDLE_BOOST', '0'))
@@ -174,6 +176,8 @@ class build_ext(_build_ext):
static_lib_option,
]
+ if self.cmake_generator:
+ cmake_options += ['-G', self.cmake_generator]
if self.with_parquet:
cmake_options.append('-DPYARROW_BUILD_PARQUET=on')
if self.with_static_parquet:
@@ -230,16 +234,12 @@ class build_ext(_build_ext):
self.spawn(args)
print("-- Finished cmake --build for pyarrow")
else:
- cmake_generator = 'Visual Studio 14 2015 Win64'
if not is_64_bit:
raise RuntimeError('Not supported on 32-bit Windows')
# Generate the build files
cmake_command = (['cmake'] + extra_cmake_args +
- cmake_options +
- [source, '-G', cmake_generator])
- if "-G" in self.extra_cmake_args:
- cmake_command = cmake_command[:-2]
+ cmake_options + [source])
print("-- Runnning cmake for pyarrow")
self.spawn(cmake_command)
@@ -359,7 +359,12 @@ class build_ext(_build_ext):
if sys.platform == 'win32':
head, tail = os.path.split(name)
suffix = sysconfig.get_config_var('SO')
- return pjoin(head, self.build_type, tail + suffix)
+ # Visual Studio seems to differ from other generators in
+ # where it places output files.
+ if self.cmake_generator.startswith('Visual Studio'):
+ return pjoin(head, self.build_type, tail + suffix)
+ else:
+ return pjoin(head, tail + suffix)
else:
suffix = sysconfig.get_config_var('SO')
return pjoin(self.build_type, name + suffix)
--
To stop receiving notification emails like this one, please contact
apitrou@apache.org.