You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2021/01/08 00:17:04 UTC
[GitHub] [arrow] kou commented on a change in pull request #9096: [Python][Packaging] Refactor manylinux and windows wheel building [WIP]
kou commented on a change in pull request #9096:
URL: https://github.com/apache/arrow/pull/9096#discussion_r553621162
##########
File path: ci/scripts/wheel_windows_build.bat
##########
@@ -0,0 +1,92 @@
+echo "Building windows wheel..."
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
+
+echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ==="
+del /s /q C:\arrow-build
+del /s /q C:\arrow-dist
+del /s /q C:\arrow\python\dist
+del /s /q C:\arrow\python\build
+del /s /q C:\arrow\python\pyarrow\*.so
+del /s /q C:\arrow\python\pyarrow\*.so.*
+
+echo "=== (%PYTHON_VERSION%) Building Arrow C++ libraries ==="
+set ARROW_DATASET=ON
+set ARROW_FLIGHT=ON
+set ARROW_GANDIVA=OFF
+set ARROW_HDFS=ON
+set ARROW_ORC=OFF
+set ARROW_PARQUET=ON
+set ARROW_MIMALLOC=ON
+set ARROW_S3=ON
+set ARROW_TENSORFLOW=ON
+set ARROW_WITH_BROTLI=ON
+set ARROW_WITH_BZ2=ON
+set ARROW_WITH_LZ4=ON
+set ARROW_WITH_SNAPPY=ON
+set ARROW_WITH_ZLIB=ON
+set ARROW_WITH_ZSTD=ON
+set CMAKE_UNITY_BUILD=ON
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+
+mkdir C:\arrow-build
+pushd C:\arrow-build
+cmake ^
+ -DARROW_BUILD_SHARED=ON ^
+ -DARROW_BUILD_STATIC=OFF ^
+ -DARROW_BUILD_TESTS=OFF ^
+ -DARROW_CXXFLAGS="/MP" ^
+ -DARROW_DATASET=%ARROW_DATASET% ^
+ -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
+ -DARROW_DEPENDENCY_USE_SHARED=OFF ^
+ -DARROW_FLIGHT=%ARROW_FLIGHT% ^
+ -DARROW_GANDIVA=%ARROW_GANDIVA% ^
+ -DARROW_HDFS=%ARROW_HDFS% ^
+ -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^
+ -DARROW_ORC=%ARROW_ORC% ^
+ -DARROW_PACKAGE_KIND="wheel-windows" ^
+ -DARROW_PARQUET=%ARROW_PARQUET% ^
+ -DARROW_PYTHON=ON ^
+ -DARROW_S3=%ARROW_S3% ^
+ -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^
+ -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^
+ -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^
+ -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^
+ -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
+ -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
+ -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+ -DLZ4_MSVC_LIB_PREFIX="" ^
+ -DLZ4_MSVC_STATIC_LIB_SUFFIX="" ^
+ -DZSTD_MSVC_LIB_PREFIX="" ^
+ -DCMAKE_CXX_COMPILER=clcache ^
+ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
+ -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
+ -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
+ -DMSVC_LINK_VERBOSE=ON ^
+ -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^
+ -G "%CMAKE_GENERATOR%" ^
+ C:\arrow\cpp || exit /B
+cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B
+popd
+
+echo "=== (%PYTHON_VERSION%) Building wheel ==="
+set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
+set PYARROW_BUNDLE_ARROW_CPP=ON
+set PYARROW_BUNDLE_BOOST=OFF
+set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
+set PYARROW_INSTALL_TESTS=ON
+set PYARROW_WITH_DATASET=%ARROW_DATASET%
+set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
+set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
+set PYARROW_WITH_HDFS=%ARROW_HDFS%
+set PYARROW_WITH_ORC=%ARROW_ORC%
+set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
+set PYARROW_WITH_S3=%ARROW_S3%
+set ARROW_HOME=C:\arrow-dist
+
+pushd C:\arrow\python
+@REM bundle the msvc runtime
+cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" pyarrow\
Review comment:
We can detect redist version (`14.16.27012` in this command line) by reading content at `C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\Microsoft.VCRedistVersion.default.txt`.
##########
File path: ci/scripts/wheel_windows_build.bat
##########
@@ -0,0 +1,92 @@
+echo "Building windows wheel..."
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
+
+echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ==="
+del /s /q C:\arrow-build
+del /s /q C:\arrow-dist
+del /s /q C:\arrow\python\dist
+del /s /q C:\arrow\python\build
+del /s /q C:\arrow\python\pyarrow\*.so
+del /s /q C:\arrow\python\pyarrow\*.so.*
+
+echo "=== (%PYTHON_VERSION%) Building Arrow C++ libraries ==="
+set ARROW_DATASET=ON
+set ARROW_FLIGHT=ON
+set ARROW_GANDIVA=OFF
+set ARROW_HDFS=ON
+set ARROW_ORC=OFF
+set ARROW_PARQUET=ON
+set ARROW_MIMALLOC=ON
+set ARROW_S3=ON
+set ARROW_TENSORFLOW=ON
+set ARROW_WITH_BROTLI=ON
+set ARROW_WITH_BZ2=ON
+set ARROW_WITH_LZ4=ON
+set ARROW_WITH_SNAPPY=ON
+set ARROW_WITH_ZLIB=ON
+set ARROW_WITH_ZSTD=ON
+set CMAKE_UNITY_BUILD=ON
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+
+mkdir C:\arrow-build
+pushd C:\arrow-build
+cmake ^
+ -DARROW_BUILD_SHARED=ON ^
+ -DARROW_BUILD_STATIC=OFF ^
+ -DARROW_BUILD_TESTS=OFF ^
+ -DARROW_CXXFLAGS="/MP" ^
+ -DARROW_DATASET=%ARROW_DATASET% ^
+ -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
+ -DARROW_DEPENDENCY_USE_SHARED=OFF ^
+ -DARROW_FLIGHT=%ARROW_FLIGHT% ^
+ -DARROW_GANDIVA=%ARROW_GANDIVA% ^
+ -DARROW_HDFS=%ARROW_HDFS% ^
+ -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^
+ -DARROW_ORC=%ARROW_ORC% ^
+ -DARROW_PACKAGE_KIND="wheel-windows" ^
+ -DARROW_PARQUET=%ARROW_PARQUET% ^
+ -DARROW_PYTHON=ON ^
+ -DARROW_S3=%ARROW_S3% ^
+ -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^
+ -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^
+ -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^
+ -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^
+ -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
+ -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
+ -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+ -DLZ4_MSVC_LIB_PREFIX="" ^
+ -DLZ4_MSVC_STATIC_LIB_SUFFIX="" ^
+ -DZSTD_MSVC_LIB_PREFIX="" ^
+ -DCMAKE_CXX_COMPILER=clcache ^
+ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
+ -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
+ -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
+ -DMSVC_LINK_VERBOSE=ON ^
+ -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^
+ -G "%CMAKE_GENERATOR%" ^
+ C:\arrow\cpp || exit /B
+cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B
+popd
+
+echo "=== (%PYTHON_VERSION%) Building wheel ==="
+set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
+set PYARROW_BUNDLE_ARROW_CPP=ON
+set PYARROW_BUNDLE_BOOST=OFF
+set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
+set PYARROW_INSTALL_TESTS=ON
+set PYARROW_WITH_DATASET=%ARROW_DATASET%
+set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
+set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
+set PYARROW_WITH_HDFS=%ARROW_HDFS%
+set PYARROW_WITH_ORC=%ARROW_ORC%
+set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
+set PYARROW_WITH_S3=%ARROW_S3%
+set ARROW_HOME=C:\arrow-dist
+
+pushd C:\arrow\python
+@REM bundle the msvc runtime
+cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" pyarrow\
Review comment:
I think that we also need to bundle `vcruntime140.dll` that is depended by `msvcp140.dll`. It's located at the same directory of `msvcp140.dll`: `C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\vcruntime140.dll`
##########
File path: ci/scripts/wheel_windows_build.bat
##########
@@ -0,0 +1,92 @@
+echo "Building windows wheel..."
+
+call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
+
+echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ==="
+del /s /q C:\arrow-build
+del /s /q C:\arrow-dist
+del /s /q C:\arrow\python\dist
+del /s /q C:\arrow\python\build
+del /s /q C:\arrow\python\pyarrow\*.so
+del /s /q C:\arrow\python\pyarrow\*.so.*
+
+echo "=== (%PYTHON_VERSION%) Building Arrow C++ libraries ==="
+set ARROW_DATASET=ON
+set ARROW_FLIGHT=ON
+set ARROW_GANDIVA=OFF
+set ARROW_HDFS=ON
+set ARROW_ORC=OFF
+set ARROW_PARQUET=ON
+set ARROW_MIMALLOC=ON
+set ARROW_S3=ON
+set ARROW_TENSORFLOW=ON
+set ARROW_WITH_BROTLI=ON
+set ARROW_WITH_BZ2=ON
+set ARROW_WITH_LZ4=ON
+set ARROW_WITH_SNAPPY=ON
+set ARROW_WITH_ZLIB=ON
+set ARROW_WITH_ZSTD=ON
+set CMAKE_UNITY_BUILD=ON
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+
+mkdir C:\arrow-build
+pushd C:\arrow-build
+cmake ^
+ -DARROW_BUILD_SHARED=ON ^
+ -DARROW_BUILD_STATIC=OFF ^
+ -DARROW_BUILD_TESTS=OFF ^
+ -DARROW_CXXFLAGS="/MP" ^
+ -DARROW_DATASET=%ARROW_DATASET% ^
+ -DARROW_DEPENDENCY_SOURCE=SYSTEM ^
+ -DARROW_DEPENDENCY_USE_SHARED=OFF ^
+ -DARROW_FLIGHT=%ARROW_FLIGHT% ^
+ -DARROW_GANDIVA=%ARROW_GANDIVA% ^
+ -DARROW_HDFS=%ARROW_HDFS% ^
+ -DARROW_MIMALLOC=%ARROW_MIMALLOC% ^
+ -DARROW_ORC=%ARROW_ORC% ^
+ -DARROW_PACKAGE_KIND="wheel-windows" ^
+ -DARROW_PARQUET=%ARROW_PARQUET% ^
+ -DARROW_PYTHON=ON ^
+ -DARROW_S3=%ARROW_S3% ^
+ -DARROW_TENSORFLOW=%ARROW_TENSORFLOW% ^
+ -DARROW_WITH_BROTLI=%ARROW_WITH_BROTLI% ^
+ -DARROW_WITH_BZ2=%ARROW_WITH_BZ2% ^
+ -DARROW_WITH_LZ4=%ARROW_WITH_LZ4% ^
+ -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
+ -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
+ -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
+ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+ -DLZ4_MSVC_LIB_PREFIX="" ^
+ -DLZ4_MSVC_STATIC_LIB_SUFFIX="" ^
+ -DZSTD_MSVC_LIB_PREFIX="" ^
+ -DCMAKE_CXX_COMPILER=clcache ^
+ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^
+ -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
+ -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
+ -DMSVC_LINK_VERBOSE=ON ^
+ -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^
+ -G "%CMAKE_GENERATOR%" ^
+ C:\arrow\cpp || exit /B
+cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B
+popd
+
+echo "=== (%PYTHON_VERSION%) Building wheel ==="
+set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
+set PYARROW_BUNDLE_ARROW_CPP=ON
+set PYARROW_BUNDLE_BOOST=OFF
+set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
+set PYARROW_INSTALL_TESTS=ON
+set PYARROW_WITH_DATASET=%ARROW_DATASET%
+set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
+set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
+set PYARROW_WITH_HDFS=%ARROW_HDFS%
+set PYARROW_WITH_ORC=%ARROW_ORC%
+set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
+set PYARROW_WITH_S3=%ARROW_S3%
+set ARROW_HOME=C:\arrow-dist
+
+pushd C:\arrow\python
+@REM bundle the msvc runtime
+cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" pyarrow\
Review comment:
Ah, Python for Windows includes `vcruntime140.dll`.
We don't need to bundle them.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org