You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2022/11/16 19:41:41 UTC
[arrow-adbc] branch main updated: ci: add Windows wheel build (#183)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 334a08b ci: add Windows wheel build (#183)
334a08b is described below
commit 334a08b98d4d6516f69966ae860f197b59f56dd9
Author: David Li <li...@gmail.com>
AuthorDate: Wed Nov 16 14:41:35 2022 -0500
ci: add Windows wheel build (#183)
---
.github/workflows/packaging-wheels.yml | 90 ++++++++++++++++++++++
c/driver/postgres/CMakeLists.txt | 12 ++-
c/driver/postgres/statement.cc | 1 -
c/driver/postgres/util.h | 25 +++---
ci/scripts/python_wheel_unix_test.sh | 4 +-
..._wheel_unix_test.sh => python_wheel_upload.ps1} | 48 +++++-------
ci/scripts/python_wheel_windows_build.bat | 64 +++++++++++++++
ci/scripts/python_wheel_windows_test.bat | 39 ++++++++++
python/adbc_driver_manager/setup.py | 11 ++-
9 files changed, 242 insertions(+), 52 deletions(-)
diff --git a/.github/workflows/packaging-wheels.yml b/.github/workflows/packaging-wheels.yml
index 43188e0..7f27b64 100644
--- a/.github/workflows/packaging-wheels.yml
+++ b/.github/workflows/packaging-wheels.yml
@@ -198,6 +198,96 @@ jobs:
env:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
+ python-windows:
+ name: "Python ${{ matrix.python_version }} Windows"
+ runs-on: windows-latest
+ strategy:
+ matrix:
+ python_version: ["3.9.13", "3.10.8"]
+ include:
+ - python_version: "3.9.13"
+ PYTHON_HOME: "C:\\Python39"
+ - python_version: "3.10.8"
+ PYTHON_HOME: "C:\\Python310"
+
+ # Limit parallelism; gemfury appears to return 409 CONFLICT on concurrent uploads
+ max-parallel: 1
+ env:
+ PYTHON_VERSION: "${{ matrix.python_version }}"
+ # Where to install vcpkg
+ VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Show inputs
+ shell: pwsh
+ run: |
+ echo "upload_artifacts: ${{ inputs.upload_artifacts }}"
+ echo "schedule: ${{ github.event.schedule }}"
+ echo "ref: ${{ github.ref }}"
+
+ - name: Install Chocolatey Dependencies
+ shell: pwsh
+ run: |
+ choco install --no-progress -y cmake --installargs 'ADD_CMAKE_TO_PATH=System'
+ choco install --no-progress -y visualcpp-build-tools
+
+ - name: Retrieve VCPKG version from .env
+ shell: pwsh
+ run: |
+ Select-String -Path .env -Pattern 'VCPKG="(.+)"' | % {"VCPKG_VERSION=$($_.matches.groups[1])"} >> $env:GITHUB_ENV
+
+ - name: Install vcpkg
+ shell: pwsh
+ run: |
+ echo $env:VCPKG_VERSION
+ git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg $env:VCPKG_ROOT
+ pushd $env:VCPKG_ROOT
+ .\bootstrap-vcpkg.bat -disableMetrics
+ popd
+
+ - name: Install Python ${{ matrix.python_version }}
+ shell: pwsh
+ run: |
+ choco install --no-progress -y python --version=${{ matrix.python_version }}
+ echo ("PATH=${{ matrix.PYTHON_HOME }};" + $env:PATH) >> $env:GITHUB_ENV
+
+ - name: Build wheel
+ shell: cmd
+ run: |
+ where python.exe
+ .\ci\scripts\python_wheel_windows_build.bat %cd% %cd%\build
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v3
+ with:
+ name: python${{ matrix.python_version }}-windows
+ retention-days: 7
+ path: |
+ python/adbc_driver_manager/dist/*.whl
+ python/adbc_driver_postgres/dist/*.whl
+
+ - name: Test wheel
+ shell: cmd
+ run: |
+ where python.exe
+ python -m venv venv
+ CALL venv\Scripts\activate.bat
+ .\ci\scripts\python_wheel_windows_test.bat %cd%
+
+ - name: Upload wheels to Gemfury
+ shell: pwsh
+ if: github.ref == 'refs/heads/main' && (github.event.schedule || inputs.upload_artifacts)
+ run: |
+ foreach ($wheel in Get-ChildItem python\adbc_driver_*\dist\*.whl) {
+ .\ci\scripts\python_wheel_upload.ps1 $wheel.FullName
+ }
+ env:
+ GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
+
python-sdist:
name: "Python sdist"
runs-on: ubuntu-latest
diff --git a/c/driver/postgres/CMakeLists.txt b/c/driver/postgres/CMakeLists.txt
index d7d7a72..62c587d 100644
--- a/c/driver/postgres/CMakeLists.txt
+++ b/c/driver/postgres/CMakeLists.txt
@@ -25,9 +25,17 @@ project(adbc_driver_postgres
VERSION "${ADBC_BASE_VERSION}"
LANGUAGES CXX)
include(CTest)
-find_package(PkgConfig)
-pkg_check_modules(LIBPQ REQUIRED libpq)
+if(WIN32)
+ # XXX: for now, assume vcpkg
+ find_package(PostgreSQL REQUIRED)
+ set(LIBPQ_LINK_LIBRARIES PostgreSQL::PostgreSQL)
+ set(LIBPQ_STATIC_LIBRARIES PostgreSQL::PostgreSQL)
+ set(LIBPQ_INCLUDE_DIRS)
+else()
+ find_package(PkgConfig)
+ pkg_check_modules(LIBPQ REQUIRED libpq)
+endif()
add_arrow_lib(adbc_driver_postgres
SOURCES
diff --git a/c/driver/postgres/statement.cc b/c/driver/postgres/statement.cc
index a187f82..ba489f1 100644
--- a/c/driver/postgres/statement.cc
+++ b/c/driver/postgres/statement.cc
@@ -17,7 +17,6 @@
#include "statement.h"
-#include <netinet/in.h>
#include <array>
#include <cerrno>
#include <cstring>
diff --git a/c/driver/postgres/util.h b/c/driver/postgres/util.h
index 83b6304..69a48c4 100644
--- a/c/driver/postgres/util.h
+++ b/c/driver/postgres/util.h
@@ -17,7 +17,11 @@
#pragma once
+#ifdef _WIN32
+#include <winsock2.h>
+#else
#include <netinet/in.h>
+#endif
#include <cstring>
#include <sstream>
#include <string>
@@ -25,10 +29,6 @@
#if defined(__linux__)
#include <endian.h>
-#elif defined(__APPLE__)
-#include <machine/endian.h>
-#else
-static_assert(false, "Not supported on this platform");
#endif
#include "adbc.h"
@@ -38,6 +38,11 @@ namespace adbcpq {
#define CONCAT(x, y) x##y
#define MAKE_NAME(x, y) CONCAT(x, y)
+#if defined(__linux__)
+static inline uint64_t ntohll(uint64_t x) { return be64toh(x); }
+static inline uint64_t htonll(uint64_t x) { return htobe64(x); }
+#endif
+
// see arrow/util/string_builder.h
template <typename Head>
@@ -122,13 +127,7 @@ static inline uint32_t LoadNetworkUInt32(const char* buf) {
static inline int64_t LoadNetworkUInt64(const char* buf) {
uint64_t v = 0;
std::memcpy(&v, buf, sizeof(uint64_t));
-#if defined(__linux__)
- return be64toh(v);
-#elif defined(__APPLE__)
return ntohll(v);
-#else
- static_assert(false, "Not supported on this platform");
-#endif
}
static inline int32_t LoadNetworkInt32(const char* buf) {
@@ -140,13 +139,7 @@ static inline int64_t LoadNetworkInt64(const char* buf) {
}
static inline uint64_t ToNetworkInt64(int64_t v) {
-#if defined(__linux__)
- return htobe64(static_cast<uint64_t>(v));
-#elif defined(__APPLE__)
return htonll(static_cast<uint64_t>(v));
-#else
- static_assert(false, "Not supported on this platform");
-#endif
}
} // namespace adbcpq
diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh
index 0e71707..2f9b6d8 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -31,7 +31,7 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${script_dir}/python_util.sh"
-echo "=== (${PYTHON_VERSION}) Installing sdists ==="
+echo "=== (${PYTHON_VERSION}) Installing wheels ==="
for component in ${COMPONENTS}; do
if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
pip install --force-reinstall \
@@ -44,5 +44,5 @@ done
pip install pytest pyarrow pandas
-echo "=== (${PYTHON_VERSION}) Testing sdists ==="
+echo "=== (${PYTHON_VERSION}) Testing wheels ==="
test_packages
diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_upload.ps1
old mode 100755
new mode 100644
similarity index 51%
copy from ci/scripts/python_wheel_unix_test.sh
copy to ci/scripts/python_wheel_upload.ps1
index 0e71707..72a9cbb
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_upload.ps1
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,32 +15,22 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-set -o pipefail
-
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <adbc-src-dir>"
- exit 1
-fi
-
-source_dir=${1}
-script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-source "${script_dir}/python_util.sh"
-
-echo "=== (${PYTHON_VERSION}) Installing sdists ==="
-for component in ${COMPONENTS}; do
- if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
- pip install --force-reinstall \
- ${source_dir}/python/${component}/repaired_wheels/*.whl
- else
- pip install --force-reinstall \
- ${source_dir}/python/${component}/dist/*.whl
- fi
-done
-pip install pytest pyarrow pandas
-
+$Uri = "https://$($env:GEMFURY_PUSH_TOKEN)@push.fury.io/arrow-adbc-nightlies/"
-echo "=== (${PYTHON_VERSION}) Testing sdists ==="
-test_packages
+for ($i = 0; $i -lt $args.count; $i++) {
+ echo "Uploading $($args[$i])"
+ $Form = @{
+ package = Get-Item -Path $args[$i]
+ }
+ try {
+ Invoke-WebRequest -uri $Uri -Method Post -Form $Form
+ } catch {
+ $StatusCode = $_.Exception.Response.StatusCode.value__
+ if ($StatusCode -eq 409) {
+ continue
+ } else {
+ echo "Failed to upload: $($StatusCode) $($_.Exception)"
+ exit 1
+ }
+ }
+}
diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat
new file mode 100644
index 0000000..04239f4
--- /dev/null
+++ b/ci/scripts/python_wheel_windows_build.bat
@@ -0,0 +1,64 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements. See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership. The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied. See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+set source_dir=%1
+set build_dir=%2
+
+echo "=== (%PYTHON_VERSION%) Building ADBC libpq driver ==="
+
+set CMAKE_BUILD_TYPE=release
+set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+set CMAKE_UNITY_BUILD=ON
+set VCPKG_FEATURE_FLAGS=-manifests
+set VCPKG_TARGET_TRIPLET=x64-windows-static
+
+IF NOT DEFINED VCPKG_ROOT (echo "Must set VCPKG_ROOT" && exit /B 1)
+
+mkdir %build_dir%
+pushd %build_dir%
+
+cmake ^
+ -G "%CMAKE_GENERATOR%" ^
+ -DADBC_BUILD_SHARED=ON ^
+ -DADBC_BUILD_STATIC=OFF ^
+ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+ -DCMAKE_INSTALL_PREFIX=%build_dir% ^
+ -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake ^
+ -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
+ -DVCPKG_TARGET_TRIPLET=%VCPKG_TARGET_TRIPLET% ^
+ %source_dir%\c\driver\postgres || exit /B 1
+cmake --build . --config %CMAKE_BUILD_TYPE% --target install -j || exit /B 1
+
+@REM XXX: CMake installs it to bin instead of lib for some reason
+set ADBC_POSTGRES_LIBRARY=%build_dir%\bin\adbc_driver_postgres.dll
+
+popd
+
+python -m pip install --upgrade pip
+
+FOR %%c IN (adbc_driver_manager adbc_driver_postgres) DO (
+ pushd %source_dir%\python\%%c
+
+ echo "=== (%PYTHON_VERSION%) Building %%c wheel ==="
+ @REM bundle the MSVC runtime
+ cp "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Redist\MSVC\14.16.27012\x64\Microsoft.VC141.CRT\msvcp140.dll" %%c\msvcp140.dll
+ python -m pip wheel -w dist -vvv . || exit /B 1
+
+ popd
+)
diff --git a/ci/scripts/python_wheel_windows_test.bat b/ci/scripts/python_wheel_windows_test.bat
new file mode 100644
index 0000000..7273f35
--- /dev/null
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -0,0 +1,39 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements. See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership. The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License. You may obtain a copy of the License at
+@rem
+@rem http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied. See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+set source_dir=%1
+
+echo "=== (%PYTHON_VERSION%) Installing wheels ==="
+
+FOR %%c IN (adbc_driver_manager adbc_driver_postgres) DO (
+ FOR %%w IN (%source_dir%\python\%%c\dist\*.whl) DO (
+ pip install --force-reinstall %%w || exit /B 1
+ )
+)
+
+pip install pytest pyarrow pandas
+
+echo "=== (%PYTHON_VERSION%) Testing wheels ==="
+
+FOR %%c IN (adbc_driver_manager adbc_driver_postgres) DO (
+ echo "=== Testing %%c ==="
+ python -c "import %%c" || exit /B 1
+ python -c "import %%c.dbapi" || exit /B 1
+ python -m pytest -vvx --import-mode append -k "not sqlite" %source_dir%\python\%%c\tests || exit /B 1
+)
diff --git a/python/adbc_driver_manager/setup.py b/python/adbc_driver_manager/setup.py
index e34c916..dde05a0 100644
--- a/python/adbc_driver_manager/setup.py
+++ b/python/adbc_driver_manager/setup.py
@@ -19,6 +19,7 @@
import os
import shutil
+import sys
from pathlib import Path
from setuptools import Extension, setup
@@ -68,6 +69,14 @@ def get_version_and_cmdclass(pkg_path):
version, cmdclass = get_version_and_cmdclass("adbc_driver_manager")
+# ------------------------------------------------------------
+# Resolve compiler flags
+
+if sys.platform == "win32":
+ extra_compile_args = ["/std:c++17", "/DADBC_EXPORTING"]
+else:
+ extra_compile_args = ["-std=c++17"]
+
# ------------------------------------------------------------
# Setup
@@ -76,7 +85,7 @@ setup(
ext_modules=[
Extension(
name="adbc_driver_manager._lib",
- extra_compile_args=["-std=c++17"],
+ extra_compile_args=extra_compile_args,
include_dirs=[str(source_root.joinpath("adbc_driver_manager").resolve())],
language="c++",
sources=[