You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2022/10/28 18:14:18 UTC
[arrow-adbc] branch main updated: ci(c/driver_manager,c/driver/postgres): add wheel builds (#139)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new b94348c ci(c/driver_manager,c/driver/postgres): add wheel builds (#139)
b94348c is described below
commit b94348c637c48a01562fa7caf30bbf3720b07688
Author: David Li <li...@gmail.com>
AuthorDate: Fri Oct 28 14:14:13 2022 -0400
ci(c/driver_manager,c/driver/postgres): add wheel builds (#139)
---
.env | 1 +
.github/workflows/packaging-wheels.yml | 77 ++++++++++++++++++++++
.gitignore | 1 +
CONTRIBUTING.md | 3 +
.../.gitignore => ci/conda_env_dev.txt | 4 +-
.../scripts/python_wheel_unix_test.sh | 45 +++++++------
docker-compose.yml | 13 ++--
python/adbc_driver_manager/.gitignore | 1 +
.../adbc_driver_manager/__init__.py | 1 +
python/adbc_driver_manager/pyproject.toml | 6 +-
python/adbc_driver_manager/tests/test_lowlevel.py | 4 ++
python/adbc_driver_postgres/.gitignore | 1 +
.../adbc_driver_postgres/__init__.py | 6 +-
python/adbc_driver_postgres/pyproject.toml | 6 +-
python/adbc_driver_postgres/tests/test_lowlevel.py | 4 ++
15 files changed, 142 insertions(+), 31 deletions(-)
diff --git a/.env b/.env
index ab06601..dc3d7b7 100644
--- a/.env
+++ b/.env
@@ -27,6 +27,7 @@ ARCH_ALIAS=x86_64
ARCH_SHORT=amd64
# Default versions for various dependencies
+MANYLINUX=2014
PYTHON=3.8
# Used through docker-compose.yml and serves as the default version for the
diff --git a/.github/workflows/packaging-wheels.yml b/.github/workflows/packaging-wheels.yml
new file mode 100644
index 0000000..2a40fbb
--- /dev/null
+++ b/.github/workflows/packaging-wheels.yml
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Packaging - Python Wheels
+
+on:
+ schedule:
+ - cron: "0 0 * * *"
+ workflow_dispatch:
+ inputs:
+ upload_wheels:
+ description: "Upload wheels to Gemfury"
+ required: true
+ type: boolean
+ default: false
+
+jobs:
+ manylinux:
+ name: "Python ${{ matrix.python_version }} manylinux${{ matrix.manylinux_version }}"
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ manylinux_version: ["2014"]
+ python_version: ["3.8", "3.9", "3.10"]
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Build wheel
+ shell: bash
+ run: |
+ export MANYLINUX=${{ matrix.manylinux_version }}
+ export PYTHON=${{ matrix.python_version }}
+ docker-compose run python-wheel-manylinux
+
+ - name: Test wheel
+ shell: bash
+ run: |
+ export MANYLINUX=${{ matrix.manylinux_version }}
+ export PYTHON=${{ matrix.python_version }}
+ docker-compose run python-wheel-manylinux-test
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v3
+ with:
+ name: python${{ matrix.python_version }}-manylinux${{ matrix.manylinux_version }}
+ retention-days: 7
+ path: |
+ python/adbc_driver_manager/repaired_wheels/*.whl
+ python/adbc_driver_postgres/repaired_wheels/*.whl
+
+ - name: Upload wheels to Gemfury
+ shell: bash
+ if: github.ref == 'main' && (github.event.schedule || inputs.upload_wheels)
+ run: |
+ path=$(ls python/adbc_driver_manager/repaired_wheels/*.whl)
+ echo curl -F "package=@${path}" https://${GEMFURY_PUSH_TOKEN}@push.fury.io/arrow-adbc-nightlies/
+ path=$(ls python/adbc_driver_postgres/repaired_wheels/*.whl)
+ echo curl -F "package=@${path}" https://${GEMFURY_PUSH_TOKEN}@push.fury.io/arrow-adbc-nightlies/
+ env:
+ GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
diff --git a/.gitignore b/.gitignore
index 9d83b69..7c71f9c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -77,6 +77,7 @@ site/
# Python
dist/
.hypothesis/
+repaired_wheels/
# R files
**/.Rproj.user
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4267385..6b809de 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -169,7 +169,10 @@ enforced via [`pre-commit`](https://pre-commit.com/). This will run
linters, formatters, and other analysis. For example:
```shell
+# Install pre-commit
$ pip install pre-commit
+# or alternatively
+$ conda install -c conda-forge --file ci/conda_env_dev.txt
# Set up hooks
$ pre-commit install
# Run manually
diff --git a/python/adbc_driver_postgres/.gitignore b/ci/conda_env_dev.txt
similarity index 93%
copy from python/adbc_driver_postgres/.gitignore
copy to ci/conda_env_dev.txt
index 126488c..99ad394 100644
--- a/python/adbc_driver_postgres/.gitignore
+++ b/ci/conda_env_dev.txt
@@ -15,6 +15,4 @@
# specific language governing permissions and limitations
# under the License.
-adbc_driver_postgres/*.c
-adbc_driver_postgres/*.cpp
-build/
+pre-commit
diff --git a/python/adbc_driver_postgres/tests/test_lowlevel.py b/ci/scripts/python_wheel_unix_test.sh
old mode 100644
new mode 100755
similarity index 54%
copy from python/adbc_driver_postgres/tests/test_lowlevel.py
copy to ci/scripts/python_wheel_unix_test.sh
index 0bd925c..09cf7f7
--- a/python/adbc_driver_postgres/tests/test_lowlevel.py
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,28 +17,31 @@
# specific language governing permissions and limitations
# under the License.
-import os
-
-import pyarrow
-import pytest
+set -e
+set -x
+set -o pipefail
-import adbc_driver_manager
-import adbc_driver_postgres
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 <arrow-src-dir>"
+ exit 1
+fi
+source_dir=${1}
-@pytest.fixture
-def postgres():
- postgres_uri = os.environ.get("ADBC_POSTGRES_TEST_URI")
- if not postgres_uri:
- pytest.skip("Set ADBC_POSTGRES_TEST_URI to run tests")
- with adbc_driver_postgres.connect(postgres_uri) as db:
- with adbc_driver_manager.AdbcConnection(db) as conn:
- yield conn
+# Install the built wheels
+pip install --force-reinstall ${source_dir}/python/adbc_driver_manager/repaired_wheels/*.whl
+pip install --force-reinstall ${source_dir}/python/adbc_driver_postgres/repaired_wheels/*.whl
+pip install pytest pyarrow pandas
+# Test that the modules are importable
+python -c "
+import adbc_driver_manager
+import adbc_driver_manager.dbapi
+import adbc_driver_postgres
+import adbc_driver_postgres.dbapi
+"
-def test_query_trivial(postgres):
- with adbc_driver_manager.AdbcStatement(postgres) as stmt:
- stmt.set_sql_query("SELECT 1")
- stream, _ = stmt.execute_query()
- reader = pyarrow.RecordBatchReader._import_from_c(stream.address)
- assert reader.read_all()
+# Can't yet run tests (need Postgres database, SQLite driver)
+# for component in adbc_driver_manager adbc_driver_postgres; do
+# python -m pytest -vvx ${source_dir}/python/$component/tests
+# done
diff --git a/docker-compose.yml b/docker-compose.yml
index 794a2e4..7f7c3bc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -26,11 +26,14 @@ services:
# These reuse Arrow's images for simplicity. You won't be able to
# build the image from here.
- python-wheel-manylinux-2014:
- image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
- # environment:
- # <<: *ccache
+ python-wheel-manylinux:
+ image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
volumes:
- .:/adbc:delegated
- # - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
command: /adbc/ci/scripts/python_wheel_manylinux_build.sh
+
+ python-wheel-manylinux-test:
+ image: ${ARCH}/python:${PYTHON}
+ volumes:
+ - .:/adbc:delegated
+ command: /adbc/ci/scripts/python_wheel_unix_test.sh /adbc
diff --git a/python/adbc_driver_manager/.gitignore b/python/adbc_driver_manager/.gitignore
index 0bcceba..6605a43 100644
--- a/python/adbc_driver_manager/.gitignore
+++ b/python/adbc_driver_manager/.gitignore
@@ -18,4 +18,5 @@
adbc_driver_manager/*.c
adbc_driver_manager/*.cc
adbc_driver_manager/*.cpp
+adbc_driver_manager/_version.py
build/
diff --git a/python/adbc_driver_manager/adbc_driver_manager/__init__.py b/python/adbc_driver_manager/adbc_driver_manager/__init__.py
index 4d92221..2c808f5 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/__init__.py
+++ b/python/adbc_driver_manager/adbc_driver_manager/__init__.py
@@ -36,3 +36,4 @@ from ._lib import ( # noqa: F401
OperationalError,
ProgrammingError,
)
+from ._version import version as __version__ # noqa: F401
diff --git a/python/adbc_driver_manager/pyproject.toml b/python/adbc_driver_manager/pyproject.toml
index c97351e..67a5959 100644
--- a/python/adbc_driver_manager/pyproject.toml
+++ b/python/adbc_driver_manager/pyproject.toml
@@ -17,11 +17,11 @@
[project]
name = "adbc_driver_manager"
-version = "1.0.0-alpha0"
description = ""
authors = [{name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}]
license = {text = "Apache-2.0"}
requires-python = ">=3.8"
+dynamic = ["version"]
[project.optional-dependencies]
dbapi = ["pandas", "pyarrow>=8.0.0"]
@@ -34,3 +34,7 @@ repository = "https://github.com/apache/arrow-adbc"
[build-system]
requires = ["Cython", "setuptools >= 61.0.0", "setuptools-scm"]
build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../.."
+write_to = "python/adbc_driver_manager/adbc_driver_manager/_version.py"
diff --git a/python/adbc_driver_manager/tests/test_lowlevel.py b/python/adbc_driver_manager/tests/test_lowlevel.py
index 5734cc2..889470b 100644
--- a/python/adbc_driver_manager/tests/test_lowlevel.py
+++ b/python/adbc_driver_manager/tests/test_lowlevel.py
@@ -45,6 +45,10 @@ def _bind(stmt, batch):
stmt.bind(array, schema)
+def test_version():
+ assert adbc_driver_manager.__version__
+
+
def test_database_init():
with pytest.raises(
adbc_driver_manager.ProgrammingError,
diff --git a/python/adbc_driver_postgres/.gitignore b/python/adbc_driver_postgres/.gitignore
index 126488c..9402188 100644
--- a/python/adbc_driver_postgres/.gitignore
+++ b/python/adbc_driver_postgres/.gitignore
@@ -17,4 +17,5 @@
adbc_driver_postgres/*.c
adbc_driver_postgres/*.cpp
+adbc_driver_postgres/_version.py
build/
diff --git a/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py b/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
index a2342bf..d1186ea 100644
--- a/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
+++ b/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
@@ -15,10 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-import importlib.resources
+import importlib.metadata
import adbc_driver_manager
+from ._version import version as __version__
+
+__all__ = ["connect", "__version__"]
+
def connect(uri: str) -> adbc_driver_manager.AdbcDatabase:
"""Create a low level ADBC connection to Postgres."""
diff --git a/python/adbc_driver_postgres/pyproject.toml b/python/adbc_driver_postgres/pyproject.toml
index 6f1e8e3..09e376f 100644
--- a/python/adbc_driver_postgres/pyproject.toml
+++ b/python/adbc_driver_postgres/pyproject.toml
@@ -17,11 +17,11 @@
[project]
name = "adbc_driver_postgres"
-version = "1.0.0-alpha0"
description = "A libpq-based ADBC driver for working with Postgres."
authors = [{name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}]
license = {text = "Apache-2.0"}
requires-python = ">=3.8"
+dynamic = ["version"]
[project.optional-dependencies]
dbapi = ["pandas", "pyarrow>=8.0.0"]
@@ -37,3 +37,7 @@ build-backend = "setuptools.build_meta"
[tool.setuptools]
include-package-data = true
+
+[tool.setuptools_scm]
+root = "../.."
+write_to = "python/adbc_driver_postgres/adbc_driver_postgres/_version.py"
diff --git a/python/adbc_driver_postgres/tests/test_lowlevel.py b/python/adbc_driver_postgres/tests/test_lowlevel.py
index 0bd925c..66e9b4d 100644
--- a/python/adbc_driver_postgres/tests/test_lowlevel.py
+++ b/python/adbc_driver_postgres/tests/test_lowlevel.py
@@ -40,3 +40,7 @@ def test_query_trivial(postgres):
stream, _ = stmt.execute_query()
reader = pyarrow.RecordBatchReader._import_from_c(stream.address)
assert reader.read_all()
+
+
+def test_version():
+ assert adbc_driver_postgres.__version__