You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2022/11/16 19:23:07 UTC
[arrow-adbc] branch main updated: ci(c/driver_manager,c/driver/postgres): set up Python sdist build (#179)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 5f5429e ci(c/driver_manager,c/driver/postgres): set up Python sdist build (#179)
5f5429e is described below
commit 5f5429eac32930e79e0869a8118df7e18a0de901
Author: David Li <li...@gmail.com>
AuthorDate: Wed Nov 16 14:23:02 2022 -0500
ci(c/driver_manager,c/driver/postgres): set up Python sdist build (#179)
Co-authored-by: Joris Van den Bossche <jo...@gmail.com>
---
.env | 2 +-
.gitattributes | 4 +-
.github/workflows/packaging-wheels.yml | 43 +++++
NOTICE.txt | 3 +
ci/conda_env_python.txt | 1 +
.../scripts/python_sdist_build.sh | 30 ++-
...hon_wheel_unix_test.sh => python_sdist_test.sh} | 35 ++--
ci/scripts/python_util.sh | 75 ++++++++
ci/scripts/python_wheel_unix_build.sh | 38 +---
ci/scripts/python_wheel_unix_test.sh | 21 +--
docker-compose.yml | 17 +-
python/adbc_driver_manager/.gitignore | 4 +-
.../.gitignore => adbc_driver_manager/MANIFEST.in} | 9 +-
.../adbc_driver_manager/__init__.py | 2 +-
.../adbc_driver_manager/_static_version.py} | 24 ++-
.../adbc_driver_manager/_version.py | 206 +++++++++++++++++++++
python/adbc_driver_manager/pyproject.toml | 6 +-
python/adbc_driver_manager/setup.py | 59 +++++-
python/adbc_driver_postgres/.gitignore | 1 -
.../adbc_driver_postgres/__init__.py | 2 +-
.../_static_version.py} | 24 ++-
.../adbc_driver_postgres/_version.py | 206 +++++++++++++++++++++
python/adbc_driver_postgres/pyproject.toml | 6 +-
python/adbc_driver_postgres/setup.py | 45 ++++-
24 files changed, 723 insertions(+), 140 deletions(-)
diff --git a/.env b/.env
index 85a09dc..95a1c39 100644
--- a/.env
+++ b/.env
@@ -30,7 +30,7 @@ ARCH_SHORT=amd64
JDK=8
MANYLINUX=2014
MAVEN=3.5.4
-PYTHON=3.8
+PYTHON=3.10
# Used through docker-compose.yml and serves as the default version for the
# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
diff --git a/.gitattributes b/.gitattributes
index 2a3bd0c..e5fde9d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -16,5 +16,5 @@
# under the License.
c/vendor/* linguist-vendored
-python/adbc_driver_manager/poetry.lock linguist-generated=true
-python/adbc_driver_manager/requirements-dev.txt linguist-generated=true
+python/adbc_driver_manager/adbc_driver_manager/_static_version.py export-subst
+python/adbc_driver_postgres/adbc_driver_postgres/_static_version.py export-subst
diff --git a/.github/workflows/packaging-wheels.yml b/.github/workflows/packaging-wheels.yml
index 7cd1713..43188e0 100644
--- a/.github/workflows/packaging-wheels.yml
+++ b/.github/workflows/packaging-wheels.yml
@@ -197,3 +197,46 @@ jobs:
./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgres}/dist/*.whl
env:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
+
+ python-sdist:
+ name: "Python sdist"
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ persist-credentials: false
+
+ - name: Show inputs
+ shell: bash
+ run: |
+ echo "upload_artifacts: ${{ github.event.inputs.upload_artifacts }}"
+ echo "schedule: ${{ github.event.schedule }}"
+ echo "ref: ${{ github.ref }}"
+
+ - name: Build sdist
+ shell: bash
+ run: |
+ docker-compose run python-sdist
+
+ - name: Archive sdist
+ uses: actions/upload-artifact@v3
+ with:
+ name: python${{ matrix.python_version }}-manylinux${{ matrix.manylinux_version }}
+ retention-days: 7
+ path: |
+ python/adbc_driver_manager/dist/*.tar.gz
+ python/adbc_driver_postgres/dist/*.tar.gz
+
+ - name: Test sdist
+ shell: bash
+ run: |
+ docker-compose run python-sdist-test
+
+ - name: Upload sdist to Gemfury
+ shell: bash
+ if: github.ref == 'refs/heads/main' && (github.event.schedule || github.event.inputs.upload_artifacts == true || github.event.inputs.upload_artifacts == 'true')
+ run: |
+ ./ci/scripts/python_wheel_upload.sh python/adbc_driver_{manager,postgres}/dist/*.tar.gz
+ env:
+ GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
diff --git a/NOTICE.txt b/NOTICE.txt
index bc07e6d..a5ef1a2 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -3,3 +3,6 @@ Copyright 2022 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software from the miniver project (CC0).
+https://github.com/jbweston/miniver
diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt
index 8f60ddc..702f899 100644
--- a/ci/conda_env_python.txt
+++ b/ci/conda_env_python.txt
@@ -18,3 +18,4 @@
Cython
pyarrow>=8.0.0
pytest
+setuptools
diff --git a/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py b/ci/scripts/python_sdist_build.sh
old mode 100644
new mode 100755
similarity index 53%
copy from python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
copy to ci/scripts/python_sdist_build.sh
index b09c2f4..ea33375
--- a/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
+++ b/ci/scripts/python_sdist_build.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,18 +17,26 @@
# specific language governing permissions and limitations
# under the License.
-import importlib.resources
+set -ex
+
+source_dir=${1}
+
+echo "=== (${PYTHON_VERSION}) Building ADBC sdists ==="
-import adbc_driver_manager
+# https://github.com/pypa/pip/issues/7555
+# Get the latest pip so we have in-tree-build by default
+pip install --upgrade pip setuptools
-from ._version import version as __version__
+# For drivers, which bundle shared libraries, defer that to install time
+export _ADBC_IS_SDIST=1
-__all__ = ["connect", "__version__"]
+for component in adbc_driver_manager adbc_driver_postgres; do
+ pushd ${source_dir}/python/$component
+ echo "=== (${PYTHON_VERSION}) Building $component sdist ==="
+ # python -m build copies to a tempdir, so we can't reference other files in the repo
+ # https://github.com/pypa/pip/issues/5519
+ python setup.py sdist
-def connect(uri: str) -> adbc_driver_manager.AdbcDatabase:
- """Create a low level ADBC connection to Postgres."""
- with importlib.resources.path(
- __package__, "libadbc_driver_postgres.so"
- ) as entrypoint:
- return adbc_driver_manager.AdbcDatabase(driver=str(entrypoint), uri=uri)
+ popd
+done
diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_sdist_test.sh
similarity index 54%
copy from ci/scripts/python_wheel_unix_test.sh
copy to ci/scripts/python_sdist_test.sh
index 452a941..e339533 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_sdist_test.sh
@@ -26,32 +26,21 @@ if [ "$#" -ne 1 ]; then
exit 1
fi
-COMPONENTS="adbc_driver_manager adbc_driver_postgres"
-
source_dir=${1}
+build_dir="${source_dir}/build"
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
-# Install the built wheels
+echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
+# Sets ADBC_POSTGRES_LIBRARY
+build_drivers "${source_dir}" "${build_dir}"
+
+echo "=== (${PYTHON_VERSION}) Installing sdists ==="
for component in ${COMPONENTS}; do
- if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
- pip install --force-reinstall \
- ${source_dir}/python/${component}/repaired_wheels/*.whl
- else
- pip install --force-reinstall \
- ${source_dir}/python/${component}/dist/*.whl
- fi
+ pip install --force-reinstall ${source_dir}/python/${component}/dist/*.tar.gz
done
pip install pytest pyarrow pandas
-# Test that the modules are importable
-python -c "
-import adbc_driver_manager
-import adbc_driver_manager.dbapi
-import adbc_driver_postgres
-import adbc_driver_postgres.dbapi
-"
-
-# Will only run some smoke tests
-echo "=== Testing adbc_driver_manager ==="
-python -m pytest -vvx -k "not sqlite" ${source_dir}/python/adbc_driver_manager/tests
-echo "=== Testing adbc_driver_postgres ==="
-python -m pytest -vvx ${source_dir}/python/adbc_driver_postgres/tests
+echo "=== (${PYTHON_VERSION}) Testing sdists ==="
+test_packages
diff --git a/ci/scripts/python_util.sh b/ci/scripts/python_util.sh
new file mode 100644
index 0000000..fb06d6c
--- /dev/null
+++ b/ci/scripts/python_util.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+COMPONENTS="adbc_driver_manager adbc_driver_postgres"
+
+function build_drivers {
+ local -r source_dir="$1"
+ local -r build_dir="$2"
+
+ : ${CMAKE_BUILD_TYPE:=release}
+ : ${CMAKE_UNITY_BUILD:=ON}
+ : ${CMAKE_GENERATOR:=Ninja}
+ : ${VCPKG_ROOT:=/opt/vcpkg}
+ # Enable manifest mode
+ : ${VCPKG_FEATURE_FLAGS:=manifests}
+ # Add our custom triplets
+ : ${VCPKG_OVERLAY_TRIPLETS:="${source_dir}/ci/vcpkg/triplets/"}
+
+ if [[ $(uname) == "Linux" ]]; then
+ export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
+ export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
+ else # macOS
+ export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
+ export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
+ fi
+
+ echo ${VCPKG_DEFAULT_TRIPLET}
+
+ mkdir -p ${build_dir}
+ pushd ${build_dir}
+
+ cmake \
+ -G ${CMAKE_GENERATOR} \
+ -DADBC_BUILD_SHARED=ON \
+ -DADBC_BUILD_STATIC=OFF \
+ -DCMAKE_INSTALL_LIBDIR=lib \
+ -DCMAKE_INSTALL_PREFIX=${build_dir} \
+ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
+ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
+ ${source_dir}/c/driver/postgres
+ cmake --build . --target install -j
+ popd
+}
+
+function test_packages {
+ python -c "
+import adbc_driver_manager
+import adbc_driver_manager.dbapi
+import adbc_driver_postgres
+import adbc_driver_postgres.dbapi
+"
+
+ # Will only run some smoke tests
+ # --import-mode required, else tries to import from the source dir instead of installed package
+ echo "=== Testing adbc_driver_manager ==="
+ python -m pytest -vvx --import-mode append -k "not sqlite" ${source_dir}/python/adbc_driver_manager/tests
+ echo "=== Testing adbc_driver_postgres ==="
+ python -m pytest -vvx --import-mode append ${source_dir}/python/adbc_driver_postgres/tests
+}
diff --git a/ci/scripts/python_wheel_unix_build.sh b/ci/scripts/python_wheel_unix_build.sh
index 2842cb4..58c4e61 100755
--- a/ci/scripts/python_wheel_unix_build.sh
+++ b/ci/scripts/python_wheel_unix_build.sh
@@ -22,6 +22,9 @@ set -ex
arch=${1}
source_dir=${2}
build_dir=${3}
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
function check_visibility {
if [[ $(uname) != "Linux" ]]; then
@@ -60,39 +63,8 @@ function check_wheels {
}
echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
-: ${CMAKE_BUILD_TYPE:=release}
-: ${CMAKE_UNITY_BUILD:=ON}
-: ${CMAKE_GENERATOR:=Ninja}
-: ${VCPKG_ROOT:=/opt/vcpkg}
-# Enable manifest mode
-: ${VCPKG_FEATURE_FLAGS:=manifests}
-# Add our custom triplets
-: ${VCPKG_OVERLAY_TRIPLETS:="${source_dir}/ci/vcpkg/triplets/"}
-
-if [[ $(uname) == "Linux" ]]; then
- export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
- export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
-else # macOS
- export ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
- export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
-fi
-
-echo ${VCPKG_DEFAULT_TRIPLET}
-
-mkdir -p ${build_dir}
-pushd ${build_dir}
-
-cmake \
- -G ${CMAKE_GENERATOR} \
- -DADBC_BUILD_SHARED=ON \
- -DADBC_BUILD_STATIC=OFF \
- -DCMAKE_INSTALL_LIBDIR=lib \
- -DCMAKE_INSTALL_PREFIX=${build_dir} \
- -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \
- -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
- ${source_dir}/c/driver/postgres
-cmake --build . --target install -j
-popd
+# Sets ADBC_POSTGRES_LIBRARY
+build_drivers "${source_dir}" "${build_dir}"
# Check that we don't expose any unwanted symbols
check_visibility $ADBC_POSTGRES_LIBRARY
diff --git a/ci/scripts/python_wheel_unix_test.sh b/ci/scripts/python_wheel_unix_test.sh
index 452a941..0e71707 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -26,11 +26,12 @@ if [ "$#" -ne 1 ]; then
exit 1
fi
-COMPONENTS="adbc_driver_manager adbc_driver_postgres"
-
source_dir=${1}
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+source "${script_dir}/python_util.sh"
-# Install the built wheels
+echo "=== (${PYTHON_VERSION}) Installing sdists ==="
for component in ${COMPONENTS}; do
if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
pip install --force-reinstall \
@@ -42,16 +43,6 @@ for component in ${COMPONENTS}; do
done
pip install pytest pyarrow pandas
-# Test that the modules are importable
-python -c "
-import adbc_driver_manager
-import adbc_driver_manager.dbapi
-import adbc_driver_postgres
-import adbc_driver_postgres.dbapi
-"
-# Will only run some smoke tests
-echo "=== Testing adbc_driver_manager ==="
-python -m pytest -vvx -k "not sqlite" ${source_dir}/python/adbc_driver_manager/tests
-echo "=== Testing adbc_driver_postgres ==="
-python -m pytest -vvx ${source_dir}/python/adbc_driver_postgres/tests
+echo "=== (${PYTHON_VERSION}) Testing sdists ==="
+test_packages
diff --git a/docker-compose.yml b/docker-compose.yml
index ce16eac..04ed6e0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -38,13 +38,28 @@ services:
- .:/adbc:delegated
command: "/bin/bash -c '/adbc/ci/scripts/java_build.sh /adbc && /adbc/ci/scripts/java_test.sh /adbc'"
+ ############################ Python sdist ##################################
+
+ python-sdist:
+ image: ${ARCH}/python:${PYTHON}
+ volumes:
+ - .:/adbc:delegated
+ command: /adbc/ci/scripts/python_sdist_build.sh /adbc
+
+ python-sdist-test:
+ image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
+ volumes:
+ - .:/adbc:delegated
+ command: "'/adbc/ci/scripts/python_sdist_test.sh /adbc'"
+
############################ Python wheels ##################################
python-wheel-manylinux:
image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
volumes:
- .:/adbc:delegated
- command: "'/adbc/ci/scripts/python_wheel_unix_build.sh x86_64 /adbc /adbc/build'"
+ # Must set safe.directory so miniver won't error when calling git
+ command: "'git config --global --add safe.directory /adbc && git config --global --get safe.directory && /adbc/ci/scripts/python_wheel_unix_build.sh x86_64 /adbc /adbc/build'"
python-wheel-manylinux-test:
image: ${ARCH}/python:${PYTHON}
diff --git a/python/adbc_driver_manager/.gitignore b/python/adbc_driver_manager/.gitignore
index 6605a43..9c6ef7f 100644
--- a/python/adbc_driver_manager/.gitignore
+++ b/python/adbc_driver_manager/.gitignore
@@ -18,5 +18,7 @@
adbc_driver_manager/*.c
adbc_driver_manager/*.cc
adbc_driver_manager/*.cpp
-adbc_driver_manager/_version.py
+adbc_driver_manager/*.h
build/
+dist/
+repaired_wheels/
diff --git a/python/adbc_driver_postgres/.gitignore b/python/adbc_driver_manager/MANIFEST.in
similarity index 83%
copy from python/adbc_driver_postgres/.gitignore
copy to python/adbc_driver_manager/MANIFEST.in
index 9402188..df023c5 100644
--- a/python/adbc_driver_postgres/.gitignore
+++ b/python/adbc_driver_manager/MANIFEST.in
@@ -15,7 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-adbc_driver_postgres/*.c
-adbc_driver_postgres/*.cpp
-adbc_driver_postgres/_version.py
-build/
+# setuptools manifest
+
+include adbc_driver_manager/adbc.h
+include adbc_driver_manager/adbc_driver_manager.cc
+include adbc_driver_manager/adbc_driver_manager.h
diff --git a/python/adbc_driver_manager/adbc_driver_manager/__init__.py b/python/adbc_driver_manager/adbc_driver_manager/__init__.py
index 4ad59ab..ff19744 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/__init__.py
+++ b/python/adbc_driver_manager/adbc_driver_manager/__init__.py
@@ -46,7 +46,7 @@ from ._lib import (
OperationalError,
ProgrammingError,
)
-from ._version import version as __version__ # noqa: F401
+from ._version import __version__
__all__ = [
"__version__",
diff --git a/python/adbc_driver_postgres/setup.py b/python/adbc_driver_manager/adbc_driver_manager/_static_version.py
similarity index 64%
copy from python/adbc_driver_postgres/setup.py
copy to python/adbc_driver_manager/adbc_driver_manager/_static_version.py
index d7ee3ae..7ca903c 100644
--- a/python/adbc_driver_postgres/setup.py
+++ b/python/adbc_driver_manager/adbc_driver_manager/_static_version.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-
+# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,17 +16,16 @@
# specific language governing permissions and limitations
# under the License.
-import os
-import shutil
-from pathlib import Path
-
-from setuptools import setup
+# Generated by miniver (CC0).
-library = os.environ.get("ADBC_POSTGRES_LIBRARY")
-if not library:
- raise ValueError("Must provide ADBC_POSTGRES_LIBRARY")
+# This file is part of 'miniver': https://github.com/jbweston/miniver
+#
+# This file will be overwritten by setup.py when a source or binary
+# distribution is made. The magic value "__use_git__" is interpreted by
+# _version.py.
-target = Path("./adbc_driver_postgres/libadbc_driver_postgres.so").resolve()
-shutil.copy(library, target)
+version = "__use_git__"
-setup()
+# These values are only set if the distribution was created with 'git archive'
+refnames = "$Format:%D$"
+git_hash = "$Format:%h$"
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_version.py b/python/adbc_driver_manager/adbc_driver_manager/_version.py
new file mode 100644
index 0000000..2d2fa6c
--- /dev/null
+++ b/python/adbc_driver_manager/adbc_driver_manager/_version.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generated by miniver (CC0).
+
+import os
+
+# This file is part of 'miniver': https://github.com/jbweston/miniver
+#
+from collections import namedtuple
+
+Version = namedtuple("Version", ("release", "dev", "labels"))
+
+# No public API
+__all__ = []
+
+package_root = os.path.dirname(os.path.realpath(__file__))
+package_name = os.path.basename(package_root)
+
+STATIC_VERSION_FILE = "_static_version.py"
+
+
+def get_version(version_file=STATIC_VERSION_FILE):
+ version_info = get_static_version_info(version_file)
+ version = version_info["version"]
+ if version == "__use_git__":
+ version = get_version_from_git()
+ if not version:
+ version = get_version_from_git_archive(version_info)
+ if not version:
+ version = Version("unknown", None, None)
+ return pep440_format(version)
+ else:
+ return version
+
+
+def get_static_version_info(version_file=STATIC_VERSION_FILE):
+ version_info = {}
+ with open(os.path.join(package_root, version_file), "rb") as f:
+ exec(f.read(), {}, version_info)
+ return version_info
+
+
+def version_is_from_git(version_file=STATIC_VERSION_FILE):
+ return get_static_version_info(version_file)["version"] == "__use_git__"
+
+
+def pep440_format(version_info):
+ release, dev, labels = version_info
+
+ version_parts = [release]
+ if dev:
+ if release.endswith("-dev") or release.endswith(".dev"):
+ version_parts.append(dev)
+ else: # prefer PEP440 over strict adhesion to semver
+ version_parts.append(".dev{}".format(dev))
+
+ if labels:
+ version_parts.append("+")
+ version_parts.append(".".join(labels))
+
+ return "".join(version_parts)
+
+
+def get_version_from_git():
+ import subprocess
+
+ # git describe --first-parent does not take into account tags from branches
+ # that were merged-in. The '--long' flag gets us the 'dev' version and
+ # git hash, '--always' returns the git hash even if there are no tags.
+ for opts in [["--first-parent"], []]:
+ try:
+ p = subprocess.Popen(
+ ["git", "describe", "--long", "--always"] + opts,
+ cwd=package_root,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+ except OSError:
+ return
+ if p.wait() == 0:
+ break
+ else:
+ return
+
+ description = (
+ p.communicate()[0]
+ .decode()
+ .strip("v") # Tags can have a leading 'v', but the version should not
+ .rstrip("\n")
+ .rsplit("-", 2) # Split the latest tag, commits since tag, and hash
+ )
+
+ try:
+ release, dev, git = description
+ except ValueError: # No tags, only the git hash
+ # prepend 'g' to match with format returned by 'git describe'
+ git = "g{}".format(*description)
+ # XXX: assume version if not given
+ release = "0.0.0"
+ dev = None
+
+ labels = []
+ if dev == "0":
+ dev = None
+ else:
+ labels.append(git)
+
+ try:
+ p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root)
+ except OSError:
+ labels.append("confused") # This should never happen.
+ else:
+ if p.wait() == 1:
+ labels.append("dirty")
+
+ return Version(release, dev, labels)
+
+
+# TODO: change this logic when there is a git pretty-format
+# that gives the same output as 'git describe'.
+# Currently we can only tell the tag the current commit is
+# pointing to, or its hash (with no version info)
+# if it is not tagged.
+def get_version_from_git_archive(version_info):
+ try:
+ refnames = version_info["refnames"]
+ git_hash = version_info["git_hash"]
+ except KeyError:
+ # These fields are not present if we are running from an sdist.
+ # Execution should never reach here, though
+ return None
+
+ if git_hash.startswith("$Format") or refnames.startswith("$Format"):
+ # variables not expanded during 'git archive'
+ return None
+
+ VTAG = "tag: v"
+ refs = set(r.strip() for r in refnames.split(","))
+ version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG))
+ if version_tags:
+ release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1"
+ return Version(release, dev=None, labels=None)
+ else:
+ return Version("unknown", dev=None, labels=["g{}".format(git_hash)])
+
+
+__version__ = get_version()
+
+
+# The following section defines a 'get_cmdclass' function
+# that can be used from setup.py. The '__version__' module
+# global is used (but not modified).
+
+
+def _write_version(fname):
+ # This could be a hard link, so try to delete it first. Is there any way
+ # to do this atomically together with opening?
+ try:
+ os.remove(fname)
+ except OSError:
+ pass
+ with open(fname, "w") as f:
+ f.write(
+ "# This file has been created by setup.py.\n"
+ "version = '{}'\n".format(__version__)
+ )
+
+
+def get_cmdclass(pkg_source_path):
+ from setuptools.command.build_py import build_py as build_py_orig
+ from setuptools.command.sdist import sdist as sdist_orig
+
+ class _build_py(build_py_orig):
+ def run(self):
+ super().run()
+
+ src_marker = "".join(["src", os.path.sep])
+
+ if pkg_source_path.startswith(src_marker):
+ path = pkg_source_path[len(src_marker) :]
+ else:
+ path = pkg_source_path
+ _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE))
+
+ class _sdist(sdist_orig):
+ def make_release_tree(self, base_dir, files):
+ super().make_release_tree(base_dir, files)
+ _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE))
+
+ return dict(sdist=_sdist, build_py=_build_py)
diff --git a/python/adbc_driver_manager/pyproject.toml b/python/adbc_driver_manager/pyproject.toml
index b65723d..fd4d168 100644
--- a/python/adbc_driver_manager/pyproject.toml
+++ b/python/adbc_driver_manager/pyproject.toml
@@ -32,14 +32,10 @@ homepage = "https://arrow.apache.org"
repository = "https://github.com/apache/arrow-adbc"
[build-system]
-requires = ["Cython", "setuptools >= 61.0.0", "setuptools-scm"]
+requires = ["Cython", "setuptools >= 61.0.0"]
build-backend = "setuptools.build_meta"
[tool.pytest.ini_options]
markers = [
"sqlite: tests that require the SQLite driver",
]
-
-[tool.setuptools_scm]
-root = "../.."
-write_to = "python/adbc_driver_manager/adbc_driver_manager/_version.py"
diff --git a/python/adbc_driver_manager/setup.py b/python/adbc_driver_manager/setup.py
index aef7c9b..e34c916 100644
--- a/python/adbc_driver_manager/setup.py
+++ b/python/adbc_driver_manager/setup.py
@@ -17,28 +17,73 @@
# specific language governing permissions and limitations
# under the License.
+import os
import shutil
from pathlib import Path
from setuptools import Extension, setup
-# setuptools gets confused by relative paths that extend above the project root
-target = Path(__file__).parent / "adbc_driver_manager/adbc_driver_manager.cc"
-shutil.copy(
- Path(__file__).parent / "../../c/driver_manager/adbc_driver_manager.cc", target
-)
+source_root = Path(__file__).parent
+repo_root = source_root.joinpath("../../")
+
+# ------------------------------------------------------------
+# Resolve C++ Sources
+
+sources = [
+ "adbc.h",
+ "c/driver_manager/adbc_driver_manager.cc",
+ "c/driver_manager/adbc_driver_manager.h",
+]
+
+for source in sources:
+ target_filename = source.split("/")[-1]
+ source = repo_root.joinpath(source).resolve()
+ target = source_root.joinpath("adbc_driver_manager", target_filename).resolve()
+ if source.is_file():
+ # In-tree build/creating an sdist: copy from project root to local file
+ # so that setuptools isn't confused
+ shutil.copy(source, target)
+ elif not target.is_file():
+ # Out-of-tree build missing the C++ source files
+ raise FileNotFoundError(str(target))
+ # Else, when building from sdist, the target will exist but not the source
+
+# ------------------------------------------------------------
+# Resolve Version (miniver)
+
+
+def get_version_and_cmdclass(pkg_path):
+ """
+ Load version.py module without importing the whole package.
+
+ Template code from miniver.
+ """
+ from importlib.util import module_from_spec, spec_from_file_location
+
+ spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py"))
+ module = module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.__version__, module.get_cmdclass(pkg_path)
+
+
+version, cmdclass = get_version_and_cmdclass("adbc_driver_manager")
+
+# ------------------------------------------------------------
+# Setup
setup(
+ cmdclass=cmdclass,
ext_modules=[
Extension(
name="adbc_driver_manager._lib",
extra_compile_args=["-std=c++17"],
- include_dirs=["../../", "../../c/driver_manager"],
+ include_dirs=[str(source_root.joinpath("adbc_driver_manager").resolve())],
language="c++",
sources=[
"adbc_driver_manager/_lib.pyx",
"adbc_driver_manager/adbc_driver_manager.cc",
],
)
- ]
+ ],
+ version=version,
)
diff --git a/python/adbc_driver_postgres/.gitignore b/python/adbc_driver_postgres/.gitignore
index 9402188..126488c 100644
--- a/python/adbc_driver_postgres/.gitignore
+++ b/python/adbc_driver_postgres/.gitignore
@@ -17,5 +17,4 @@
adbc_driver_postgres/*.c
adbc_driver_postgres/*.cpp
-adbc_driver_postgres/_version.py
build/
diff --git a/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py b/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
index b09c2f4..c9609ce 100644
--- a/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
+++ b/python/adbc_driver_postgres/adbc_driver_postgres/__init__.py
@@ -19,7 +19,7 @@ import importlib.resources
import adbc_driver_manager
-from ._version import version as __version__
+from ._version import __version__
__all__ = ["connect", "__version__"]
diff --git a/python/adbc_driver_postgres/setup.py b/python/adbc_driver_postgres/adbc_driver_postgres/_static_version.py
similarity index 64%
copy from python/adbc_driver_postgres/setup.py
copy to python/adbc_driver_postgres/adbc_driver_postgres/_static_version.py
index d7ee3ae..7ca903c 100644
--- a/python/adbc_driver_postgres/setup.py
+++ b/python/adbc_driver_postgres/adbc_driver_postgres/_static_version.py
@@ -1,5 +1,4 @@
-#!/usr/bin/env python
-
+# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -17,17 +16,16 @@
# specific language governing permissions and limitations
# under the License.
-import os
-import shutil
-from pathlib import Path
-
-from setuptools import setup
+# Generated by miniver (CC0).
-library = os.environ.get("ADBC_POSTGRES_LIBRARY")
-if not library:
- raise ValueError("Must provide ADBC_POSTGRES_LIBRARY")
+# This file is part of 'miniver': https://github.com/jbweston/miniver
+#
+# This file will be overwritten by setup.py when a source or binary
+# distribution is made. The magic value "__use_git__" is interpreted by
+# _version.py.
-target = Path("./adbc_driver_postgres/libadbc_driver_postgres.so").resolve()
-shutil.copy(library, target)
+version = "__use_git__"
-setup()
+# These values are only set if the distribution was created with 'git archive'
+refnames = "$Format:%D$"
+git_hash = "$Format:%h$"
diff --git a/python/adbc_driver_postgres/adbc_driver_postgres/_version.py b/python/adbc_driver_postgres/adbc_driver_postgres/_version.py
new file mode 100644
index 0000000..2d2fa6c
--- /dev/null
+++ b/python/adbc_driver_postgres/adbc_driver_postgres/_version.py
@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generated by miniver (CC0).
+
+import os
+
+# This file is part of 'miniver': https://github.com/jbweston/miniver
+#
+from collections import namedtuple
+
+Version = namedtuple("Version", ("release", "dev", "labels"))
+
+# No public API
+__all__ = []
+
+package_root = os.path.dirname(os.path.realpath(__file__))
+package_name = os.path.basename(package_root)
+
+STATIC_VERSION_FILE = "_static_version.py"
+
+
+def get_version(version_file=STATIC_VERSION_FILE):
+ version_info = get_static_version_info(version_file)
+ version = version_info["version"]
+ if version == "__use_git__":
+ version = get_version_from_git()
+ if not version:
+ version = get_version_from_git_archive(version_info)
+ if not version:
+ version = Version("unknown", None, None)
+ return pep440_format(version)
+ else:
+ return version
+
+
+def get_static_version_info(version_file=STATIC_VERSION_FILE):
+ version_info = {}
+ with open(os.path.join(package_root, version_file), "rb") as f:
+ exec(f.read(), {}, version_info)
+ return version_info
+
+
+def version_is_from_git(version_file=STATIC_VERSION_FILE):
+ return get_static_version_info(version_file)["version"] == "__use_git__"
+
+
+def pep440_format(version_info):
+ release, dev, labels = version_info
+
+ version_parts = [release]
+ if dev:
+ if release.endswith("-dev") or release.endswith(".dev"):
+ version_parts.append(dev)
+ else: # prefer PEP440 over strict adhesion to semver
+ version_parts.append(".dev{}".format(dev))
+
+ if labels:
+ version_parts.append("+")
+ version_parts.append(".".join(labels))
+
+ return "".join(version_parts)
+
+
+def get_version_from_git():
+ import subprocess
+
+ # git describe --first-parent does not take into account tags from branches
+ # that were merged-in. The '--long' flag gets us the 'dev' version and
+ # git hash, '--always' returns the git hash even if there are no tags.
+ for opts in [["--first-parent"], []]:
+ try:
+ p = subprocess.Popen(
+ ["git", "describe", "--long", "--always"] + opts,
+ cwd=package_root,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ )
+ except OSError:
+ return
+ if p.wait() == 0:
+ break
+ else:
+ return
+
+ description = (
+ p.communicate()[0]
+ .decode()
+ .strip("v") # Tags can have a leading 'v', but the version should not
+ .rstrip("\n")
+ .rsplit("-", 2) # Split the latest tag, commits since tag, and hash
+ )
+
+ try:
+ release, dev, git = description
+ except ValueError: # No tags, only the git hash
+ # prepend 'g' to match with format returned by 'git describe'
+ git = "g{}".format(*description)
+ # XXX: assume version if not given
+ release = "0.0.0"
+ dev = None
+
+ labels = []
+ if dev == "0":
+ dev = None
+ else:
+ labels.append(git)
+
+ try:
+ p = subprocess.Popen(["git", "diff", "--quiet"], cwd=package_root)
+ except OSError:
+ labels.append("confused") # This should never happen.
+ else:
+ if p.wait() == 1:
+ labels.append("dirty")
+
+ return Version(release, dev, labels)
+
+
+# TODO: change this logic when there is a git pretty-format
+# that gives the same output as 'git describe'.
+# Currently we can only tell the tag the current commit is
+# pointing to, or its hash (with no version info)
+# if it is not tagged.
+def get_version_from_git_archive(version_info):
+ try:
+ refnames = version_info["refnames"]
+ git_hash = version_info["git_hash"]
+ except KeyError:
+ # These fields are not present if we are running from an sdist.
+ # Execution should never reach here, though
+ return None
+
+ if git_hash.startswith("$Format") or refnames.startswith("$Format"):
+ # variables not expanded during 'git archive'
+ return None
+
+ VTAG = "tag: v"
+ refs = set(r.strip() for r in refnames.split(","))
+ version_tags = set(r[len(VTAG) :] for r in refs if r.startswith(VTAG))
+ if version_tags:
+ release, *_ = sorted(version_tags) # prefer e.g. "2.0" over "2.0rc1"
+ return Version(release, dev=None, labels=None)
+ else:
+ return Version("unknown", dev=None, labels=["g{}".format(git_hash)])
+
+
+__version__ = get_version()
+
+
+# The following section defines a 'get_cmdclass' function
+# that can be used from setup.py. The '__version__' module
+# global is used (but not modified).
+
+
+def _write_version(fname):
+ # This could be a hard link, so try to delete it first. Is there any way
+ # to do this atomically together with opening?
+ try:
+ os.remove(fname)
+ except OSError:
+ pass
+ with open(fname, "w") as f:
+ f.write(
+ "# This file has been created by setup.py.\n"
+ "version = '{}'\n".format(__version__)
+ )
+
+
+def get_cmdclass(pkg_source_path):
+ from setuptools.command.build_py import build_py as build_py_orig
+ from setuptools.command.sdist import sdist as sdist_orig
+
+ class _build_py(build_py_orig):
+ def run(self):
+ super().run()
+
+ src_marker = "".join(["src", os.path.sep])
+
+ if pkg_source_path.startswith(src_marker):
+ path = pkg_source_path[len(src_marker) :]
+ else:
+ path = pkg_source_path
+ _write_version(os.path.join(self.build_lib, path, STATIC_VERSION_FILE))
+
+ class _sdist(sdist_orig):
+ def make_release_tree(self, base_dir, files):
+ super().make_release_tree(base_dir, files)
+ _write_version(os.path.join(base_dir, pkg_source_path, STATIC_VERSION_FILE))
+
+ return dict(sdist=_sdist, build_py=_build_py)
diff --git a/python/adbc_driver_postgres/pyproject.toml b/python/adbc_driver_postgres/pyproject.toml
index 09e376f..ec425b3 100644
--- a/python/adbc_driver_postgres/pyproject.toml
+++ b/python/adbc_driver_postgres/pyproject.toml
@@ -32,12 +32,8 @@ homepage = "https://arrow.apache.org"
repository = "https://github.com/apache/arrow-adbc"
[build-system]
-requires = ["setuptools >= 61.0.0", "setuptools-scm"]
+requires = ["setuptools >= 61.0.0"]
build-backend = "setuptools.build_meta"
[tool.setuptools]
include-package-data = true
-
-[tool.setuptools_scm]
-root = "../.."
-write_to = "python/adbc_driver_postgres/adbc_driver_postgres/_version.py"
diff --git a/python/adbc_driver_postgres/setup.py b/python/adbc_driver_postgres/setup.py
index d7ee3ae..f4882eb 100644
--- a/python/adbc_driver_postgres/setup.py
+++ b/python/adbc_driver_postgres/setup.py
@@ -23,11 +23,48 @@ from pathlib import Path
from setuptools import setup
+source_root = Path(__file__).parent
+repo_root = source_root.joinpath("../../")
+
+# ------------------------------------------------------------
+# Resolve Shared Library
+
library = os.environ.get("ADBC_POSTGRES_LIBRARY")
if not library:
- raise ValueError("Must provide ADBC_POSTGRES_LIBRARY")
+ if os.environ.get("_ADBC_IS_SDIST", "").strip().lower() in ("1", "true"):
+ print("Building sdist, not requiring ADBC_POSTGRES_LIBRARY")
+ else:
+ raise ValueError("Must provide ADBC_POSTGRES_LIBRARY")
+else:
+ target = source_root.joinpath(
+ "./adbc_driver_postgres/libadbc_driver_postgres.so"
+ ).resolve()
+ shutil.copy(library, target)
+
+# ------------------------------------------------------------
+# Resolve Version (miniver)
+
+
+def get_version_and_cmdclass(pkg_path):
+ """
+ Load version.py module without importing the whole package.
+
+ Template code from miniver.
+ """
+ from importlib.util import module_from_spec, spec_from_file_location
+
+ spec = spec_from_file_location("version", os.path.join(pkg_path, "_version.py"))
+ module = module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.__version__, module.get_cmdclass(pkg_path)
+
+
+version, cmdclass = get_version_and_cmdclass("adbc_driver_postgres")
-target = Path("./adbc_driver_postgres/libadbc_driver_postgres.so").resolve()
-shutil.copy(library, target)
+# ------------------------------------------------------------
+# Setup
-setup()
+setup(
+ cmdclass=cmdclass,
+ version=version,
+)