You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2025/06/25 16:23:14 UTC
(arrow-adbc) branch main updated: feat(python/adbc_driver_manager): Update python driver_manager to load manifests (#3018)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new de9f1e799 feat(python/adbc_driver_manager): Update python driver_manager to load manifests (#3018)
de9f1e799 is described below
commit de9f1e799c59486fa6812a79c60b52f79d405f97
Author: Matt Topol <zo...@gmail.com>
AuthorDate: Wed Jun 25 12:23:08 2025 -0400
feat(python/adbc_driver_manager): Update python driver_manager to load manifests (#3018)
Adding a new function `AdbcDriverManagerDatabaseSetLoadFlags` so that
bindings can set the load flags into the Driver Manager's internal
`TempDatabase` to control the functionality of when to look for and load
manifests.
This also changes the Python driver_manager to load manifests by default
but allows passing a `load_flags` option to manually pass in load flags
to control the behavior. Also adding a new `driver_example_manifest.py`
recipe to the driver_example docs.
---
.gitattributes | 5 +-
.gitignore | 1 +
c/driver_manager/adbc_driver_manager.cc | 23 ++++++--
c/include/arrow-adbc/adbc_driver_manager.h | 15 +++++
docs/source/cpp/driver_example.rst | 10 ++++
docs/source/cpp/recipe_driver/CMakeLists.txt | 8 +++
.../cpp/recipe_driver/driver_example.toml.in | 24 ++++----
.../cpp/recipe_driver/driver_example_manifest.py | 64 ++++++++++++++++++++
.../driver_example_manifest.py.stdout.txt | 1 +
docs/source/cpp/recipe_driver/get_arch.cmake | 68 ++++++++++++++++++++++
go/adbc/drivermgr/adbc_driver_manager.cc | 23 ++++++--
go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h | 15 +++++
.../adbc_driver_manager/_lib.pxd | 7 +++
.../adbc_driver_manager/_lib.pyx | 8 +++
14 files changed, 251 insertions(+), 21 deletions(-)
diff --git a/.gitattributes b/.gitattributes
index 745833b12..3ccebae19 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -17,8 +17,11 @@
*.stdout.txt linguist-generated
c/vendor/* linguist-vendored
-go/adbc/drivermgr/adbc.h linguist-generated
+go/adbc/drivermgr/arrow-adbc/adbc.h linguist-generated
+go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h linguist-generated
go/adbc/drivermgr/adbc_driver_manager.cc linguist-generated
+go/adbc/drivermgr/current_arch.h linguist-generated
+go/adbc/pkg/bigquery/* linguist-generated
go/adbc/pkg/flightsql/* linguist-generated
go/adbc/pkg/panicdummy/* linguist-generated
go/adbc/pkg/snowflake/* linguist-generated
diff --git a/.gitignore b/.gitignore
index c586a3771..68ffd9040 100644
--- a/.gitignore
+++ b/.gitignore
@@ -61,6 +61,7 @@ c/apidoc/objects.inv
docs/example.gz
docs/example1.dat
docs/example3.dat
+docs/source/cpp/recipe_driver/driver_example.toml
python/.eggs/
python/doc/
# Egg metadata
diff --git a/c/driver_manager/adbc_driver_manager.cc b/c/driver_manager/adbc_driver_manager.cc
index e0b244129..b880209bf 100644
--- a/c/driver_manager/adbc_driver_manager.cc
+++ b/c/driver_manager/adbc_driver_manager.cc
@@ -983,6 +983,7 @@ struct TempDatabase {
std::string driver;
std::string entrypoint;
AdbcDriverInitFunc init_func = nullptr;
+ AdbcLoadFlags load_flags = ADBC_LOAD_FLAG_ALLOW_RELATIVE_PATHS;
};
/// Temporary state while the database is being configured.
@@ -1341,6 +1342,19 @@ AdbcStatusCode AdbcDatabaseSetOptionDouble(struct AdbcDatabase* database, const
return ADBC_STATUS_OK;
}
+AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
+ AdbcLoadFlags flags,
+ struct AdbcError* error) {
+ if (database->private_driver) {
+ SetError(error, "Cannot SetLoadFlags after AdbcDatabaseInit");
+ return ADBC_STATUS_INVALID_STATE;
+ }
+
+ TempDatabase* args = reinterpret_cast<TempDatabase*>(database->private_data);
+ args->load_flags = flags;
+ return ADBC_STATUS_OK;
+}
+
AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* database,
AdbcDriverInitFunc init_func,
struct AdbcError* error) {
@@ -1376,11 +1390,12 @@ AdbcStatusCode AdbcDatabaseInit(struct AdbcDatabase* database, struct AdbcError*
status = AdbcLoadDriverFromInitFunc(args->init_func, ADBC_VERSION_1_1_0,
database->private_driver, error);
} else if (!args->entrypoint.empty()) {
- status = AdbcLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
- ADBC_VERSION_1_1_0, database->private_driver, error);
+ status = AdbcFindLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
+ ADBC_VERSION_1_1_0, args->load_flags,
+ database->private_driver, error);
} else {
- status = AdbcLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
- database->private_driver, error);
+ status = AdbcFindLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
+ args->load_flags, database->private_driver, error);
}
if (status != ADBC_STATUS_OK) {
diff --git a/c/include/arrow-adbc/adbc_driver_manager.h b/c/include/arrow-adbc/adbc_driver_manager.h
index 7d221c40a..477584d7c 100644
--- a/c/include/arrow-adbc/adbc_driver_manager.h
+++ b/c/include/arrow-adbc/adbc_driver_manager.h
@@ -137,6 +137,21 @@ AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* databas
AdbcDriverInitFunc init_func,
struct AdbcError* error);
+/// \brief Set the load flags for the driver manager.
+///
+/// This is an extension to the ADBC API. The driver manager shims
+/// the AdbcDatabase* functions to allow you to specify the
+/// driver/entrypoint dynamically. This function lets you set the
+/// load flags explicitly, for applications that can dynamically
+/// load drivers on their own.
+///
+/// If this function isn't called, the default load flags are just to
+/// allow relative paths, disallowing the lookups of manifests.
+ADBC_EXPORT
+AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
+ AdbcLoadFlags flags,
+ struct AdbcError* error);
+
/// \brief Get a human-friendly description of a status code.
ADBC_EXPORT
const char* AdbcStatusCodeMessage(AdbcStatusCode code);
diff --git a/docs/source/cpp/driver_example.rst b/docs/source/cpp/driver_example.rst
index 3730328e0..fd3ac1208 100644
--- a/docs/source/cpp/driver_example.rst
+++ b/docs/source/cpp/driver_example.rst
@@ -45,3 +45,13 @@ package.
data.frame(col = 1:3) |> write_adbc(con, "example.arrows")
con |> read_adbc("SELECT * FROM example.arrows") |> as.data.frame()
unlink("example.arrows")
+
+Driver Manifests
+================
+
+.. recipe:: recipe_driver/driver_example_manifest.py
+
+Driver manifests can provide an easier way to install and manage ADBC drivers,
+via TOML files that describe some metadata along with the path to the driver
+shared library. The driver manager can read these manifests to locate and load
+drivers dynamically.
diff --git a/docs/source/cpp/recipe_driver/CMakeLists.txt b/docs/source/cpp/recipe_driver/CMakeLists.txt
index 39d6df8c5..11bf0282e 100644
--- a/docs/source/cpp/recipe_driver/CMakeLists.txt
+++ b/docs/source/cpp/recipe_driver/CMakeLists.txt
@@ -51,6 +51,14 @@ target_link_libraries(driver_example PRIVATE adbc_driver_framework
install(TARGETS driver_example)
+include(./get_arch.cmake)
+cmake_path(SET DRIVER_PATH "${CMAKE_INSTALL_PREFIX}")
+cmake_path(APPEND DRIVER_PATH "${CMAKE_INSTALL_LIBDIR}"
+ "${CMAKE_SHARED_LIBRARY_PREFIX}driver_example${CMAKE_SHARED_LIBRARY_SUFFIX}")
+
+configure_file(driver_example.toml.in "${CMAKE_CURRENT_SOURCE_DIR}/driver_example.toml"
+ @ONLY)
+
if(ADBC_DRIVER_EXAMPLE_BUILD_TESTS)
fetchcontent_declare(googletest
URL https://github.com/google/googletest/archive/refs/tags/v1.15.1.tar.gz
diff --git a/.gitattributes b/docs/source/cpp/recipe_driver/driver_example.toml.in
similarity index 52%
copy from .gitattributes
copy to docs/source/cpp/recipe_driver/driver_example.toml.in
index 745833b12..724fbbbcf 100644
--- a/.gitattributes
+++ b/docs/source/cpp/recipe_driver/driver_example.toml.in
@@ -15,15 +15,15 @@
# specific language governing permissions and limitations
# under the License.
-*.stdout.txt linguist-generated
-c/vendor/* linguist-vendored
-go/adbc/drivermgr/adbc.h linguist-generated
-go/adbc/drivermgr/adbc_driver_manager.cc linguist-generated
-go/adbc/pkg/flightsql/* linguist-generated
-go/adbc/pkg/panicdummy/* linguist-generated
-go/adbc/pkg/snowflake/* linguist-generated
-python/adbc_driver_flightsql/adbc_driver_flightsql/_static_version.py export-subst
-python/adbc_driver_manager/adbc_driver_manager/_static_version.py export-subst
-python/adbc_driver_postgresql/adbc_driver_postgresql/_static_version.py export-subst
-python/adbc_driver_snowflake/adbc_driver_snowflake/_static_version.py export-subst
-python/adbc_driver_sqlite/adbc_driver_sqlite/_static_version.py export-subst
+name = 'Driver Example'
+publisher = 'arrow-adbc-docs'
+license = 'Apache-2.0'
+version = '@CMAKE_PROJECT_VERSION@'
+source = 'recipe'
+
+[ADBC]
+version = 'v1.1.0'
+
+[Driver]
+[Driver.shared]
+@OS@_@ARCH@@EXTRA@ = '@DRIVER_PATH@'
diff --git a/docs/source/cpp/recipe_driver/driver_example_manifest.py b/docs/source/cpp/recipe_driver/driver_example_manifest.py
new file mode 100644
index 000000000..ce01f8920
--- /dev/null
+++ b/docs/source/cpp/recipe_driver/driver_example_manifest.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# RECIPE STARTS HERE
+#: After verifying the basic driver functionality, we can use the
+#: ``adbc_driver_manager`` Python package's built-in dbapi implementation
+#: to expose a ready-to-go Pythonic database API. This is also useful for
+#: high-level testing!
+#:
+#: First, we'll import pathlib for a few path calculations and the
+#: ``adbc_driver_manager``'s ``dbapi`` module:
+from pathlib import Path
+
+from adbc_driver_manager import dbapi
+
+
+#: Next, we'll define a ``connect()`` function that wraps ``dbapi.connect()``
+#: with the location the .toml manifest file which points to the shared library
+#: we built using ``cmake`` in the previous section.
+#: For the purposes of our tutorial, this will be in current directory.
+def connect(uri: str):
+ # we can point to the manifest file directly
+ manifest_file = Path(".") / "driver_example.toml"
+ if manifest_file.exists():
+ return dbapi.connect(
+ driver=str(manifest_file.resolve()), db_kwargs={"uri": uri}
+ )
+
+ # alternatively, it can look for the manifest file in the user's config
+ # directory ($HOME/.config/adbc/driver_example.toml) or the system's
+ # config directory (/etc/adbc/driver_example.toml)
+ return dbapi.connect(driver="driver_example", db_kwargs={"uri": uri})
+
+
+#: Next, we can give our driver a go! The two pieces we implemented in the driver
+#: were the "bulk ingest" feature and "select all from", so let's see if it works!
+if __name__ == "__main__":
+ import pyarrow
+
+ with connect(uri=Path(__file__).parent.as_uri()) as con:
+ data = pyarrow.table({"col": [1, 2, 3]})
+ with con.cursor() as cur:
+ cur.adbc_ingest("example.arrows", data, mode="create")
+
+ with con.cursor() as cur:
+ cur.execute("SELECT * FROM example.arrows")
+ print(cur.fetchall())
+ # Output: [(1,), (2,), (3,)]
+
+ (Path(__file__).parent / "example.arrows").unlink()
diff --git a/docs/source/cpp/recipe_driver/driver_example_manifest.py.stdout.txt b/docs/source/cpp/recipe_driver/driver_example_manifest.py.stdout.txt
new file mode 100644
index 000000000..aeec2cdf5
--- /dev/null
+++ b/docs/source/cpp/recipe_driver/driver_example_manifest.py.stdout.txt
@@ -0,0 +1 @@
+[(1,), (2,), (3,)]
diff --git a/docs/source/cpp/recipe_driver/get_arch.cmake b/docs/source/cpp/recipe_driver/get_arch.cmake
new file mode 100644
index 000000000..c86e74dcf
--- /dev/null
+++ b/docs/source/cpp/recipe_driver/get_arch.cmake
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if(WIN32)
+ set(OS "windows")
+elseif(APPLE)
+ set(OS "osx")
+else()
+ if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
+ set(OS "freebsd")
+ else()
+ set(OS "linux")
+ endif()
+endif()
+
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64)$")
+ set(ARCH "amd64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i386|i486|i586|i686)$")
+ set(ARCH "x86")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv8b|armv8l|aarch64|arm64|AARCH64|ARM64)$")
+ set(ARCH "arm64")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(armv7l|armhf|arm)$")
+ set(ARCH "arm")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc|ppc64|ppcle|ppc64le)$")
+ set(ARCH "powerpc")
+elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "riscv64")
+ set(ARCH "riscv")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(s390|s390x)$")
+ set(ARCH "s390x")
+else()
+ message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}")
+endif()
+
+if(NOT APPLE)
+ include(CheckCSourceRuns)
+
+ check_c_source_runs([=[
+ #include <stdlib.h>
+ #if defined(__GLIBC__)
+ #error "GLIBC detected"
+ #elif defined(__MUSL__)
+ int main(void) { return EXIT_SUCCESS; }
+ #else
+ #error "Neither GLIBC nor Musl detected"
+ #endif
+ ]=]
+ IS_MUSL)
+
+ if(MINGW)
+ set(EXTRA "_mingw")
+ elseif(IS_MUSL)
+ set(EXTRA "_musl")
+ endif()
+endif()
diff --git a/go/adbc/drivermgr/adbc_driver_manager.cc b/go/adbc/drivermgr/adbc_driver_manager.cc
index e0b244129..b880209bf 100644
--- a/go/adbc/drivermgr/adbc_driver_manager.cc
+++ b/go/adbc/drivermgr/adbc_driver_manager.cc
@@ -983,6 +983,7 @@ struct TempDatabase {
std::string driver;
std::string entrypoint;
AdbcDriverInitFunc init_func = nullptr;
+ AdbcLoadFlags load_flags = ADBC_LOAD_FLAG_ALLOW_RELATIVE_PATHS;
};
/// Temporary state while the database is being configured.
@@ -1341,6 +1342,19 @@ AdbcStatusCode AdbcDatabaseSetOptionDouble(struct AdbcDatabase* database, const
return ADBC_STATUS_OK;
}
+AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
+ AdbcLoadFlags flags,
+ struct AdbcError* error) {
+ if (database->private_driver) {
+ SetError(error, "Cannot SetLoadFlags after AdbcDatabaseInit");
+ return ADBC_STATUS_INVALID_STATE;
+ }
+
+ TempDatabase* args = reinterpret_cast<TempDatabase*>(database->private_data);
+ args->load_flags = flags;
+ return ADBC_STATUS_OK;
+}
+
AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* database,
AdbcDriverInitFunc init_func,
struct AdbcError* error) {
@@ -1376,11 +1390,12 @@ AdbcStatusCode AdbcDatabaseInit(struct AdbcDatabase* database, struct AdbcError*
status = AdbcLoadDriverFromInitFunc(args->init_func, ADBC_VERSION_1_1_0,
database->private_driver, error);
} else if (!args->entrypoint.empty()) {
- status = AdbcLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
- ADBC_VERSION_1_1_0, database->private_driver, error);
+ status = AdbcFindLoadDriver(args->driver.c_str(), args->entrypoint.c_str(),
+ ADBC_VERSION_1_1_0, args->load_flags,
+ database->private_driver, error);
} else {
- status = AdbcLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
- database->private_driver, error);
+ status = AdbcFindLoadDriver(args->driver.c_str(), nullptr, ADBC_VERSION_1_1_0,
+ args->load_flags, database->private_driver, error);
}
if (status != ADBC_STATUS_OK) {
diff --git a/go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h b/go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h
index 7d221c40a..477584d7c 100644
--- a/go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h
+++ b/go/adbc/drivermgr/arrow-adbc/adbc_driver_manager.h
@@ -137,6 +137,21 @@ AdbcStatusCode AdbcDriverManagerDatabaseSetInitFunc(struct AdbcDatabase* databas
AdbcDriverInitFunc init_func,
struct AdbcError* error);
+/// \brief Set the load flags for the driver manager.
+///
+/// This is an extension to the ADBC API. The driver manager shims
+/// the AdbcDatabase* functions to allow you to specify the
+/// driver/entrypoint dynamically. This function lets you set the
+/// load flags explicitly, for applications that can dynamically
+/// load drivers on their own.
+///
+/// If this function isn't called, the default load flags are just to
+/// allow relative paths, disallowing the lookups of manifests.
+ADBC_EXPORT
+AdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(struct AdbcDatabase* database,
+ AdbcLoadFlags flags,
+ struct AdbcError* error);
+
/// \brief Get a human-friendly description of a status code.
ADBC_EXPORT
const char* AdbcStatusCodeMessage(AdbcStatusCode code);
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd b/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd
index d900729aa..013acf8e0 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd
+++ b/python/adbc_driver_manager/adbc_driver_manager/_lib.pxd
@@ -291,3 +291,10 @@ cdef object convert_error(CAdbcStatusCode status, CAdbcError* error)
cdef extern from "arrow-adbc/adbc_driver_manager.h":
const char* CAdbcStatusCodeMessage"AdbcStatusCodeMessage"(CAdbcStatusCode code)
+
+ ctypedef uint32_t CAdbcLoadFlags"AdbcLoadFlags"
+ cdef CAdbcLoadFlags CAdbcLoadFlagDefault"ADBC_LOAD_FLAG_DEFAULT"
+ CAdbcStatusCode AdbcDriverManagerDatabaseSetLoadFlags(
+ CAdbcDatabase* database,
+ CAdbcLoadFlags flags,
+ CAdbcError* error)
diff --git a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
index c9a6250bb..389114934 100644
--- a/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
+++ b/python/adbc_driver_manager/adbc_driver_manager/_lib.pyx
@@ -536,10 +536,18 @@ cdef class AdbcDatabase(_AdbcHandle):
status = AdbcDatabaseNew(&self.database, &c_error)
check_error(status, &c_error)
+ # by default, allow searching for manifests and relative paths
+ status = AdbcDriverManagerDatabaseSetLoadFlags(
+ &self.database, CAdbcLoadFlagDefault, &c_error)
+ check_error(status, &c_error)
+
for key, value in kwargs.items():
if key == "init_func":
status = AdbcDriverManagerDatabaseSetInitFunc(
&self.database, <CAdbcDriverInitFunc> (<uintptr_t> value), &c_error)
+ elif key == "load_flags":
+ status = AdbcDriverManagerDatabaseSetLoadFlags(
+ &self.database, <CAdbcLoadFlags> (value), &c_error)
elif key is None:
raise ValueError("key cannot be None")
elif value is None: