You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/06/12 16:08:24 UTC
(sedona-db) branch main updated: refactor(python/sedonadb)!: Refactor registration of extension components (#940)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new a8321e22a refactor(python/sedonadb)!: Refactor registration of extension components (#940)
a8321e22a is described below
commit a8321e22a135cdbc4e95d9a4f537d688de5ef959
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Fri Jun 12 11:08:19 2026 -0500
refactor(python/sedonadb)!: Refactor registration of extension components (#940)
---
.../sedonadb-zarr/python/sedonadb_zarr/__init__.py | 56 +++++++++-----
python/sedonadb-zarr/tests/test_zarr.py | 10 +--
python/sedonadb/python/sedonadb/context.py | 43 +++++++++--
python/sedonadb/python/sedonadb/datasource.py | 2 +-
python/sedonadb/python/sedonadb/udf.py | 19 ++---
python/sedonadb/src/context.rs | 87 ++++++++++++----------
python/sedonadb/src/import_from.rs | 11 +--
python/sedonadb/tests/io/test_pyogrio.py | 22 ++++++
python/sedonadb/tests/test_aggregate_udf.py | 20 ++---
python/sedonadb/tests/test_udf.py | 45 ++++-------
10 files changed, 181 insertions(+), 134 deletions(-)
diff --git a/python/sedonadb-zarr/python/sedonadb_zarr/__init__.py b/python/sedonadb-zarr/python/sedonadb_zarr/__init__.py
index 653e23557..96a7343de 100644
--- a/python/sedonadb-zarr/python/sedonadb_zarr/__init__.py
+++ b/python/sedonadb-zarr/python/sedonadb_zarr/__init__.py
@@ -18,11 +18,10 @@
"""Zarr support for SedonaDB.
```python
-import sedonadb
-import sedonadb_zarr
+from sedonadb_zarr import Zarr
-con = sedonadb.connect()
-con.read_format(sedonadb_zarr.ZarrFormatSpec(), "file:///path/to/foo.zarr").show()
+sd = sedona.db.connect()
+sd.read_format(Zarr(), "file:///path/to/foo.zarr").show()
```
Importing `sedonadb_zarr` is opt-in — applications that don't import
@@ -31,23 +30,46 @@ it pay no runtime cost.
from typing import Any, Mapping, Optional
+from sedonadb.context import SedonaContext
from sedonadb.datasource import ExternalFormatSpec
+from sedonadb.utility import sedona # noqa: F401
from sedonadb_zarr._lib import PyZarrChunkReader
-class ZarrFormatSpec(ExternalFormatSpec):
+class ZarrExtension:
+ """SedonaDB Zarr extension entrypoint
+
+ This interface enables registration of Zarr components with a Python
+ SedonaContext.
+
+ Examples:
+ >>> from sedonadb_zarr import ZarrExtension
+ >>> sd = sedona.db.connect()
+ >>> sd.register(ZarrExtension())
+ """
+
+ def __sedonadb_extension__(self, ctx: SedonaContext, **kwargs) -> None:
+ if kwargs:
+ raise ValueError("Registration options not supported for ZarrExtension")
+
+ # Register the Zarr() format as a FileFormatFactory for SQL support
+ ctx.register(Zarr())
+
+
+class Zarr(ExternalFormatSpec):
"""`ExternalFormatSpec` for Zarr groups.
- Use with `con.read_format(spec, uri)`:
+ This is registered automatically when registering the module with a
+ SedonaContext. Use with `sd.read_format(spec, uri)`:
```python
- con.read_format(ZarrFormatSpec(), "file:///path/to/foo.zarr")
+ sd.read_format(Zarr(), "file:///path/to/foo.zarr")
```
- Supported `with_options` keys:
-
- - `arrays` (`list[str]`) — explicit subset of group arrays to read.
+ Args:
+ options: Supported options include
+ - `arrays` (`list[str]`) — explicit subset of group arrays to read.
"""
_SUPPORTED_OPTIONS = frozenset({"arrays"})
@@ -57,7 +79,7 @@ class ZarrFormatSpec(ExternalFormatSpec):
@property
def extension(self) -> str:
- return ".zarr"
+ return "zarr"
@property
def list_single_object(self) -> bool:
@@ -65,25 +87,23 @@ class ZarrFormatSpec(ExternalFormatSpec):
# returns zero objects at a `.zarr` prefix.
return True
- def with_options(self, options: Mapping[str, Any]) -> "ZarrFormatSpec":
+ def with_options(self, options: Mapping[str, Any]) -> "Zarr":
unknown = set(options) - self._SUPPORTED_OPTIONS
if unknown:
raise ValueError(
- f"ZarrFormatSpec: unknown option(s) {sorted(unknown)!r}; "
+ f"Zarr: unknown option(s) {sorted(unknown)!r}; "
f"supported: {sorted(self._SUPPORTED_OPTIONS)!r}"
)
merged = {**self._options, **options}
- return ZarrFormatSpec(merged)
+ return Zarr(merged)
def open_reader(self, args: Any) -> PyZarrChunkReader:
uri = args.src.to_url()
if uri is None:
- raise ValueError(
- "ZarrFormatSpec: could not resolve a URL from the source object"
- )
+ raise ValueError("Zarr: could not resolve a URL from the source object")
arrays = self._options.get("arrays")
batch_size = args.batch_size if args.batch_size is not None else 8192
return PyZarrChunkReader(uri, arrays, batch_size)
-__all__ = ["ZarrFormatSpec", "PyZarrChunkReader"]
+__all__ = ["Zarr"]
diff --git a/python/sedonadb-zarr/tests/test_zarr.py b/python/sedonadb-zarr/tests/test_zarr.py
index caaf21c86..f02830ccf 100644
--- a/python/sedonadb-zarr/tests/test_zarr.py
+++ b/python/sedonadb-zarr/tests/test_zarr.py
@@ -50,7 +50,7 @@ def zarr_group(tmp_path):
def test_format_spec_via_read_format(zarr_group):
con = sedonadb.connect()
- df = con.read_format(sedonadb_zarr.ZarrFormatSpec(), f"file://{zarr_group}")
+ df = con.read_format(sedonadb_zarr.Zarr(), f"file://{zarr_group}")
arrow_tab = df.to_arrow_table()
assert arrow_tab.num_rows == 2
assert arrow_tab.column_names == ["raster"]
@@ -71,14 +71,14 @@ def test_format_spec_via_read_format(zarr_group):
def test_format_spec_with_arrays_option(zarr_group):
con = sedonadb.connect()
- spec = sedonadb_zarr.ZarrFormatSpec().with_options({"arrays": ["temperature"]})
+ spec = sedonadb_zarr.Zarr().with_options({"arrays": ["temperature"]})
df = con.read_format(spec, f"file://{zarr_group}")
assert df.to_arrow_table().num_rows == 2
def test_format_spec_class_invariants():
- spec = sedonadb_zarr.ZarrFormatSpec()
- assert spec.extension == ".zarr"
+ spec = sedonadb_zarr.Zarr()
+ assert spec.extension == "zarr"
spec2 = spec.with_options({"arrays": ["temperature"]})
assert spec2 is not spec
assert spec2._options.get("arrays") == ["temperature"]
@@ -115,7 +115,7 @@ def test_dtype_mapping_roundtrips(tmp_path, numpy_dtype):
arr[:] = np.ones((2, 2), dtype=numpy_dtype)
con = sedonadb.connect()
- df = con.read_format(sedonadb_zarr.ZarrFormatSpec(), f"file://{tmp_path}")
+ df = con.read_format(sedonadb_zarr.Zarr(), f"file://{tmp_path}")
tab = df.to_arrow_table()
assert tab.num_rows == 2
diff --git a/python/sedonadb/python/sedonadb/context.py b/python/sedonadb/python/sedonadb/context.py
index e7beb03e3..0e6912e9f 100644
--- a/python/sedonadb/python/sedonadb/context.py
+++ b/python/sedonadb/python/sedonadb/context.py
@@ -498,13 +498,22 @@ class SedonaContext:
else:
return df
- def register_udf(self, udf: Any):
- """Register a user-defined function
+ def register(self, component: Any, **kwargs: Any) -> None:
+ """Register an extension component
+
+ The following types of components are currently supported:
+
+ - Python UDFs annotated with arrow_aggregate_udf or arrow_udf
+ - An ExternalFormatSpec implementing a custom datasource type
+ - An object implementing __sedonadb_extension__(ctx, **kwargs), which
+ is called with this context and any keyword arguments passed.
+
+ The extension interface is experimental and may change.
Args:
- udf: An object implementing the DataFusion PyCapsule protocol
- (i.e., `__datafusion_scalar_udf__`) or a function annotated
- with [arrow_udf][sedonadb.udf.arrow_udf].
+ component: A Python object implementing one of the above protocols.
+ **kwargs: Extension-specific options, supported for specific types
+ of components.
Examples:
@@ -520,7 +529,7 @@ class SedonaContext:
... pa.int64()
... )
...
- >>> sd.register_udf(char_count)
+ >>> sd.register(char_count)
>>> sd.sql("SELECT char_count('abcde') as col").show()
┌───────┐
│ col │
@@ -530,7 +539,27 @@ class SedonaContext:
└───────┘
"""
- self._impl.register_udf(udf)
+ if hasattr(component, "__sedonadb_extension__"):
+ component.__sedonadb_extension__(self, **kwargs)
+ return
+
+ supported_interfaces = (
+ "__sedonadb_internal_udf__",
+ "__sedonadb_internal_aggregate_udf__",
+ "__sedonadb_external_format__",
+ )
+ for interface in supported_interfaces:
+ if hasattr(component, interface):
+ if kwargs:
+ raise ValueError(
+ f"register options not supported for interface {interface}"
+ )
+ self._impl.register_component(component)
+ return
+
+ raise ValueError(
+ f"Can't register extension for object of type {type(component).__name__}"
+ )
@cached_property
def funcs(self) -> Functions:
diff --git a/python/sedonadb/python/sedonadb/datasource.py b/python/sedonadb/python/sedonadb/datasource.py
index 17c5ebdba..7156d46f0 100644
--- a/python/sedonadb/python/sedonadb/datasource.py
+++ b/python/sedonadb/python/sedonadb/datasource.py
@@ -119,7 +119,7 @@ class ExternalFormatSpec:
"""
raise NotImplementedError()
- def __sedona_external_format__(self):
+ def __sedonadb_external_format__(self):
return PyExternalFormat(self)
diff --git a/python/sedonadb/python/sedonadb/udf.py b/python/sedonadb/python/sedonadb/udf.py
index 352df65bc..64a77b4e1 100644
--- a/python/sedonadb/python/sedonadb/udf.py
+++ b/python/sedonadb/python/sedonadb/udf.py
@@ -104,7 +104,7 @@ def arrow_udf(
... pa.string(),
... )
...
- >>> sd.register_udf(some_udf)
+ >>> sd.register(some_udf)
>>> sd.sql("SELECT some_udf(123, 'abc') as col").show()
┌───────────┐
│ col │
@@ -126,7 +126,7 @@ def arrow_udf(
... pa.int64()
... )
...
- >>> sd.register_udf(char_count)
+ >>> sd.register(char_count)
>>> sd.sql("SELECT char_count('abcde') as col").show()
┌───────┐
│ col │
@@ -164,7 +164,7 @@ def arrow_udf(
... return pa.array(shapely.to_wkb(result_shapely))
...
>>>
- >>> sd.register_udf(shapely_udf)
+ >>> sd.register(shapely_udf)
>>> sd.sql("SELECT ST_SRID(shapely_udf(ST_Point(0, 0), 2.0)) as col").show()
┌────────┐
│ col │
@@ -199,8 +199,8 @@ def arrow_udf(
... return random_impl(return_type, num_rows)
...
>>> np.random.seed(487)
- >>> sd.register_udf(random_f32)
- >>> sd.register_udf(random_f64)
+ >>> sd.register(random_f32)
+ >>> sd.register(random_f64)
>>> sd.sql("SELECT random_f32() AS f32, random_f64() as f64;").show()
┌────────────┬─────────────────────┐
│ f32 ┆ f64 │
@@ -297,7 +297,7 @@ class ScalarUdfImpl:
self._volatility = volatility
- def __sedona_internal_udf__(self):
+ def __sedonadb_internal_udf__(self):
return sedona_scalar_udf(
self._invoke_batch,
self._return_type,
@@ -306,9 +306,6 @@ class ScalarUdfImpl:
self._name,
)
- def __datafusion_scalar_udf__(self):
- return self.__sedona_internal_udf__().__datafusion_scalar_udf__()
-
def _callable_kwarg_only_names(f):
sig = inspect.signature(f)
@@ -406,7 +403,7 @@ def arrow_aggregate_udf(
... def evaluate(self):
... return None if self.count == 0 else self.total / self.count
...
- >>> sd.register_udf(my_mean)
+ >>> sd.register(my_mean)
>>> sd.create_data_frame(
... pd.DataFrame({"k": ["a", "a", "b"], "v": [1.0, 3.0, 7.0]})
... ).to_view("t", overwrite=True)
@@ -461,7 +458,7 @@ class AggregateUdfImpl:
else:
self._name = name
- def __sedona_internal_udf__(self):
+ def __sedonadb_internal_aggregate_udf__(self):
# The factory closure is invoked by the Rust kernel once per
# accumulator instance; it captures `self` (which only holds the
# user class and small type lists).
diff --git a/python/sedonadb/src/context.rs b/python/sedonadb/src/context.rs
index 56220616d..bf35c3f0b 100644
--- a/python/sedonadb/src/context.rs
+++ b/python/sedonadb/src/context.rs
@@ -21,13 +21,14 @@ use datafusion_expr::{AggregateUDFImpl, ScalarUDFImpl};
use pyo3::prelude::*;
use sedona::context::SedonaContext;
use sedona::context_builder::SedonaContextBuilder;
+use sedona_datasource::format::ExternalFormatFactory;
use tokio::runtime::Runtime;
use crate::{
dataframe::InternalDataFrame,
datasource::PyExternalFormat,
error::PySedonaError,
- import_from::{import_ffi_scalar_udf, import_table_provider_from_any},
+ import_from::import_table_provider_from_any,
runtime::wait_for_future,
udf::{PyAggregateUdf, PyScalarUdf, PySedonaAggregateUdf, PySedonaScalarUdf},
};
@@ -143,7 +144,7 @@ impl InternalContext {
partitioning: Option<Vec<String>>,
) -> Result<InternalDataFrame, PySedonaError> {
let spec = format_spec
- .call_method0("__sedona_external_format__")?
+ .call_method0("__sedonadb_external_format__")?
.extract::<PyExternalFormat>()?;
let df = wait_for_future(
py,
@@ -257,53 +258,57 @@ impl InternalContext {
.collect())
}
- pub fn register_udf(&mut self, udf: Bound<PyAny>) -> Result<(), PySedonaError> {
- if udf.hasattr("__sedona_internal_udf__")? {
- let internal = udf.getattr("__sedona_internal_udf__")?.call0()?;
- if let Ok(py_scalar_udf) = internal.extract::<PySedonaScalarUdf>() {
- let name = py_scalar_udf.inner.name();
+ pub fn register_component(&mut self, component: Bound<PyAny>) -> Result<(), PySedonaError> {
+ if component.hasattr("__sedonadb_internal_udf__")? {
+ let py_scalar_udf = component
+ .getattr("__sedonadb_internal_udf__")?
+ .call0()?
+ .extract::<PySedonaScalarUdf>()?;
+ let name = py_scalar_udf.inner.name();
+ self.inner
+ .functions
+ .insert_scalar_udf(py_scalar_udf.inner.clone());
+ self.inner.ctx.register_udf(
self.inner
.functions
- .insert_scalar_udf(py_scalar_udf.inner.clone());
- self.inner.ctx.register_udf(
- self.inner
- .functions
- .scalar_udf(name)
- .unwrap()
- .clone()
- .into(),
- );
- return Ok(());
- }
- if let Ok(py_agg_udf) = internal.extract::<PySedonaAggregateUdf>() {
- let name = py_agg_udf.inner.name();
+ .scalar_udf(name)
+ .unwrap()
+ .clone()
+ .into(),
+ );
+ return Ok(());
+ } else if component.hasattr("__sedonadb_internal_aggregate_udf__")? {
+ let py_agg_udf = component
+ .getattr("__sedonadb_internal_aggregate_udf__")?
+ .call0()?
+ .extract::<PySedonaAggregateUdf>()?;
+ let name = py_agg_udf.inner.name();
+ self.inner
+ .functions
+ .insert_aggregate_udf(py_agg_udf.inner.clone());
+ self.inner.ctx.register_udaf(
self.inner
.functions
- .insert_aggregate_udf(py_agg_udf.inner.clone());
- self.inner.ctx.register_udaf(
- self.inner
- .functions
- .aggregate_udf(name)
- .unwrap()
- .clone()
- .into(),
- );
- return Ok(());
- }
- return Err(PySedonaError::SedonaPython(
- "__sedona_internal_udf__ returned an unrecognized UDF type \
- (expected scalar or aggregate)"
- .to_string(),
- ));
- } else if udf.hasattr("__datafusion_scalar_udf__")? {
- let scalar_udf = import_ffi_scalar_udf(&udf)?;
- self.inner.ctx.register_udf(scalar_udf);
+ .aggregate_udf(name)
+ .unwrap()
+ .clone()
+ .into(),
+ );
+ return Ok(());
+ } else if component.hasattr("__sedonadb_external_format__")? {
+ let spec = component
+ .call_method0("__sedonadb_external_format__")?
+ .extract::<PyExternalFormat>()?;
+ let state_ref = self.inner.ctx.state_ref();
+ let mut writable = state_ref.write();
+ writable
+ .register_file_format(Arc::new(ExternalFormatFactory::new(Arc::new(spec))), true)?;
return Ok(());
}
+ // A better error is raised in Python before this point
Err(PySedonaError::SedonaPython(
- "Expected an object implementing __sedona_internal_udf__ or __datafusion_scalar_udf__"
- .to_string(),
+ "Unsupported object".to_string(),
))
}
}
diff --git a/python/sedonadb/src/import_from.rs b/python/sedonadb/src/import_from.rs
index 414c7deb2..530efa3a8 100644
--- a/python/sedonadb/src/import_from.rs
+++ b/python/sedonadb/src/import_from.rs
@@ -27,8 +27,8 @@ use arrow_array::{
use arrow_schema::{Field, Schema};
use datafusion::catalog::TableProvider;
use datafusion_common::{metadata::ScalarAndMetadata, ScalarValue};
-use datafusion_expr::{expr::FieldMetadata, ScalarUDF, ScalarUDFImpl};
-use datafusion_ffi::{table_provider::FFI_TableProvider, udf::FFI_ScalarUDF};
+use datafusion_expr::expr::FieldMetadata;
+use datafusion_ffi::table_provider::FFI_TableProvider;
use pyo3::{
types::{PyAnyMethods, PyCapsule, PyCapsuleMethods},
Bound, PyAny, Python,
@@ -69,13 +69,6 @@ pub fn import_ffi_table_provider(
Ok(provider)
}
-pub fn import_ffi_scalar_udf(obj: &Bound<PyAny>) -> Result<ScalarUDF, PySedonaError> {
- let capsule = obj.getattr("__datafusion_scalar_udf__")?.call0()?;
- let udf_ptr = check_pycapsule(&capsule, "datafusion_scalar_udf")? as *mut FFI_ScalarUDF;
- let udf_impl: Arc<dyn ScalarUDFImpl> = unsafe { udf_ptr.as_ref().unwrap().into() };
- Ok(ScalarUDF::new_from_shared_impl(udf_impl))
-}
-
pub fn import_arrow_array_stream<'py>(
py: Python<'py>,
obj: &Bound<PyAny>,
diff --git a/python/sedonadb/tests/io/test_pyogrio.py b/python/sedonadb/tests/io/test_pyogrio.py
index 8618be5b3..3052761b6 100644
--- a/python/sedonadb/tests/io/test_pyogrio.py
+++ b/python/sedonadb/tests/io/test_pyogrio.py
@@ -28,6 +28,7 @@ import pandas as pd
import pyarrow as pa
import pytest
import sedonadb
+from sedonadb.datasource import PyogrioFormatSpec
import shapely
@@ -373,3 +374,24 @@ def test_read_ogr_partitioned(con):
con.sql("SELECT * FROM partitioned_disabled ORDER BY idx").to_pandas(),
gdf.filter(["idx", "wkb_geometry"]),
)
+
+
+def test_pyogrio_format_register():
+ # Create a dedicated connection here because we're about to modify options
+ sd = sedonadb.connect()
+ sd.register(PyogrioFormatSpec("fgb"))
+
+ n = 1024
+ series = geopandas.GeoSeries.from_xy(
+ list(range(n)), list(range(1, n + 1)), crs="EPSG:3857"
+ )
+ gdf = geopandas.GeoDataFrame({"idx": list(range(n)), "wkb_geometry": series})
+ gdf = gdf.set_geometry(gdf["wkb_geometry"])
+
+ with tempfile.TemporaryDirectory() as td:
+ temp_fgb_path = f"{td}/temp.fgb"
+ gdf.to_file(temp_fgb_path)
+
+ # Should be able to SELECT * from 'file' after registering the format
+ df = sd.sql(f"SELECT * FROM '{temp_fgb_path}' ORDER BY idx")
+ geopandas.testing.assert_geodataframe_equal(df.to_pandas(), gdf)
diff --git a/python/sedonadb/tests/test_aggregate_udf.py b/python/sedonadb/tests/test_aggregate_udf.py
index d742e9ff0..86cbe4af1 100644
--- a/python/sedonadb/tests/test_aggregate_udf.py
+++ b/python/sedonadb/tests/test_aggregate_udf.py
@@ -55,7 +55,7 @@ def _mean_class():
def test_aggregate_udf_global_via_sql(con):
- con.register_udf(_mean_class())
+ con.register(_mean_class())
con.create_data_frame(pd.DataFrame({"v": [1.0, 3.0, 5.0, 7.0]})).to_view(
"t_global", overwrite=True
)
@@ -66,7 +66,7 @@ def test_aggregate_udf_global_via_sql(con):
def test_aggregate_udf_grouped(con):
# Grouped aggregation exercises both update_batch and the merge path
# across groups under the slow Accumulator-per-group fallback.
- con.register_udf(_mean_class())
+ con.register(_mean_class())
df = con.create_data_frame(
pd.DataFrame({"k": ["a", "a", "b", "b", "b"], "v": [1.0, 3.0, 2.0, 4.0, 6.0]})
)
@@ -75,7 +75,7 @@ def test_aggregate_udf_grouped(con):
def test_aggregate_udf_input_nulls_ignored(con):
- con.register_udf(_mean_class())
+ con.register(_mean_class())
df = con.create_data_frame(pd.DataFrame({"v": [1.0, None, 3.0, None, 5.0]}))
out = df.agg(m=con.funcs.my_mean(col("v"))).to_pandas()
pdt.assert_frame_equal(out, pd.DataFrame({"m": [3.0]}))
@@ -120,7 +120,7 @@ def test_aggregate_udf_multi_batch_forces_merge(con):
def evaluate(self):
return None if self.count == 0 else self.total / self.count
- con.register_udf(mean_with_probe)
+ con.register(mean_with_probe)
# 50k rows >> the 8192-row default batch size; keys interleave so both
# groups span every batch. k='a' holds the even values, k='b' the odds.
n = 50_000
@@ -148,7 +148,7 @@ def test_aggregate_udf_multi_batch_forces_merge(con):
def test_aggregate_udf_empty_input_returns_null(con):
# No rows → count==0 → evaluate returns None → SQL NULL surfaces as
# NaN in float64 pandas.
- con.register_udf(_mean_class())
+ con.register(_mean_class())
con.create_data_frame(pd.DataFrame({"v": pd.Series([], dtype="float64")})).to_view(
"t_empty", overwrite=True
)
@@ -183,7 +183,7 @@ def test_aggregate_udf_evaluate_returning_wrong_type_raises(con):
def evaluate(self):
return "not a number"
- con.register_udf(wrong_return_type)
+ con.register(wrong_return_type)
df = con.create_data_frame(pd.DataFrame({"v": [1.0, 2.0, 3.0]}))
with pytest.raises(Exception, match="Could not convert"):
df.agg(n=con.funcs.wrong_return_type(col("v"))).to_pandas()
@@ -214,7 +214,7 @@ def test_aggregate_udf_state_arity_mismatch_raises(con):
def evaluate(self):
return self.n
- con.register_udf(wrong_state_arity)
+ con.register(wrong_state_arity)
# A grouped multi-batch input forces a partial state() call.
n = 20_000
df = con.create_data_frame(
@@ -260,7 +260,7 @@ def test_aggregate_udf_exception_in_method_propagates(con, failing_method):
raise RuntimeError("boom in evaluate")
return self.total
- con.register_udf(boom)
+ con.register(boom)
n = 20_000
df = con.create_data_frame(
pa.table({"k": ["a", "b"] * (n // 2), "v": [float(i) for i in range(n)]})
@@ -294,7 +294,7 @@ def test_aggregate_udf_camel_case_name(con):
return self.n
assert MyRowCount._name == "my_row_count"
- con.register_udf(MyRowCount)
+ con.register(MyRowCount)
df = con.create_data_frame(pd.DataFrame({"v": [1.0, 2.0, 3.0]}))
out = df.agg(n=con.funcs.my_row_count(col("v"))).to_pandas()
pdt.assert_frame_equal(out, pd.DataFrame({"n": [3]}))
@@ -333,7 +333,7 @@ def test_aggregate_udf_shapely_geometry(con):
def evaluate(self):
return None if self.acc is None else shapely.to_wkb(self.acc)
- con.register_udf(geom_union)
+ con.register(geom_union)
con.sql(
"SELECT * FROM (VALUES (ST_Point(0.0, 0.0)), (ST_Point(1.0, 1.0))) AS t(g)"
).to_view("t_geom", overwrite=True)
diff --git a/python/sedonadb/tests/test_udf.py b/python/sedonadb/tests/test_udf.py
index cead6de1a..caa8acc94 100644
--- a/python/sedonadb/tests/test_udf.py
+++ b/python/sedonadb/tests/test_udf.py
@@ -37,7 +37,7 @@ def test_udf_matchers(con):
udf_impl = udf.arrow_udf(pa.binary(), [udf.STRING, udf.NUMERIC])(some_udf)
assert udf_impl._name == "some_udf"
- con.register_udf(udf_impl)
+ con.register(udf_impl)
pd.testing.assert_frame_equal(
con.sql("SELECT some_udf('abcd', 123) as col").to_pandas(),
pd.DataFrame({"col": [b"abcd / 123"]}),
@@ -48,7 +48,7 @@ def test_udf_types(con):
udf_impl = udf.arrow_udf(pa.binary(), [pa.string(), pa.int64()])(some_udf)
assert udf_impl._name == "some_udf"
- con.register_udf(udf_impl)
+ con.register(udf_impl)
pd.testing.assert_frame_equal(
con.sql("SELECT some_udf('abcd', 123) as col").to_pandas(),
pd.DataFrame({"col": [b"abcd / 123"]}),
@@ -59,7 +59,7 @@ def test_udf_any_input(con):
udf_impl = udf.arrow_udf(pa.binary())(some_udf)
assert udf_impl._name == "some_udf"
- con.register_udf(udf_impl)
+ con.register(udf_impl)
pd.testing.assert_frame_equal(
con.sql("SELECT some_udf('abcd', 123) as col").to_pandas(),
pd.DataFrame({"col": [b"abcd / 123"]}),
@@ -70,7 +70,7 @@ def test_udf_return_type_fn(con):
udf_impl = udf.arrow_udf(lambda arg_types, arg_scalars: arg_types[0])(some_udf)
assert udf_impl._name == "some_udf"
- con.register_udf(udf_impl)
+ con.register(udf_impl)
pd.testing.assert_frame_equal(
con.sql("SELECT some_udf('abcd'::BYTEA, 123) as col").to_pandas(),
pd.DataFrame({"col": [b"b'abcd' / 123"]}),
@@ -81,7 +81,7 @@ def test_udf_array_input(con):
udf_impl = udf.arrow_udf(pa.binary(), [udf.STRING, udf.NUMERIC])(some_udf)
assert udf_impl._name == "some_udf"
- con.register_udf(udf_impl)
+ con.register(udf_impl)
pd.testing.assert_frame_equal(
con.sql(
"SELECT some_udf(x, 123) as col FROM (VALUES ('a'), ('b'), ('c')) as t(x)"
@@ -108,7 +108,7 @@ def test_shapely_udf(con):
result_shapely = shapely.buffer(geom, distance)
return pa.array(shapely.to_wkb(result_shapely))
- con.register_udf(shapely_udf)
+ con.register(shapely_udf)
pd.testing.assert_frame_equal(
con.sql("SELECT ST_Area(shapely_udf(ST_Point(0, 0), 2.0)) as col").to_pandas(),
@@ -146,7 +146,7 @@ def test_py_sedona_value(con):
return pa.array(range(len(pa.array(arg))))
- con.register_udf(fn_arg_only)
+ con.register(fn_arg_only)
con.sql("SELECT fn_arg_only(123)").to_arrow_table()
@@ -156,7 +156,7 @@ def test_udf_kwargs(con):
assert repr(return_type) == "SedonaType int64<Int64>"
return pa.array(range(len(pa.array(arg))))
- con.register_udf(fn_return_type)
+ con.register(fn_return_type)
con.sql("SELECT fn_return_type('123')").to_arrow_table()
@udf.arrow_udf(pa.int64())
@@ -164,7 +164,7 @@ def test_udf_kwargs(con):
assert num_rows == 1
return pa.array(range(len(pa.array(arg))))
- con.register_udf(fn_num_rows)
+ con.register(fn_num_rows)
con.sql("SELECT fn_num_rows('123')").to_arrow_table()
@udf.arrow_udf(pa.int64())
@@ -173,7 +173,7 @@ def test_udf_kwargs(con):
assert num_rows == 1
return pa.array(range(len(pa.array(arg))))
- con.register_udf(fn_num_rows_and_return_type)
+ con.register(fn_num_rows_and_return_type)
con.sql("SELECT fn_num_rows_and_return_type('123')").to_arrow_table()
@@ -182,7 +182,7 @@ def test_udf_bad_return_object(con):
def questionable_udf(arg):
return None
- con.register_udf(questionable_udf)
+ con.register(questionable_udf)
with pytest.raises(
sedonadb._lib.SedonaError,
match="Expected result of user-defined function to return an object implementing __arrow_c_array__",
@@ -195,7 +195,7 @@ def test_udf_bad_return_type(con):
def questionable_udf(arg):
return pa.array(["abc"], pa.string())
- con.register_udf(questionable_udf)
+ con.register(questionable_udf)
with pytest.raises(
sedonadb._lib.SedonaError,
match=(
@@ -212,28 +212,9 @@ def test_udf_bad_return_length(con):
def questionable_udf(arg):
return pa.array([b"abc", b"def"], pa.binary())
- con.register_udf(questionable_udf)
+ con.register(questionable_udf)
with pytest.raises(
sedonadb._lib.SedonaError,
match="Expected result of user-defined function to return array of length 1 but got 2",
):
con.sql("SELECT questionable_udf(123) as col").to_pandas()
-
-
-def test_udf_datafusion_to_sedonadb(con):
- udf_impl = udf.arrow_udf(
- pa.binary(), [udf.STRING, udf.NUMERIC], name="some_external_udf"
- )(some_udf)
-
- class UdfWrapper:
- def __init__(self, obj):
- self.obj = obj
-
- def __datafusion_scalar_udf__(self):
- return self.obj.__datafusion_scalar_udf__()
-
- con.register_udf(UdfWrapper(udf_impl))
- pd.testing.assert_frame_equal(
- con.sql("SELECT some_external_udf('abcd', 123) as col").to_pandas(),
- pd.DataFrame({"col": [b"abcd / 123"]}),
- )