You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/09/24 19:29:21 UTC
(sedona-db) branch main updated: feat(rust/sedona): unify registered-format reads (#1324)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new ea680fc44 feat(rust/sedona): unify registered-format reads (#1324)
ea680fc44 is described below
commit ea680fc448ab412117d1279455f2bb1d02c68bc0
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Thu Sep 24 19:29:15 2026 +0000
feat(rust/sedona): unify registered-format reads (#1324)
---
python/sedonadb-zarr/tests/test_zarr.py | 6 +-
python/sedonadb/python/sedonadb/read.py | 65 +++--
python/sedonadb/src/context.rs | 44 ++++
python/sedonadb/tests/io/test_read_csv_json.py | 42 +++-
python/sedonadb/tests/test_context.py | 20 +-
r/sedonadb/NAMESPACE | 2 +
r/sedonadb/R/000-wrappers.R | 25 ++
r/sedonadb/R/context.R | 81 +++++++
r/sedonadb/man/sd_read.Rd | 51 ++++
r/sedonadb/src/init.c | 16 ++
r/sedonadb/src/rust/api.h | 7 +
r/sedonadb/src/rust/src/context.rs | 53 ++++
r/sedonadb/tests/testthat/test-context.R | 47 ++++
rust/sedona-datasource/src/format.rs | 32 ++-
rust/sedona/src/context.rs | 222 ++++++++++++++---
rust/sedona/src/exec.rs | 40 +--
rust/sedona/src/lib.rs | 1 +
rust/sedona/src/object_storage.rs | 213 ++++++++--------
rust/sedona/src/read.rs | 322 +++++++++++++++++++++++++
rust/sedona/src/url_table.rs | 67 +++--
20 files changed, 1108 insertions(+), 248 deletions(-)
diff --git a/python/sedonadb-zarr/tests/test_zarr.py b/python/sedonadb-zarr/tests/test_zarr.py
index 645a24517..56928de73 100644
--- a/python/sedonadb-zarr/tests/test_zarr.py
+++ b/python/sedonadb-zarr/tests/test_zarr.py
@@ -99,7 +99,7 @@ def test_zarr_url_as_table(tmp_path):
url = zarr_path.as_uri()
- # Ground truth: the working explicit-format read of the same `.zarr` group.
+ # Ground truth: the explicit-format read of the same `.zarr` group.
expected = con.read(url, format="zarr").to_arrow_table()
def assert_matches_expected(table):
@@ -112,6 +112,10 @@ def test_zarr_url_as_table(tmp_path):
# The `file://` URL form is the primary SQL-text feature under test.
assert_matches_expected(con.sql(f"SELECT * FROM '{url}'").to_arrow_table())
+ # The generic Python read uses the same extension lookup and single-object
+ # scan as SQL.
+ assert_matches_expected(con.read(url).to_arrow_table())
+
# A bare filesystem path (no `file://` scheme) resolves the same way.
assert_matches_expected(con.sql(f"SELECT * FROM '{zarr_path}'").to_arrow_table())
diff --git a/python/sedonadb/python/sedonadb/read.py b/python/sedonadb/python/sedonadb/read.py
index 48e637328..c9c868487 100644
--- a/python/sedonadb/python/sedonadb/read.py
+++ b/python/sedonadb/python/sedonadb/read.py
@@ -38,11 +38,11 @@ class Read:
def __call__(
self,
- table_paths: Union[str, Path, Iterable[str]],
- *,
+ file_or_files: Union[str, Path, Iterable[str]],
options: Optional[Dict[str, Any]] = None,
- partitioning: Union[str, Iterable[str], None] = None,
format: Union[str, "ExternalFormatSpec", None] = None,
+ *,
+ partitioning: Union[str, Iterable[str], None] = None,
check_extension: bool = False,
) -> DataFrame:
"""Read one or more paths as a DataFrame
@@ -59,7 +59,7 @@ class Read:
- Custom formats via `ExternalFormatSpec` objects registered at runtime
Args:
- table_paths: A str, Path, or iterable of paths containing URLs or
+ file_or_files: A str, Path, or iterable of paths containing URLs or
local paths. Globs (e.g., `path/*.parquet`) and directories are
supported.
options: Optional dictionary of options to pass to the underlying
@@ -75,8 +75,10 @@ class Read:
format: Explicit format specification. Can be a string (e.g., `"parquet"`,
`"fgb"`) or an `ExternalFormatSpec` object. If `None` (the default),
the format is guessed from the file extension.
- check_extension: When `True`, validates that file extensions match the
- specified format. Defaults to `False`.
+ check_extension: When `True`, checks extensions for explicitly selected
+ formats, including compression inference and external-format
+ validation. Directory listings are always filtered to matching
+ files. Defaults to `False`.
Examples:
@@ -88,10 +90,10 @@ class Read:
<sedonadb.dataframe.DataFrame object at ...>
"""
- if isinstance(table_paths, (str, Path)):
- table_paths = [table_paths]
+ if isinstance(file_or_files, (str, Path)):
+ file_or_files = [file_or_files]
- table_paths = [str(path) for path in table_paths]
+ table_paths = [str(path) for path in file_or_files]
if isinstance(partitioning, str):
partitioning = [partitioning]
@@ -115,9 +117,7 @@ class Read:
),
)
- # Special case a few format strings. It is theoretically possible to run these
- # through a generic FileFormatFactory/ListingTable-based path, but for now we
- # special-case some strings.
+ requested_format = format
if format is None:
format = self._guess_format(table_paths)
@@ -141,7 +141,7 @@ class Read:
check_extension=check_extension,
format=PyogrioFormatSpec(format),
)
- elif format == "parquet":
+ if format == "parquet":
options = options.copy()
geometry_columns = options.pop("geometry_columns", None)
@@ -150,31 +150,20 @@ class Read:
validate = options.pop("validate", False)
- return DataFrame(
- self._ctx,
- self._ctx._impl.read_parquet(
- table_paths,
- options,
- geometry_columns,
- validate,
- None if partitioning is None else list(partitioning),
- ),
- )
- elif format == "csv":
- options = options.copy()
- has_header = options.pop("has_header", True)
- delimiter = options.pop("delimiter", ",")
- return DataFrame(
- self._ctx,
- self._ctx._impl.read_csv(table_paths, options, has_header, delimiter),
- )
- elif format == "json":
- return DataFrame(
- self._ctx,
- self._ctx._impl.read_json(table_paths, options),
- )
- else:
- raise ValueError(f"No format registered for extension '{format}'")
+ if geometry_columns is not None:
+ options["geometry_columns"] = geometry_columns
+ options["validate"] = validate
+
+ return DataFrame(
+ self._ctx,
+ self._ctx._impl.read(
+ table_paths,
+ options,
+ None if requested_format is None else format,
+ check_extension,
+ None if partitioning is None else list(partitioning),
+ ),
+ )
def parquet(
self,
diff --git a/python/sedonadb/src/context.rs b/python/sedonadb/src/context.rs
index 070431757..cd81d8667 100644
--- a/python/sedonadb/src/context.rs
+++ b/python/sedonadb/src/context.rs
@@ -56,6 +56,8 @@ fn stringify_options(
.filter_map(|(k, v)| {
if v.is_none(py) {
None
+ } else if let Ok(value) = v.extract::<bool>(py) {
+ Some((k, value.to_string()))
} else {
v.bind(py)
.str()
@@ -207,6 +209,48 @@ impl InternalContext {
Ok(InternalDataFrame::new(df, self.runtime.clone()))
}
+ pub fn read<'py>(
+ &self,
+ py: Python<'py>,
+ table_paths: Vec<String>,
+ options: HashMap<String, Py<PyAny>>,
+ format: Option<&str>,
+ check_extension: bool,
+ partitioning: Option<Vec<String>>,
+ ) -> Result<InternalDataFrame, PySedonaError> {
+ let rust_options = stringify_options(py, options);
+ let format = format
+ .map(|format| format.trim_start_matches('.').to_lowercase())
+ .map(|format| {
+ self.inner
+ .ctx
+ .state()
+ .get_file_format_factory(&format)
+ .ok_or_else(|| {
+ PySedonaError::SedonaPython(format!(
+ "No format registered for extension '{format}'"
+ ))
+ })
+ })
+ .transpose()?;
+ let df = wait_for_future(
+ py,
+ &self.runtime,
+ self.inner.read_with_partitioning(
+ table_paths,
+ &rust_options,
+ format,
+ check_extension,
+ partitioning.map(|cols| {
+ cols.into_iter()
+ .map(|name| (name, DataType::Utf8View))
+ .collect()
+ }),
+ ),
+ )??;
+ Ok(InternalDataFrame::new(df, self.runtime.clone()))
+ }
+
pub fn read_csv<'py>(
&self,
py: Python<'py>,
diff --git a/python/sedonadb/tests/io/test_read_csv_json.py b/python/sedonadb/tests/io/test_read_csv_json.py
index a207755f0..c19cf1824 100644
--- a/python/sedonadb/tests/io/test_read_csv_json.py
+++ b/python/sedonadb/tests/io/test_read_csv_json.py
@@ -31,6 +31,14 @@ def test_read_csv_basic(con):
pdt.assert_frame_equal(out, pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}))
+def test_read_csv_without_expected_extension(con):
+ with tempfile.TemporaryDirectory() as td:
+ p = Path(td) / "csv_input"
+ p.write_text("a,b\n1,x\n")
+ out = con.read.csv(p).to_pandas()
+ pdt.assert_frame_equal(out, pd.DataFrame({"a": [1], "b": ["x"]}))
+
+
def test_read_csv_no_header(con):
with tempfile.TemporaryDirectory() as td:
p = Path(td) / "t.csv"
@@ -65,7 +73,7 @@ def test_read_csv_bad_delimiter_raises(con):
with tempfile.TemporaryDirectory() as td:
p = Path(td) / "t.csv"
p.write_text("a,b\n1,x\n")
- with pytest.raises(SedonaError, match="single byte"):
+ with pytest.raises(SedonaError, match="Error parsing"):
con.read.csv(p, delimiter=";;")
@@ -77,6 +85,14 @@ def test_read_json_ndjson(con):
pdt.assert_frame_equal(out, pd.DataFrame({"a": [1, 2], "b": ["x", "y"]}))
+def test_read_json_without_expected_extension(con):
+ with tempfile.TemporaryDirectory() as td:
+ p = Path(td) / "json_input"
+ p.write_text('{"a": 1, "b": "x"}\n')
+ out = con.read.json(p).to_pandas()
+ pdt.assert_frame_equal(out, pd.DataFrame({"a": [1], "b": ["x"]}))
+
+
def test_read_json_multiple_paths(con):
with tempfile.TemporaryDirectory() as td:
p1 = Path(td) / "a.json"
@@ -93,7 +109,9 @@ def test_generic_read_guesses_csv_extension(con):
p = Path(td) / "t.csv"
p.write_text("a,b\n1,x\n")
out = con.read(p).to_pandas()
+ sql_out = con.sql(f"SELECT * FROM '{p}'").to_pandas()
pdt.assert_frame_equal(out, pd.DataFrame({"a": [1], "b": ["x"]}))
+ pdt.assert_frame_equal(sql_out, out)
def test_generic_read_csv_options_thread_through(con):
@@ -104,6 +122,28 @@ def test_generic_read_csv_options_thread_through(con):
pdt.assert_frame_equal(out, pd.DataFrame({"a": [1], "b": ["x"]}))
+def test_generic_read_explicit_format_without_matching_extension(con):
+ with tempfile.TemporaryDirectory() as td:
+ p = Path(td) / "table.data"
+ p.write_text("a,b\n1,x\n")
+ out = con.read(p, {"delimiter": ","}, "csv").to_pandas()
+ pdt.assert_frame_equal(out, pd.DataFrame({"a": [1], "b": ["x"]}))
+
+
+def test_generic_read_explicit_format_check_extension(con):
+ with tempfile.TemporaryDirectory() as td:
+ directory = Path(td)
+ (directory / "included.csv").write_text("a,b\n1,x\n")
+ (directory / "ignored.txt").write_text("a,b\n2,y\n")
+
+ unchecked = con.read(directory, format="csv", check_extension=False).to_pandas()
+ checked = con.read(directory, format="csv", check_extension=True).to_pandas()
+
+ # Directories are always filtered to protect schema inference and scans.
+ pdt.assert_frame_equal(unchecked, pd.DataFrame({"a": [1], "b": ["x"]}))
+ pdt.assert_frame_equal(checked, pd.DataFrame({"a": [1], "b": ["x"]}))
+
+
def test_generic_read_guesses_json_extension(con):
# sd.read("x.json") should route to the JSON reader by extension.
with tempfile.TemporaryDirectory() as td:
diff --git a/python/sedonadb/tests/test_context.py b/python/sedonadb/tests/test_context.py
index e743fc644..2c48392c0 100644
--- a/python/sedonadb/tests/test_context.py
+++ b/python/sedonadb/tests/test_context.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
import json
+import shutil
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, Mapping
@@ -88,6 +89,15 @@ def test_read_parquet(con, geoarrow_data):
assert len(tab) == 244
+def test_read_parquet_without_expected_extension(con, geoarrow_data, tmp_path):
+ source = geoarrow_data / "example/files/example_geometry_geo.parquet"
+ extensionless = tmp_path / "items"
+ shutil.copyfile(source, extensionless)
+
+ tab = con.read_parquet(extensionless).to_arrow_table()
+ assert tab["geometry"].type.extension_name == "geoarrow.wkb"
+
+
def test_read_parquet_local_glob(con, geoarrow_data):
# The above test uses .glob() method, this test uses the raw string
tab = con.read_parquet(
@@ -306,21 +316,21 @@ def test_read_geoparquet_s3_anonymous_access():
def test_read_parquet_invalid_aws_option():
- """Test that invalid AWS options are caught and provide helpful error messages"""
+ """Test that invalid AWS options are caught"""
con = sedonadb.connect()
url = "s3://wherobots-examples/data/onboarding_1/nyc_buildings.parquet"
# Test with a misspelled AWS option
with pytest.raises(
sedonadb._lib.SedonaError,
- match=r"Unknown AWS option 'aws\.skip_sig'\..*aws\.skip_signature",
+ match='Config value "skip_sig" not found on AwsOptions',
):
con.read_parquet(url, options={"aws.skip_sig": "true"})
# Test with completely unknown AWS option
with pytest.raises(
sedonadb._lib.SedonaError,
- match="Unknown AWS option.*aws.unknown_option.*Valid options are",
+ match='Config value "unknown_option" not found on AwsOptions',
):
con.read_parquet(url, options={"aws.unknown_option": "value"})
@@ -331,12 +341,12 @@ def test_read_parquet_invalid_azure_option():
with pytest.raises(
sedonadb._lib.SedonaError,
- match=r"Unknown Azure option 'azure\.acc_name'\..*azure\.account_name",
+ match='Config value "acc_name" not found on AzureOptions',
):
con.read_parquet(url, options={"azure.acc_name": "test"})
with pytest.raises(
sedonadb._lib.SedonaError,
- match="Unknown Azure option.*azure.unknown_option.*Valid options are",
+ match='Config value "unknown_option" not found on AzureOptions',
):
con.read_parquet(url, options={"azure.unknown_option": "value"})
diff --git a/r/sedonadb/NAMESPACE b/r/sedonadb/NAMESPACE
index 36b70048e..c638a9cf8 100644
--- a/r/sedonadb/NAMESPACE
+++ b/r/sedonadb/NAMESPACE
@@ -58,6 +58,7 @@ export(sd_connect)
export(sd_count)
export(sd_cross_join)
export(sd_ctx_drop_view)
+export(sd_ctx_read)
export(sd_ctx_read_parquet)
export(sd_ctx_read_sf)
export(sd_ctx_register_udf)
@@ -98,6 +99,7 @@ export(sd_join_touches)
export(sd_join_within)
export(sd_left_join)
export(sd_preview)
+export(sd_read)
export(sd_read_parquet)
export(sd_read_sf)
export(sd_register_udf)
diff --git a/r/sedonadb/R/000-wrappers.R b/r/sedonadb/R/000-wrappers.R
index 3c3fe1fc9..a1f99acd4 100644
--- a/r/sedonadb/R/000-wrappers.R
+++ b/r/sedonadb/R/000-wrappers.R
@@ -129,6 +129,30 @@ NULL
}
}
+`InternalContext_read` <- function(self) {
+ function(
+ `paths`,
+ `option_keys`,
+ `option_values`,
+ `partitioning`,
+ `partitioning_set`,
+ `check_extension`,
+ `format` = NULL
+ ) {
+ .savvy_wrap_InternalDataFrame(.Call(
+ savvy_InternalContext_read__impl,
+ `self`,
+ `paths`,
+ `option_keys`,
+ `option_values`,
+ `partitioning`,
+ `partitioning_set`,
+ `check_extension`,
+ `format`
+ ))
+ }
+}
+
`InternalContext_read_parquet` <- function(self) {
function(
`paths`,
@@ -192,6 +216,7 @@ NULL
)
e$`deregister_table` <- `InternalContext_deregister_table`(ptr)
e$`list_functions` <- `InternalContext_list_functions`(ptr)
+ e$`read` <- `InternalContext_read`(ptr)
e$`read_parquet` <- `InternalContext_read_parquet`(ptr)
e$`register_scalar_udf` <- `InternalContext_register_scalar_udf`(ptr)
e$`scalar_udf_xptr` <- `InternalContext_scalar_udf_xptr`(ptr)
diff --git a/r/sedonadb/R/context.R b/r/sedonadb/R/context.R
index d1e9ae19f..9fea789ab 100644
--- a/r/sedonadb/R/context.R
+++ b/r/sedonadb/R/context.R
@@ -210,6 +210,87 @@ sd_ctx_read_parquet <- function(
new_sedonadb_dataframe(ctx, df)
}
+#' Read one or more files into a DataFrame
+#'
+#' Resolves the reader from `format`, or from the file extension when `format`
+#' is `NULL`. The query is executed lazily when results are requested.
+#'
+#' @param file_or_files One or more paths or URIs.
+#' @param options A named list of scalar reader or object-store options.
+#' @param format An optional file format name such as `"parquet"`, `"csv"`,
+#' or `"json"`. By default the format is inferred from the path extension.
+#' @param partitioning Optional character vector of hive-style partition column
+#' names. `NULL` auto-discovers partitions; `character()` disables discovery.
+#' @param check_extension Whether to check extensions for explicitly selected
+#' formats, including compression inference. Directory listings are always
+#' filtered to matching files. Defaults to `FALSE`.
+#' @param ctx A SedonaDB context.
+#'
+#' @returns A sedonadb_dataframe
+#' @export
+#'
+#' @examples
+#' path <- system.file("files/natural-earth_cities_geo.parquet", package = "sedonadb")
+#' sd_read(path) |> head(5) |> sd_preview()
+sd_read <- function(
+ file_or_files,
+ options = list(),
+ format = NULL,
+ partitioning = NULL,
+ check_extension = FALSE
+) {
+ sd_ctx_read(ctx(), file_or_files, options, format, partitioning, check_extension)
+}
+
+#' @rdname sd_read
+#' @export
+sd_ctx_read <- function(
+ ctx,
+ file_or_files,
+ options = list(),
+ format = NULL,
+ partitioning = NULL,
+ check_extension = FALSE
+) {
+ check_ctx(ctx)
+
+ if (
+ !is.list(options) ||
+ (length(options) > 0 && (is.null(names(options)) || any(names(options) == "")))
+ ) {
+ stop("`options` must be a named list", call. = FALSE)
+ }
+
+ if (
+ !is.logical(check_extension) ||
+ length(check_extension) != 1L ||
+ is.na(check_extension)
+ ) {
+ stop("`check_extension` must be TRUE or FALSE", call. = FALSE)
+ }
+
+ df <- ctx$read(
+ as.character(file_or_files),
+ as.character(names(options)),
+ vapply(
+ options,
+ function(value) {
+ if (is.logical(value) && length(value) == 1) {
+ tolower(as.character(value))
+ } else {
+ as.character(value)
+ }
+ },
+ character(1)
+ ),
+ if (is.null(partitioning)) character() else as.character(partitioning),
+ !is.null(partitioning),
+ check_extension,
+ format
+ )
+ new_sedonadb_dataframe(ctx, df)
+}
+
#' Create a DataFrame from SQL
#'
#' The query will only be executed when requested.
diff --git a/r/sedonadb/man/sd_read.Rd b/r/sedonadb/man/sd_read.Rd
new file mode 100644
index 000000000..904003a8c
--- /dev/null
+++ b/r/sedonadb/man/sd_read.Rd
@@ -0,0 +1,51 @@
+% Generated by roxygen2: do not edit by hand
+\name{sd_read}
+\alias{sd_read}
+\alias{sd_ctx_read}
+\title{Read one or more files into a DataFrame}
+\usage{
+sd_read(
+ file_or_files,
+ options = list(),
+ format = NULL,
+ partitioning = NULL,
+ check_extension = FALSE
+)
+
+sd_ctx_read(
+ ctx,
+ file_or_files,
+ options = list(),
+ format = NULL,
+ partitioning = NULL,
+ check_extension = FALSE
+)
+}
+\arguments{
+\item{file_or_files}{One or more paths or URIs.}
+
+\item{options}{A named list of scalar reader or object-store options.}
+
+\item{format}{An optional file format name such as \code{"parquet"}, \code{"csv"},
+or \code{"json"}. By default the format is inferred from the path extension.}
+
+\item{partitioning}{Optional character vector of hive-style partition column
+names. \code{NULL} auto-discovers partitions; \code{character()} disables discovery.}
+
+\item{check_extension}{Whether to check extensions for explicitly selected
+formats, including compression inference. Directory listings are always
+filtered to matching files. Defaults to \code{FALSE}.}
+
+\item{ctx}{A SedonaDB context.}
+}
+\value{
+A sedonadb_dataframe
+}
+\description{
+Resolves the reader from \code{format}, or from the file extension when \code{format}
+is \code{NULL}. The query is executed lazily when results are requested.
+}
+\examples{
+path <- system.file("files/natural-earth_cities_geo.parquet", package = "sedonadb")
+sd_read(path) |> head(5) |> sd_preview()
+}
diff --git a/r/sedonadb/src/init.c b/r/sedonadb/src/init.c
index 9c09d3eb0..5a295dc57 100644
--- a/r/sedonadb/src/init.c
+++ b/r/sedonadb/src/init.c
@@ -123,6 +123,20 @@ SEXP savvy_InternalContext_new__impl(SEXP c_arg__option_keys,
return handle_result(res);
}
+SEXP savvy_InternalContext_read__impl(SEXP self__, SEXP c_arg__paths,
+ SEXP c_arg__option_keys,
+ SEXP c_arg__option_values,
+ SEXP c_arg__partitioning,
+ SEXP c_arg__partitioning_set,
+ SEXP c_arg__check_extension,
+ SEXP c_arg__format) {
+ SEXP res = savvy_InternalContext_read__ffi(
+ self__, c_arg__paths, c_arg__option_keys, c_arg__option_values,
+ c_arg__partitioning, c_arg__partitioning_set, c_arg__check_extension,
+ c_arg__format);
+ return handle_result(res);
+}
+
SEXP savvy_InternalContext_read_parquet__impl(SEXP self__, SEXP c_arg__paths,
SEXP c_arg__option_keys,
SEXP c_arg__option_values,
@@ -404,6 +418,8 @@ static const R_CallMethodDef CallEntries[] = {
(DL_FUNC)&savvy_InternalContext_list_functions__impl, 1},
{"savvy_InternalContext_new__impl",
(DL_FUNC)&savvy_InternalContext_new__impl, 2},
+ {"savvy_InternalContext_read__impl",
+ (DL_FUNC)&savvy_InternalContext_read__impl, 8},
{"savvy_InternalContext_read_parquet__impl",
(DL_FUNC)&savvy_InternalContext_read_parquet__impl, 7},
{"savvy_InternalContext_register_scalar_udf__impl",
diff --git a/r/sedonadb/src/rust/api.h b/r/sedonadb/src/rust/api.h
index da53db53d..602bf959f 100644
--- a/r/sedonadb/src/rust/api.h
+++ b/r/sedonadb/src/rust/api.h
@@ -37,6 +37,13 @@ SEXP savvy_InternalContext_deregister_table__ffi(SEXP self__,
SEXP savvy_InternalContext_list_functions__ffi(SEXP self__);
SEXP savvy_InternalContext_new__ffi(SEXP c_arg__option_keys,
SEXP c_arg__option_values);
+SEXP savvy_InternalContext_read__ffi(SEXP self__, SEXP c_arg__paths,
+ SEXP c_arg__option_keys,
+ SEXP c_arg__option_values,
+ SEXP c_arg__partitioning,
+ SEXP c_arg__partitioning_set,
+ SEXP c_arg__check_extension,
+ SEXP c_arg__format);
SEXP savvy_InternalContext_read_parquet__ffi(SEXP self__, SEXP c_arg__paths,
SEXP c_arg__option_keys,
SEXP c_arg__option_values,
diff --git a/r/sedonadb/src/rust/src/context.rs b/r/sedonadb/src/rust/src/context.rs
index f7e957e2d..9ff881826 100644
--- a/r/sedonadb/src/rust/src/context.rs
+++ b/r/sedonadb/src/rust/src/context.rs
@@ -130,6 +130,59 @@ impl InternalContext {
Ok(new_data_frame(inner, self.runtime.clone()))
}
+ #[allow(clippy::too_many_arguments)]
+ pub fn read(
+ &self,
+ paths: savvy::Sexp,
+ option_keys: savvy::Sexp,
+ option_values: savvy::Sexp,
+ partitioning: savvy::Sexp,
+ partitioning_set: bool,
+ check_extension: bool,
+ format: Option<&str>,
+ ) -> Result<InternalDataFrame> {
+ let paths = savvy::StringSexp::try_from(paths)?
+ .iter()
+ .map(|value| value.to_string())
+ .collect::<Vec<_>>();
+ let keys = savvy::StringSexp::try_from(option_keys)?;
+ let values = savvy::StringSexp::try_from(option_values)?;
+ let options = keys
+ .iter()
+ .zip(values.iter())
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect::<HashMap<_, _>>();
+ let format = format
+ .map(|format| format.trim_start_matches('.').to_lowercase())
+ .map(|format| {
+ self.inner
+ .ctx
+ .state()
+ .get_file_format_factory(&format)
+ .ok_or_else(|| savvy_err!("No format registered for extension '{format}'"))
+ })
+ .transpose()?;
+ let partitioning = if partitioning_set {
+ Some(
+ savvy::StringSexp::try_from(partitioning)?
+ .iter()
+ .map(|name| (name.to_string(), DataType::Utf8View))
+ .collect(),
+ )
+ } else {
+ None
+ };
+
+ let inner_context = self.inner.clone();
+ let inner = wait_for_future_captured_r(&self.runtime, async move {
+ inner_context
+ .read_with_partitioning(paths, &options, format, check_extension, partitioning)
+ .await
+ })??;
+
+ Ok(new_data_frame(inner, self.runtime.clone()))
+ }
+
pub fn sql(&self, query: &str) -> Result<InternalDataFrame> {
let query_string = query.to_string();
let inner_context = self.inner.clone();
diff --git a/r/sedonadb/tests/testthat/test-context.R b/r/sedonadb/tests/testthat/test-context.R
index 14eb62bdd..6eb692e8b 100644
--- a/r/sedonadb/tests/testthat/test-context.R
+++ b/r/sedonadb/tests/testthat/test-context.R
@@ -69,6 +69,53 @@ test_that("sd_read_parquet() works", {
expect_identical(sd_count(sd_read_parquet(c(path, path))), 243 * 2)
})
+test_that("sd_read() resolves registered formats and options", {
+ parquet <- system.file("files/natural-earth_cities_geo.parquet", package = "sedonadb")
+ expect_identical(sd_count(sd_read(parquet)), 243)
+ expect_identical(sd_count(sd_read(c(parquet, parquet), format = "parquet")), 486)
+
+ csv <- tempfile(fileext = ".csv")
+ writeLines(c("x;y", "1;a", "2;b"), csv)
+ expect_equal(
+ sd_read(csv, options = list(delimiter = ";")) |> sd_collect(),
+ data.frame(x = c(1, 2), y = c("a", "b"))
+ )
+
+ expect_error(sd_read(csv, options = list()), NA)
+ expect_error(sd_read(csv, options = c(delimiter = ";")), "named list")
+ expect_error(sd_read(csv, options = list(";")), "named list")
+ expect_error(sd_read(csv, check_extension = NA), "TRUE or FALSE")
+
+ mixed <- tempfile()
+ dir.create(mixed)
+ writeLines(c("x", "1"), file.path(mixed, "included.csv"))
+ writeLines(c("x", "2"), file.path(mixed, "ignored.txt"))
+ expect_identical(sd_count(sd_read(mixed, format = "csv")), 1)
+ expect_identical(
+ sd_count(sd_read(mixed, format = "csv", check_extension = TRUE)),
+ 1
+ )
+
+ partitioned <- tempfile()
+ dir.create(file.path(partitioned, "group=a"), recursive = TRUE)
+ writeLines(c("x", "1"), file.path(partitioned, "group=a", "part.csv"))
+ partitioned_out <- sd_read(
+ partitioned,
+ format = "csv",
+ partitioning = "group"
+ ) |>
+ sd_collect()
+ expect_identical(partitioned_out$group, "a")
+
+ unpartitioned_out <- sd_read(
+ partitioned,
+ format = "csv",
+ partitioning = character()
+ ) |>
+ sd_collect()
+ expect_false("group" %in% names(unpartitioned_out))
+})
+
test_that("sd_read_parquet() forwards reader options", {
path <- system.file("files/natural-earth_cities_geo.parquet", package = "sedonadb")
diff --git a/rust/sedona-datasource/src/format.rs b/rust/sedona-datasource/src/format.rs
index 98f283b1b..16c35de79 100644
--- a/rust/sedona-datasource/src/format.rs
+++ b/rust/sedona-datasource/src/format.rs
@@ -81,9 +81,28 @@ impl FileFormatFactory for ExternalFormatFactory {
_state: &dyn Session,
format_options: &HashMap<String, String>,
) -> Result<Arc<dyn FileFormat>> {
- Ok(Arc::new(ExternalFileFormat {
- spec: self.spec.with_options(format_options)?,
- }))
+ // Match DataFusion's CREATE EXTERNAL TABLE convention: reader options
+ // live in the `format` namespace, while other dotted namespaces belong
+ // to table extensions such as object stores. Keep accepting unprefixed
+ // keys for callers that construct factories directly.
+ let spec_options: HashMap<String, String> = format_options
+ .iter()
+ .filter_map(|(key, value)| {
+ if let Some(key) = key.strip_prefix("format.") {
+ Some((key.to_string(), value.clone()))
+ } else if !key.contains('.') {
+ Some((key.clone(), value.clone()))
+ } else {
+ None
+ }
+ })
+ .collect();
+ let spec = if spec_options.is_empty() {
+ self.spec.clone()
+ } else {
+ self.spec.with_options(&spec_options)?
+ };
+ Ok(Arc::new(ExternalFileFormat { spec }))
}
fn default(&self) -> Arc<dyn FileFormat> {
@@ -100,7 +119,7 @@ impl GetExt for ExternalFormatFactory {
}
#[derive(Debug)]
-pub(crate) struct ExternalFileFormat {
+pub struct ExternalFileFormat {
spec: Arc<dyn ExternalFormatSpec>,
}
@@ -108,6 +127,11 @@ impl ExternalFileFormat {
pub fn new(spec: Arc<dyn ExternalFormatSpec>) -> Self {
Self { spec }
}
+
+ /// Return the fully configured external format specification.
+ pub fn spec(&self) -> &Arc<dyn ExternalFormatSpec> {
+ &self.spec
+ }
}
#[async_trait]
diff --git a/rust/sedona/src/context.rs b/rust/sedona/src/context.rs
index a614f2592..cf3753dc1 100644
--- a/rust/sedona/src/context.rs
+++ b/rust/sedona/src/context.rs
@@ -21,6 +21,7 @@ use std::{
use crate::exec::create_plan_from_sql;
use crate::object_storage::ensure_object_store_registered_with_options;
+use crate::read::{read_provider, resolve_read_format};
use crate::url_table::install_sedona_url_table;
use crate::{
catalog::DynamicObjectStoreCatalog,
@@ -30,7 +31,7 @@ use crate::{
use arrow_array::RecordBatch;
use arrow_schema::{DataType, Schema};
use async_trait::async_trait;
-use datafusion::datasource::file_format::format_as_file_type;
+use datafusion::datasource::file_format::{format_as_file_type, FileFormatFactory};
use datafusion::{
common::plan_err,
error::{DataFusionError, Result},
@@ -50,7 +51,7 @@ use datafusion_expr::sqlparser::dialect::{dialect_from_str, Dialect};
use datafusion_expr::{AggregateUDFImpl, LogicalPlan, LogicalPlanBuilder, ScalarUDFImpl, SortExpr};
use parking_lot::Mutex;
use sedona_common::{sedona_internal_datafusion_err, SedonaOptions};
-use sedona_datasource::provider::external_table;
+use sedona_datasource::format::ExternalFormatFactory;
use sedona_datasource::spec::ExternalFormatSpec;
use sedona_expr::scalar_udf::{IntoScalarKernelRefs, SedonaScalarUDF};
use sedona_expr::{
@@ -659,6 +660,44 @@ impl SedonaContext {
self.ctx.read_table(Arc::new(provider))
}
+ /// Creates a [`DataFrame`] from a file format factory.
+ ///
+ /// When `format` is `None`, a registered factory is resolved from the path
+ /// extension. A supplied factory does not need to be registered. Reader
+ /// options are passed to [`FileFormatFactory::create`] using the same
+ /// key/value convention as SQL `CREATE EXTERNAL TABLE`; object-store options
+ /// such as `aws.*` configure the store for each path. Formats that declare
+ /// themselves single-object formats are scanned without listing contents.
+ /// Directory listings are always filtered using an explicitly supplied
+ /// factory's extension. For individual paths, `check_extension` controls
+ /// whether extension-derived settings such as compression are applied.
+ pub async fn read<P: DataFilePaths>(
+ &self,
+ table_paths: P,
+ options: &HashMap<String, String>,
+ format: Option<Arc<dyn FileFormatFactory>>,
+ check_extension: bool,
+ ) -> Result<DataFrame> {
+ self.read_with_partitioning(table_paths, options, format, check_extension, None)
+ .await
+ }
+
+ /// Internal variant of [`Self::read`] retaining the bindings' existing
+ /// hive-partitioning override.
+ pub async fn read_with_partitioning<P: DataFilePaths>(
+ &self,
+ table_paths: P,
+ options: &HashMap<String, String>,
+ format: Option<Arc<dyn FileFormatFactory>>,
+ check_extension: bool,
+ partitioning: Option<Vec<(String, DataType)>>,
+ ) -> Result<DataFrame> {
+ let urls = table_paths.to_urls()?;
+ let format = resolve_read_format(&self.ctx.state(), &urls, format, check_extension)?;
+ let provider = read_provider(&self.ctx, urls, options, format, partitioning).await?;
+ self.ctx.read_table(provider)
+ }
+
/// Creates a [`DataFrame`] for reading a [ExternalFormatSpec]
///
/// The `partitioning` parameter controls hive-style partition discovery:
@@ -674,30 +713,10 @@ impl SedonaContext {
partitioning: Option<Vec<(String, DataType)>>,
) -> Result<DataFrame> {
let urls = table_paths.to_urls()?;
-
- // Pre-register object store with our custom options before creating GeoParquetReadOptions
- if !urls.is_empty() {
- // Extract the table options from GeoParquetReadOptions for object store registration
- ensure_object_store_registered_with_options(
- &mut self.ctx.state(),
- urls[0].as_str(),
- options,
- )
- .await?;
- }
-
- let provider = if let Some(options) = options {
- // Strip the filesystem-based options
- let options_without_filesystems = options
- .iter()
- .filter(|(k, _)| !k.starts_with("gcs.") && !k.starts_with("aws."))
- .map(|(k, v)| (k.clone(), v.clone()))
- .collect::<HashMap<String, String>>();
- let spec = spec.with_options(&options_without_filesystems)?;
- external_table(spec, &self.ctx, urls, check_extension, partitioning).await?
- } else {
- external_table(spec, &self.ctx, urls, check_extension, partitioning).await?
- };
+ let factory = Arc::new(ExternalFormatFactory::new(spec));
+ let format = resolve_read_format(&self.ctx.state(), &urls, Some(factory), check_extension)?;
+ let options = options.cloned().unwrap_or_default();
+ let provider = read_provider(&self.ctx, urls, &options, format, partitioning).await?;
self.ctx.read_table(provider)
}
@@ -1037,7 +1056,7 @@ mod tests {
use arrow_array::{create_array, ArrayRef, RecordBatchIterator, RecordBatchReader};
use arrow_schema::{DataType, Field, Schema};
- use datafusion::assert_batches_eq;
+ use datafusion::{assert_batches_eq, datasource::file_format::csv::CsvFormatFactory};
use sedona_datasource::spec::{Object, OpenReaderArgs};
use sedona_geometry::types::Edges;
use sedona_schema::{
@@ -1592,6 +1611,155 @@ mod tests {
sedona_types[1],
SedonaType::WkbView(Edges::Planar, lnglat())
);
+
+ // The generic read path resolves the same registered GeoParquet
+ // factory used by SQL URL tables.
+ let df = ctx
+ .read(example, &HashMap::new(), None, false)
+ .await
+ .unwrap();
+ assert_eq!(
+ df.schema().sedona_types().nth(1).unwrap().unwrap(),
+ SedonaType::WkbView(Edges::Planar, lnglat())
+ );
+ }
+
+ #[tokio::test]
+ async fn read_csv_with_format_options() {
+ use datafusion::datasource::file_format::csv::CsvFormatFactory;
+
+ let tmpdir = tempdir().unwrap();
+ let csv = tmpdir.path().join("values.csv");
+ std::fs::write(&csv, "id;value\n1;one\n2;two\n").unwrap();
+ let ctx = SedonaContext::new_local_interactive().await.unwrap();
+ let options = HashMap::from([("delimiter".to_string(), ";".to_string())]);
+
+ let batches = ctx
+ .read(csv.to_string_lossy().to_string(), &options, None, false)
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+
+ assert_batches_eq!(
+ [
+ "+----+-------+",
+ "| id | value |",
+ "+----+-------+",
+ "| 1 | one |",
+ "| 2 | two |",
+ "+----+-------+",
+ ],
+ &batches
+ );
+
+ // An explicit factory is usable without resolving it from the session
+ // registry, including for a path whose suffix does not identify it.
+ let extensionless = tmpdir.path().join("values.data");
+ std::fs::write(&extensionless, "id;value\n3;three\n").unwrap();
+ let batches = ctx
+ .read(
+ extensionless.to_string_lossy().to_string(),
+ &options,
+ Some(Arc::new(CsvFormatFactory::new())),
+ false,
+ )
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_batches_eq!(
+ [
+ "+----+-------+",
+ "| id | value |",
+ "+----+-------+",
+ "| 3 | three |",
+ "+----+-------+",
+ ],
+ &batches
+ );
+ }
+
+ #[tokio::test]
+ async fn read_directory_with_explicit_format_filters_by_extension() {
+ let tmpdir = tempdir().unwrap();
+ std::fs::write(tmpdir.path().join("included.csv"), "id,value\n1,one\n").unwrap();
+ std::fs::write(tmpdir.path().join("ignored.txt"), "id,value\n2,two\n").unwrap();
+
+ let ctx = SedonaContext::new_local_interactive().await.unwrap();
+ let path = tmpdir.path().to_string_lossy().to_string();
+ let format = || Some(Arc::new(CsvFormatFactory::new()) as Arc<dyn FileFormatFactory>);
+
+ let batches = ctx
+ .read(path, &HashMap::new(), format(), false)
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+
+ assert_batches_eq!(
+ [
+ "+----+-------+",
+ "| id | value |",
+ "+----+-------+",
+ "| 1 | one |",
+ "+----+-------+",
+ ],
+ &batches
+ );
+ }
+
+ #[tokio::test]
+ async fn read_explicit_json_and_parquet_without_expected_extension() {
+ let tmpdir = tempdir().unwrap();
+ let ctx = SedonaContext::new_local_interactive().await.unwrap();
+
+ let json = tmpdir.path().join("records");
+ std::fs::write(&json, "{\"id\":1,\"value\":\"one\"}\n").unwrap();
+ let json_format = ctx.ctx.state().get_file_format_factory("json").unwrap();
+ let batches = ctx
+ .read(
+ json.to_string_lossy().to_string(),
+ &HashMap::new(),
+ Some(json_format),
+ false,
+ )
+ .await
+ .unwrap()
+ .collect()
+ .await
+ .unwrap();
+ assert_batches_eq!(
+ [
+ "+----+-------+",
+ "| id | value |",
+ "+----+-------+",
+ "| 1 | one |",
+ "+----+-------+",
+ ],
+ &batches
+ );
+
+ let source = test_geoparquet("example", "geometry").unwrap();
+ let parquet = tmpdir.path().join("items");
+ std::fs::copy(source, &parquet).unwrap();
+ let parquet_format = ctx.ctx.state().get_file_format_factory("parquet").unwrap();
+ let df = ctx
+ .read(
+ parquet.to_string_lossy().to_string(),
+ &HashMap::new(),
+ Some(parquet_format),
+ false,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ df.schema().sedona_types().nth(1).unwrap().unwrap(),
+ SedonaType::WkbView(Edges::Planar, lnglat())
+ );
}
#[derive(Debug)]
diff --git a/rust/sedona/src/exec.rs b/rust/sedona/src/exec.rs
index 754e1d3b5..01f31385a 100644
--- a/rust/sedona/src/exec.rs
+++ b/rust/sedona/src/exec.rs
@@ -19,7 +19,6 @@ use datafusion_expr::{DdlStatement, LogicalPlan};
use crate::{context::SedonaContext, object_storage::register_object_store_and_config_extensions};
use datafusion::{
- config::ConfigFileType,
error::{DataFusionError, Result},
sql::parser::Statement,
};
@@ -37,36 +36,17 @@ pub(crate) async fn create_plan_from_sql(
// datafusion-cli specific options before passing through to datafusion. Otherwise, datafusion
// will raise Configuration errors.
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &plan {
- // To support custom formats, treat error as None
- let format = config_file_type_from_str(&cmd.file_type);
- register_object_store_and_config_extensions(ctx, &cmd.location, &cmd.options, format)
- .await?;
+ register_object_store_and_config_extensions(ctx, &cmd.location, &cmd.options).await?;
}
if let LogicalPlan::Copy(copy_to) = &mut plan {
- let format = config_file_type_from_str(©_to.file_type.get_ext());
-
- register_object_store_and_config_extensions(
- ctx,
- ©_to.output_url,
- ©_to.options,
- format,
- )
- .await?;
+ register_object_store_and_config_extensions(ctx, ©_to.output_url, ©_to.options)
+ .await?;
}
Ok(plan)
}
-pub(crate) fn config_file_type_from_str(ext: &str) -> Option<ConfigFileType> {
- match ext.to_lowercase().as_str() {
- "csv" => Some(ConfigFileType::CSV),
- "json" => Some(ConfigFileType::JSON),
- "parquet" => Some(ConfigFileType::PARQUET),
- _ => None,
- }
-}
-
#[cfg(test)]
mod tests {
@@ -84,9 +64,7 @@ mod tests {
let plan = ctx.ctx.state().create_logical_plan(sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &plan {
- let format = config_file_type_from_str(&cmd.file_type);
- register_object_store_and_config_extensions(&ctx, &cmd.location, &cmd.options, format)
- .await?;
+ register_object_store_and_config_extensions(&ctx, &cmd.location, &cmd.options).await?;
} else {
return plan_err!("LogicalPlan is not a CreateExternalTable");
}
@@ -106,14 +84,8 @@ mod tests {
let plan = ctx.ctx.state().create_logical_plan(sql).await?;
if let LogicalPlan::Copy(cmd) = &plan {
- let format = config_file_type_from_str(&cmd.file_type.get_ext());
- register_object_store_and_config_extensions(
- &ctx,
- &cmd.output_url,
- &cmd.options,
- format,
- )
- .await?;
+ register_object_store_and_config_extensions(&ctx, &cmd.output_url, &cmd.options)
+ .await?;
} else {
return plan_err!("LogicalPlan is not a CreateExternalTable");
}
diff --git a/rust/sedona/src/lib.rs b/rust/sedona/src/lib.rs
index 9c704bfdc..a446566a8 100644
--- a/rust/sedona/src/lib.rs
+++ b/rust/sedona/src/lib.rs
@@ -23,6 +23,7 @@ mod object_storage;
pub mod pool_type;
pub mod projected_reader;
pub mod random_geometry_provider;
+mod read;
pub mod record_batch_reader_provider;
pub mod show;
pub mod size_parser;
diff --git a/rust/sedona/src/object_storage.rs b/rust/sedona/src/object_storage.rs
index f5b366e10..9690b3d64 100644
--- a/rust/sedona/src/object_storage.rs
+++ b/rust/sedona/src/object_storage.rs
@@ -25,16 +25,14 @@ use datafusion::common::config::{
};
#[allow(unused_imports)]
use datafusion::common::{config_err, exec_datafusion_err, exec_err};
-use datafusion::config::ConfigFileType;
use datafusion::datasource::listing::ListingTableUrl;
use datafusion::error::{DataFusionError, Result};
use datafusion::execution::context::SessionState;
-use datafusion::execution::SessionStateBuilder;
+use datafusion::prelude::SessionContext;
use dirs::home_dir;
#[cfg(any(feature = "aws", feature = "azure", feature = "gcp", feature = "http"))]
use object_store::CredentialProvider;
use object_store::ObjectStore;
-use sedona_geoparquet::options::TableGeoParquetOptions;
use url::Url;
#[cfg(feature = "aws")]
@@ -73,75 +71,52 @@ pub async fn ensure_object_store_registered_with_options(
match state.runtime_env().object_store_registry.get_store(url) {
Ok(_) => { /*Nothing to do here, store for this URL is already registered*/ }
Err(_) => {
- let mut builder = SessionStateBuilder::from(state.clone());
-
- // Register the store for this URL with custom options if provided
- match scheme {
- "s3" | "oss" | "cos" => {
- if let Some(table_options) = builder.table_options() {
- let mut aws_options = AwsOptions::default();
-
- // Process custom options if provided
- if let Some(options) = custom_options {
- for (key, value) in options {
- if key.starts_with("aws.") {
- let _ = aws_options.set(key, value);
- }
- }
- }
-
- table_options.extensions.insert(aws_options);
- }
- }
- "gs" | "gcs" => {
- if let Some(table_options) = builder.table_options() {
- let mut gcp_options = GcpOptions::default();
-
- // Process custom options if provided
- if let Some(options) = custom_options {
- for (key, value) in options {
- if key.starts_with("gcp.") {
- let _ = gcp_options.set(key, value);
- }
- }
- }
-
- table_options.extensions.insert(gcp_options);
- }
- }
- #[cfg(feature = "azure")]
- "az" | "abfs" | "abfss" => {
- if let Some(table_options) = builder.table_options() {
- let mut azure_options = AzureOptions::default();
-
- if let Some(options) = custom_options {
- for (key, value) in options {
- if key.starts_with("azure.") {
- let _ = azure_options.set(key, value);
- }
- }
- }
-
- table_options.extensions.insert(azure_options);
- }
- }
- _ => {}
- };
- let new_state = builder.build();
- let store = get_object_store(
- &new_state,
- table_url.scheme(),
- url,
- &new_state.default_table_options(),
- )
- .await?;
- new_state.runtime_env().register_object_store(url, store);
+ let table_options = object_store_table_options(state, scheme, custom_options)?;
+ let store = get_object_store(state, table_url.scheme(), url, &table_options).await?;
+ state.runtime_env().register_object_store(url, store);
}
}
Ok(())
}
+fn object_store_table_options(
+ state: &SessionState,
+ scheme: &str,
+ custom_options: Option<&HashMap<String, String>>,
+) -> Result<TableOptions> {
+ let mut table_options = state.default_table_options();
+ let option_prefix: Option<&str> = match scheme {
+ #[cfg(feature = "aws")]
+ "s3" | "oss" | "cos" => {
+ table_options.extensions.insert(AwsOptions::default());
+ Some("aws.")
+ }
+ #[cfg(feature = "gcp")]
+ "gs" | "gcs" => {
+ table_options.extensions.insert(GcpOptions::default());
+ Some("gcp.")
+ }
+ #[cfg(feature = "azure")]
+ "az" | "abfs" | "abfss" => {
+ table_options.extensions.insert(AzureOptions::default());
+ Some("azure.")
+ }
+ _ => None,
+ };
+
+ if let (Some(prefix), Some(options)) = (option_prefix, custom_options) {
+ let options = options
+ .iter()
+ .filter(|(key, _)| key.starts_with(prefix))
+ .map(|(key, value)| (key.clone(), value.clone()))
+ .collect();
+ table_options.alter_with_string_hash_map(&options)?;
+ }
+
+ Ok(table_options)
+}
+
// Backward compatibility wrapper
pub async fn ensure_object_store_registered(state: &mut SessionState, name: &str) -> Result<()> {
ensure_object_store_registered_with_options(state, name, None).await
@@ -468,7 +443,7 @@ impl ExtensionOptions for AwsOptions {
self.skip_signature.set(rem, value)?;
}
_ => {
- return config_err!("Config value \"{}\" not found on AwsOptions", rem);
+ return config_err!("Config value \"{}\" not found on AwsOptions", key);
}
}
Ok(())
@@ -776,25 +751,21 @@ pub(crate) async fn get_object_store(
}
#[allow(unused_variables)]
-pub(crate) fn register_table_options_extension_from_scheme(ctx: &SedonaContext, scheme: &str) {
+pub(crate) fn register_table_options_extension_from_scheme(ctx: &SessionContext, scheme: &str) {
match scheme {
#[cfg(feature = "aws")]
"s3" | "oss" | "cos" => {
// Register AWS specific table options in the session context:
- ctx.ctx
- .register_table_options_extension(AwsOptions::default())
+ ctx.register_table_options_extension(AwsOptions::default())
}
// For Google Cloud Storage
#[cfg(feature = "gcp")]
"gs" | "gcs" => {
// Register GCP specific table options in the session context:
- ctx.ctx
- .register_table_options_extension(GcpOptions::default())
+ ctx.register_table_options_extension(GcpOptions::default())
}
#[cfg(feature = "azure")]
- "az" | "abfs" | "abfss" => ctx
- .ctx
- .register_table_options_extension(AzureOptions::default()),
+ "az" | "abfs" | "abfss" => ctx.register_table_options_extension(AzureOptions::default()),
// For unsupported schemes, do nothing:
_ => {}
}
@@ -804,7 +775,6 @@ pub(crate) async fn register_object_store_and_config_extensions(
ctx: &SedonaContext,
location: &String,
options: &HashMap<String, String>,
- format: Option<ConfigFileType>,
) -> Result<()> {
// Parse the location URL to extract the scheme and other components
let table_path = ListingTableUrl::parse(location)?;
@@ -815,26 +785,8 @@ pub(crate) async fn register_object_store_and_config_extensions(
// Obtain a reference to the URL
let url = table_path.as_ref();
- // Register the options based on the scheme extracted from the location
- register_table_options_extension_from_scheme(ctx, scheme);
-
- // Clone and modify the default table options based on the provided options
- let mut table_options = ctx.ctx.state().default_table_options();
- if let Some(ref format) = format {
- table_options.set_config_format(format.clone());
- }
-
- let mut options = options.clone();
-
- // If this is an explicitly Parquet configuration, we need to strip GeoParquet
- // options before calling alter_string_with_hash_map.
- if let Some(&ConfigFileType::PARQUET) = format.as_ref() {
- for key in TableGeoParquetOptions::TABLE_OPTIONS_KEYS {
- options.remove(key);
- }
- }
-
- table_options.alter_with_string_hash_map(&options)?;
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
+ let table_options = object_store_table_options(&ctx.ctx.state(), scheme, Some(options))?;
// Retrieve the appropriate object store based on the scheme, URL, and modified table options
let store = get_object_store(&ctx.ctx.state(), scheme, url, &table_options).await?;
@@ -856,6 +808,63 @@ mod tests {
use super::*;
use crate::context::SedonaContext;
+ #[cfg(feature = "aws")]
+ #[test]
+ fn invalid_aws_option_reports_option_name() {
+ let error = AwsOptions::default()
+ .set("aws.skip_sig", "true")
+ .unwrap_err();
+ assert!(error
+ .to_string()
+ .contains("Config value \"skip_sig\" not found on AwsOptions"));
+ }
+
+ #[cfg(feature = "aws")]
+ #[tokio::test]
+ async fn signed_and_unsigned_s3_buckets_use_distinct_stores() -> Result<()> {
+ // Regression for https://github.com/apache/sedona-db/issues/48.
+ let ctx = SedonaContext::new();
+ let unsigned_url = Url::parse("s3://public-bucket/path/a.parquet").unwrap();
+ let signed_url = Url::parse("s3://private-bucket/path/b.parquet").unwrap();
+ let unsigned_options = HashMap::from([
+ ("aws.skip_signature".to_string(), "true".to_string()),
+ ("aws.region".to_string(), "us-east-1".to_string()),
+ ]);
+ let signed_options = HashMap::from([
+ ("aws.access_key_id".to_string(), "test-key".to_string()),
+ (
+ "aws.secret_access_key".to_string(),
+ "test-secret".to_string(),
+ ),
+ ("aws.region".to_string(), "us-east-1".to_string()),
+ ]);
+
+ ensure_object_store_registered_with_options(
+ &mut ctx.ctx.state(),
+ unsigned_url.as_str(),
+ Some(&unsigned_options),
+ )
+ .await?;
+ ensure_object_store_registered_with_options(
+ &mut ctx.ctx.state(),
+ signed_url.as_str(),
+ Some(&signed_options),
+ )
+ .await?;
+
+ let registry = &ctx.ctx.runtime_env().object_store_registry;
+ let unsigned = registry.get_store(&unsigned_url)?;
+ let signed = registry.get_store(&signed_url)?;
+ assert!(!Arc::ptr_eq(&unsigned, &signed));
+
+ let same_public_bucket = Url::parse("s3://public-bucket/other/file.parquet").unwrap();
+ assert!(Arc::ptr_eq(
+ &unsigned,
+ ®istry.get_store(&same_public_bucket)?
+ ));
+ Ok(())
+ }
+
#[cfg(feature = "aws")]
use object_store::{aws::AmazonS3ConfigKey, gcp::GoogleConfigKey};
@@ -886,7 +895,7 @@ mod tests {
let mut plan = ctx.ctx.state().create_logical_plan(&sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &mut plan {
- register_table_options_extension_from_scheme(&ctx, scheme);
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
let mut table_options = ctx.ctx.state().default_table_options();
table_options.alter_with_string_hash_map(&cmd.options)?;
let aws_options = table_options.extensions.get::<AwsOptions>().unwrap();
@@ -930,7 +939,7 @@ mod tests {
let mut plan = ctx.ctx.state().create_logical_plan(&sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &mut plan {
- register_table_options_extension_from_scheme(&ctx, scheme);
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
let mut table_options = ctx.ctx.state().default_table_options();
table_options.alter_with_string_hash_map(&cmd.options)?;
let aws_options = table_options.extensions.get::<AwsOptions>().unwrap();
@@ -956,7 +965,7 @@ mod tests {
let mut plan = ctx.ctx.state().create_logical_plan(&sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &mut plan {
- register_table_options_extension_from_scheme(&ctx, scheme);
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
let mut table_options = ctx.ctx.state().default_table_options();
table_options.alter_with_string_hash_map(&cmd.options)?;
let aws_options = table_options.extensions.get::<AwsOptions>().unwrap();
@@ -984,7 +993,7 @@ mod tests {
let mut plan = ctx.ctx.state().create_logical_plan(&sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &mut plan {
- register_table_options_extension_from_scheme(&ctx, scheme);
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
let mut table_options = ctx.ctx.state().default_table_options();
table_options.alter_with_string_hash_map(&cmd.options)?;
let aws_options = table_options.extensions.get::<AwsOptions>().unwrap();
@@ -1021,7 +1030,7 @@ mod tests {
let mut plan = ctx.ctx.state().create_logical_plan(&sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &mut plan {
- register_table_options_extension_from_scheme(&ctx, scheme);
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
let mut table_options = ctx.ctx.state().default_table_options();
table_options.alter_with_string_hash_map(&cmd.options)?;
let gcp_options = table_options.extensions.get::<GcpOptions>().unwrap();
@@ -1073,7 +1082,7 @@ mod tests {
let mut plan = ctx.ctx.state().create_logical_plan(&sql).await?;
if let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = &mut plan {
- register_table_options_extension_from_scheme(&ctx, scheme);
+ register_table_options_extension_from_scheme(&ctx.ctx, scheme);
let mut table_options = ctx.ctx.state().default_table_options();
table_options.alter_with_string_hash_map(&cmd.options)?;
let azure_options = table_options.extensions.get::<AzureOptions>().unwrap();
diff --git a/rust/sedona/src/read.rs b/rust/sedona/src/read.rs
new file mode 100644
index 000000000..a55e8418a
--- /dev/null
+++ b/rust/sedona/src/read.rs
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Shared implementation for reading URLs through registered file formats.
+
+use std::{collections::HashMap, sync::Arc};
+
+use arrow_schema::DataType;
+use datafusion::{
+ catalog::TableProvider,
+ datasource::{
+ file_format::FileFormatFactory,
+ listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl},
+ },
+ execution::SessionState,
+ prelude::SessionContext,
+};
+use datafusion_common::{exec_err, plan_err, Result};
+
+use crate::object_storage::{
+ ensure_object_store_registered_with_options, register_table_options_extension_from_scheme,
+};
+use sedona_datasource::{format::ExternalFileFormat, provider::external_table};
+
+/// A file format factory plus the path-derived details needed for listing.
+pub(crate) struct ResolvedReadFormat {
+ factory: Arc<dyn FileFormatFactory>,
+ compression: Option<String>,
+ listing_extension: Option<String>,
+}
+
+/// Resolve paths and options into a table provider using an already-selected
+/// [`FileFormatFactory`](datafusion::datasource::file_format::FileFormatFactory).
+///
+/// This is the common implementation behind [`SedonaContext::read`](crate::context::SedonaContext::read)
+/// and SQL URL tables. External formats that opt into
+/// [`list_single_object`](sedona_datasource::spec::ExternalFormatSpec::list_single_object)
+/// are routed through the single-object provider; all other formats use a
+/// [`ListingTable`].
+pub(crate) async fn read_provider(
+ context: &SessionContext,
+ table_paths: Vec<ListingTableUrl>,
+ options: &HashMap<String, String>,
+ format: ResolvedReadFormat,
+ partitioning: Option<Vec<(String, DataType)>>,
+) -> Result<Arc<dyn TableProvider>> {
+ if table_paths.is_empty() {
+ return exec_err!("No table paths were provided");
+ }
+
+ for path in &table_paths {
+ register_table_options_extension_from_scheme(context, path.scheme());
+ ensure_object_store_registered_with_options(
+ &mut context.state(),
+ path.as_str(),
+ Some(options),
+ )
+ .await?;
+ }
+
+ let state = context.state();
+ let mut factory_options = sql_style_options(options);
+ if let Some(compression) = &format.compression {
+ factory_options
+ .entry("format.compression".to_string())
+ .or_insert_with(|| compression.clone());
+ }
+
+ // This is the same key/value entry point used by DataFusion's
+ // ListingTableFactory for CREATE EXTERNAL TABLE. Any format-specific
+ // deserialization belongs to the selected factory, not this generic path.
+ let file_format = format.factory.create(&state, &factory_options)?;
+ if let Some(external) = file_format.downcast_ref::<ExternalFileFormat>() {
+ return external_table(
+ external.spec().clone(),
+ context,
+ table_paths,
+ false,
+ partitioning,
+ )
+ .await;
+ }
+
+ let session_config = context.copied_config();
+ let mut listing_options = ListingOptions::new(file_format)
+ .with_file_extension(format.listing_extension.unwrap_or_default())
+ .with_session_config_options(&session_config);
+
+ if let Some(partitioning) = partitioning {
+ listing_options = listing_options.with_table_partition_cols(partitioning);
+ } else if session_config
+ .options()
+ .execution
+ .listing_table_factory_infer_partitions
+ {
+ let inferred = listing_options
+ .infer_partitions(&state, &table_paths[0])
+ .await?;
+ if !inferred.is_empty() {
+ listing_options = listing_options.with_table_partition_cols(
+ inferred
+ .into_iter()
+ .map(|name| (name, DataType::Utf8View))
+ .collect(),
+ );
+ }
+ }
+
+ let config = ListingTableConfig::new_with_multi_paths(table_paths)
+ .with_listing_options(listing_options)
+ .infer_schema(&state)
+ .await?;
+ Ok(Arc::new(ListingTable::try_new(config)?))
+}
+
+fn sql_style_options(options: &HashMap<String, String>) -> HashMap<String, String> {
+ options
+ .iter()
+ .filter_map(|(key, value)| {
+ // Dotted non-format keys belong to connection/table option
+ // namespaces. Object-store registration consumes the known ones;
+ // retain the historical behavior of ignoring unknown namespaces
+ // rather than passing them to a built-in file-format parser.
+ if key.starts_with("format.") {
+ Some((key.clone(), value.clone()))
+ } else if key.contains('.') {
+ None
+ } else {
+ Some((format!("format.{key}"), value.clone()))
+ }
+ })
+ .collect()
+}
+
+pub(crate) fn resolve_read_format(
+ state: &SessionState,
+ table_paths: &[ListingTableUrl],
+ requested: Option<Arc<dyn FileFormatFactory>>,
+ check_extension: bool,
+) -> Result<ResolvedReadFormat> {
+ if table_paths.is_empty() {
+ return exec_err!("No table paths were provided");
+ }
+
+ if let Some(factory) = requested {
+ let requested_extension = factory.get_ext().trim_start_matches('.').to_lowercase();
+
+ // A collection's own path does not identify the format of the files it
+ // contains. Use the explicitly requested format to keep unrelated files
+ // in the collection out of schema inference and scans.
+ if table_paths.iter().all(ListingTableUrl::is_collection) {
+ return Ok(ResolvedReadFormat {
+ factory,
+ compression: None,
+ listing_extension: Some(requested_extension),
+ });
+ }
+
+ // If we've been instructed not to check the extension, skip
+ if !check_extension {
+ return Ok(ResolvedReadFormat {
+ factory,
+ compression: None,
+ listing_extension: None,
+ });
+ }
+
+ // Keep extension filtering when the paths have the requested suffix,
+ // but allow an explicit format to read extensionless or differently
+ // named objects.
+ let inferred = infer_path_format(table_paths).ok();
+ if let Some((inferred_format, compression)) = inferred {
+ if inferred_format == requested_extension {
+ let extension = match &compression {
+ Some(compression) => format!("{requested_extension}.{compression}"),
+ None => requested_extension,
+ };
+ return Ok(ResolvedReadFormat {
+ factory,
+ compression,
+ listing_extension: Some(extension),
+ });
+ }
+ }
+ return Ok(ResolvedReadFormat {
+ factory,
+ compression: None,
+ listing_extension: None,
+ });
+ }
+
+ let (extension, compression) = infer_path_format(table_paths)?;
+ let factory = state.get_file_format_factory(&extension).ok_or_else(|| {
+ datafusion_common::plan_datafusion_err!("No format registered for extension '{extension}'")
+ })?;
+ let extension = match &compression {
+ Some(compression) => format!("{extension}.{compression}"),
+ None => extension,
+ };
+ Ok(ResolvedReadFormat {
+ factory,
+ compression,
+ listing_extension: Some(extension),
+ })
+}
+
+fn infer_path_format(table_paths: &[ListingTableUrl]) -> Result<(String, Option<String>)> {
+ let mut resolved: Option<(String, Option<String>)> = None;
+ for path in table_paths {
+ let path_without_query = path
+ .as_str()
+ .split(['?', '#'])
+ .next()
+ .unwrap_or(path.as_str())
+ .trim_end_matches('/');
+ let mut next =
+ ListingTableConfig::infer_file_extension_and_compression_type(path_without_query)?;
+ if next.0.is_empty() {
+ return plan_err!(
+ "Can't guess format from path without an extension: '{}'",
+ path.as_str()
+ );
+ }
+ next.0.make_ascii_lowercase();
+ if let Some(current) = &resolved {
+ if current != &next {
+ return plan_err!(
+ "Can't guess format from paths with multiple extensions ('{}' and '{}')",
+ current.0,
+ next.0
+ );
+ }
+ } else {
+ resolved = Some(next);
+ }
+ }
+
+ Ok(resolved.expect("non-empty paths checked by resolve_read_format"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use datafusion::datasource::file_format::csv::CsvFormatFactory;
+
+ #[test]
+ fn sql_style_options_excludes_table_option_namespaces() {
+ let options = HashMap::from([
+ ("batch_size".to_string(), "1024".to_string()),
+ (
+ "format.schema_force_view_types".to_string(),
+ "true".to_string(),
+ ),
+ ("aws.region".to_string(), "us-west-2".to_string()),
+ ("test.option".to_string(), "value".to_string()),
+ ]);
+
+ assert_eq!(
+ sql_style_options(&options),
+ HashMap::from([
+ ("format.batch_size".to_string(), "1024".to_string()),
+ (
+ "format.schema_force_view_types".to_string(),
+ "true".to_string()
+ ),
+ ])
+ );
+ }
+
+ #[test]
+ fn resolve_read_format_rejects_empty_paths() {
+ let state = SessionContext::new().state();
+ let error = match resolve_read_format(&state, &[], None, true) {
+ Ok(_) => panic!("empty paths should fail"),
+ Err(error) => error,
+ };
+ assert_eq!(
+ error.strip_backtrace(),
+ "Execution error: No table paths were provided"
+ );
+ }
+
+ #[test]
+ fn check_extension_controls_explicit_format_inference() {
+ let state = SessionContext::new().state();
+ let paths = [ListingTableUrl::parse("file:///tmp/values.csv.gz").unwrap()];
+
+ let unchecked = resolve_read_format(
+ &state,
+ &paths,
+ Some(Arc::new(CsvFormatFactory::new())),
+ false,
+ )
+ .unwrap();
+ assert_eq!(unchecked.compression, None);
+ assert_eq!(unchecked.listing_extension, None);
+
+ let checked = resolve_read_format(
+ &state,
+ &paths,
+ Some(Arc::new(CsvFormatFactory::new())),
+ true,
+ )
+ .unwrap();
+ assert_eq!(checked.compression.as_deref(), Some("gz"));
+ assert_eq!(checked.listing_extension.as_deref(), Some("csv.gz"));
+ }
+}
diff --git a/rust/sedona/src/url_table.rs b/rust/sedona/src/url_table.rs
index 2b95e077d..51759b004 100644
--- a/rust/sedona/src/url_table.rs
+++ b/rust/sedona/src/url_table.rs
@@ -29,15 +29,13 @@
//! (e.g. `zarr.json` or a raw binary chunk) as the wrong format, which fails.
//!
//! [`enable_sedona_url_table`] installs [`SedonaUrlTableFactory`] instead. It
-//! matches the URL's extension against the session's registered
-//! [`ExternalFormatSpec`](sedona_datasource::spec::ExternalFormatSpec)s: when
-//! the match is a directory-shaped format
-//! ([`list_single_object`](sedona_datasource::spec::ExternalFormatSpec::list_single_object)
-//! `== true`), it builds a
-//! [`SingleObjectExternalTable`](sedona_datasource::provider::SingleObjectExternalTable)
-//! (via [`external_table`]) that passes the URL through untouched. Everything
-//! else — the file-shaped formats DataFusion already handles (GeoParquet,
-//! CSV, ...) — delegates to DataFusion's default resolver unchanged.
+//! resolves the URL's extension through the session's registered
+//! [`FileFormatFactory`](datafusion::datasource::file_format::FileFormatFactory)
+//! using the same implementation as
+//! [`SedonaContext::read`](crate::context::SedonaContext::read). This lets
+//! directory-shaped external formats use a single-object scan while ordinary
+//! file formats continue to use a listing scan. URLs without a registered
+//! extension delegate to DataFusion's default resolver.
use std::sync::Arc;
@@ -52,7 +50,7 @@ use datafusion_catalog::{DynamicFileCatalog, UrlTableFactory};
use datafusion_common::Result;
use datafusion_session::SessionStore;
-use sedona_datasource::{format::ExternalFormatFactory, provider::external_table};
+use crate::read::{read_provider, resolve_read_format};
/// Install SedonaDB's URL-as-table resolver on `ctx`.
///
@@ -87,15 +85,15 @@ pub fn install_sedona_url_table(ctx: &SessionContext) {
factory.session_store().with_state(ctx.state_weak_ref());
}
-/// [`UrlTableFactory`] that pre-routes directory-shaped external formats to
-/// the single-object table path and delegates everything else to
-/// DataFusion's default [`DynamicListTableFactory`].
+/// [`UrlTableFactory`] that resolves registered formats through SedonaDB's
+/// shared read path and delegates unknown URLs to DataFusion's default
+/// [`DynamicListTableFactory`].
///
/// **Experimental.**
#[derive(Debug)]
pub struct SedonaUrlTableFactory {
/// DataFusion's default resolver, used for every URL that does not
- /// resolve to a registered directory-shaped format. Owns the
+ /// resolve to a registered format. Owns the
/// [`SessionStore`] that both it and our routing logic read the live
/// [`SessionState`] from.
inner: DynamicListTableFactory,
@@ -133,10 +131,9 @@ impl SedonaUrlTableFactory {
})
}
- /// Build a single-object table for `url` if its extension resolves to a
- /// registered directory-shaped [`ExternalFormatFactory`]; otherwise
- /// `None` so the caller delegates to DataFusion's default resolver.
- async fn try_single_object(&self, url: &str) -> Result<Option<Arc<dyn TableProvider>>> {
+ /// Resolve a URL through the same registered-format path as
+ /// [`SedonaContext::read`](crate::context::SedonaContext::read).
+ async fn try_read(&self, url: &str) -> Result<Option<Arc<dyn TableProvider>>> {
let Ok(table_url) = ListingTableUrl::parse(url) else {
return Ok(None);
};
@@ -149,19 +146,15 @@ impl SedonaUrlTableFactory {
let Some(factory) = state.get_file_format_factory(&extension) else {
return Ok(None);
};
- let Some(external) = factory.downcast_ref::<ExternalFormatFactory>() else {
- return Ok(None);
- };
- if !external.spec().list_single_object() {
- return Ok(None);
- }
-
- let spec = external.spec().clone();
- // `SingleObjectExternalTable` only needs the runtime environment
- // (for the object store registry), which the reconstructed context
- // shares with the live session via its `Arc<RuntimeEnv>`.
+ let format = resolve_read_format(
+ &state,
+ std::slice::from_ref(&table_url),
+ Some(factory),
+ true,
+ )?;
let ctx = SessionContext::new_with_state(state);
- let provider = external_table(spec, &ctx, vec![table_url], false, None).await?;
+ let provider =
+ read_provider(&ctx, vec![table_url], &Default::default(), format, None).await?;
Ok(Some(provider))
}
}
@@ -175,7 +168,7 @@ impl Default for SedonaUrlTableFactory {
#[async_trait]
impl UrlTableFactory for SedonaUrlTableFactory {
async fn try_new(&self, url: &str) -> Result<Option<Arc<dyn TableProvider>>> {
- if let Some(provider) = self.try_single_object(url).await? {
+ if let Some(provider) = self.try_read(url).await? {
return Ok(Some(provider));
}
self.inner.try_new(url).await
@@ -217,7 +210,10 @@ mod test {
prelude::{col, lit, SessionContext},
};
use datafusion_common::{plan_err, Result, Statistics};
- use sedona_datasource::spec::{ExternalFormatSpec, Object, OpenReaderArgs};
+ use sedona_datasource::{
+ format::ExternalFormatFactory,
+ spec::{ExternalFormatSpec, Object, OpenReaderArgs},
+ };
use tempfile::TempDir;
use url::Url;
@@ -571,10 +567,9 @@ mod test {
#[tokio::test]
async fn url_table_file_shape_still_lists() {
- // A file-shaped format (`list_single_object = false`) must keep
- // resolving through DataFusion's listing path unchanged: the glob
- // fans out to one row per matching file rather than treating the
- // directory as a single object.
+ // A file-shaped format (`list_single_object = false`) must use a
+ // listing scan: the glob fans out to one row per matching file rather
+ // than treating the directory as a single object.
let ctx = create_echo_spec_ctx();
let (temp_dir, _files) = create_echo_spec_temp_dir();