You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/10/09 23:02:47 UTC
(sedona-db) branch main updated: fix(rust/sedona): let CREATE DATABASE fall back past catalog lists that cannot create one (#1410)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new ae4a8e378 fix(rust/sedona): let CREATE DATABASE fall back past catalog lists that cannot create one (#1410)
ae4a8e378 is described below
commit ae4a8e3784500a6d3a670759e547ba9fdd410145
Author: Animesh Kumar <61...@users.noreply.github.com>
AuthorDate: Sat Oct 10 04:32:41 2026 +0530
fix(rust/sedona): let CREATE DATABASE fall back past catalog lists that cannot create one (#1410)
---
c/sedona-extension/src/catalog.rs | 60 ++++++++++++++++-
c/sedona-extension/src/sedona_extension.h | 4 +-
rust/sedona-catalog/src/lib.rs | 6 ++
rust/sedona/src/catalog.rs | 20 ++++--
rust/sedona/src/exec.rs | 107 ++++++++++++++++++++++++++++--
5 files changed, 184 insertions(+), 13 deletions(-)
diff --git a/c/sedona-extension/src/catalog.rs b/c/sedona-extension/src/catalog.rs
index bb8ca718c..edbe3eb48 100644
--- a/c/sedona-extension/src/catalog.rs
+++ b/c/sedona-extension/src/catalog.rs
@@ -274,7 +274,13 @@ unsafe fn write_plan(
}
Err(error) => {
set_ffi_error!(err, "{}", error);
- libc::EINVAL
+ // ENOTSUP tells the caller that this catalog list doesn't support the
+ // operation, so CREATE DATABASE can try the next one.
+ if matches!(error.find_root(), DataFusionError::NotImplemented(_)) {
+ libc::ENOTSUP
+ } else {
+ libc::EINVAL
+ }
}
}
}
@@ -465,6 +471,8 @@ impl SedonaCatalogList for ImportedCatalogProviderList {
fn check_error(code: c_int, err: &SedonaCError) -> Result<()> {
if code == ERRNO_OK {
Ok(())
+ } else if code == libc::ENOTSUP {
+ datafusion_common::not_impl_err!("Catalog callback is not supported ({code}): {err}")
} else {
datafusion_common::exec_err!("Catalog callback failed ({code}): {err}")
}
@@ -565,6 +573,7 @@ mod tests {
calls: Mutex<Vec<String>>,
mutations: Arc<AtomicUsize>,
fail: bool,
+ unsupported: bool,
}
#[async_trait]
@@ -616,6 +625,9 @@ mod tests {
if self.fail {
return datafusion_common::exec_err!("create failed");
}
+ if self.unsupported {
+ return datafusion_common::not_impl_err!("create not supported");
+ }
self.calls.lock().unwrap().push(format!(
"create:{identifier:?}:{options:?}:{}",
input.is_some()
@@ -777,6 +789,52 @@ mod tests {
Ok(())
}
+ #[test]
+ fn unsupported_create_returns_enotsup() {
+ let raw = raw(Arc::new(TestCatalog {
+ unsupported: true,
+ ..Default::default()
+ }));
+ let args = CString::new(r#"{"identifier":["catalog"],"options":{}}"#).unwrap();
+ let mut out = SedonaCExecutionPlan::default();
+ let mut err = SedonaCError::default();
+ let code = unsafe {
+ raw.create_object.unwrap()(&raw, args.as_ptr(), null_mut(), &mut out, &mut err)
+ };
+ assert_eq!(code, libc::ENOTSUP);
+ assert!(out.release.is_none());
+ assert!(err.to_string().contains("create not supported"));
+ }
+
+ #[tokio::test]
+ async fn imported_unsupported_create_is_not_implemented() -> Result<()> {
+ // A NotImplemented error survives the round trip through the C interface, so an
+ // imported catalog list can decline CREATE DATABASE and let the next one handle it.
+ let catalog = Arc::new(TestCatalog {
+ unsupported: true,
+ ..Default::default()
+ });
+ let imported = ImportedCatalogProviderList::try_new(raw(catalog), runtime())?;
+ let error = imported
+ .create_object(
+ &SessionContext::new().state(),
+ &["catalog"],
+ &CreateObjectOptions::default(),
+ None,
+ )
+ .await
+ .unwrap_err();
+ assert!(
+ matches!(error.find_root(), DataFusionError::NotImplemented(_)),
+ "{error}"
+ );
+ assert!(
+ error.to_string().contains("create not supported"),
+ "{error}"
+ );
+ Ok(())
+ }
+
#[test]
fn callback_transfers_input_ownership_and_rejects_invalid_json() {
let raw = raw(Arc::new(TestCatalog::default()));
diff --git a/c/sedona-extension/src/sedona_extension.h b/c/sedona-extension/src/sedona_extension.h
index 69b521275..5f200d66b 100644
--- a/c/sedona-extension/src/sedona_extension.h
+++ b/c/sedona-extension/src/sedona_extension.h
@@ -526,7 +526,9 @@ struct SedonaCCatalogProviderList {
/// "definition": null|string}}. Missing options fields use their Rust defaults.
/// A NULL input means no plan; otherwise move it and clear its release callback
/// when taking ownership. out is required on success and defers the mutation
- /// (including existence/conflict checks) until execution.
+ /// (including existence/conflict checks) until execution. Return ENOTSUP if this
+ /// catalog list cannot create the object: for a new top-level catalog (CREATE
+ /// DATABASE), SedonaDB then offers it to the next registered catalog list.
int (*create_object)(const struct SedonaCCatalogProviderList* self, const char* args,
struct SedonaCExecutionPlan* input,
struct SedonaCExecutionPlan* out, struct SedonaCError* err);
diff --git a/rust/sedona-catalog/src/lib.rs b/rust/sedona-catalog/src/lib.rs
index 8083b5d29..9fcedddbe 100644
--- a/rust/sedona-catalog/src/lib.rs
+++ b/rust/sedona-catalog/src/lib.rs
@@ -134,6 +134,12 @@ pub trait SedonaCatalogList: Debug + Send + Sync {
/// external tables, namespaces, and indexes. `session` belongs to
/// this call and must not be retained by the catalog. The returned plan applies
/// conflict behavior and validates the parent namespace when executed.
+ ///
+ /// A new top-level catalog (`CREATE DATABASE`) is offered to the newest
+ /// registration first. An implementation that cannot create one should return a
+ /// `NotImplemented` error, so that the next older registration, and then the
+ /// built-in catalog list, can create it instead. Any other error is returned to
+ /// the caller.
async fn create_object(
&self,
session: &dyn Session,
diff --git a/rust/sedona/src/catalog.rs b/rust/sedona/src/catalog.rs
index 6d6e8e8bb..2d0b342b6 100644
--- a/rust/sedona/src/catalog.rs
+++ b/rust/sedona/src/catalog.rs
@@ -37,7 +37,9 @@ use sedona_catalog::{CatalogObjectType, SedonaCatalogList};
/// register object stores dynamically for file locations.
#[derive(Debug)]
pub struct SedonaCatalogRegistry {
- foreign: RwLock<Vec<Arc<dyn SedonaCatalogList>>>,
+ /// Registered foreign catalog lists, oldest first. Registration replaces the
+ /// slice so readers can take a snapshot without copying it.
+ foreign: RwLock<Arc<[Arc<dyn SedonaCatalogList>]>>,
fallback: Arc<dyn CatalogProviderList>,
state: Weak<RwLock<SessionState>>,
dynamic_object_store: bool,
@@ -51,7 +53,7 @@ impl SedonaCatalogRegistry {
dynamic_object_store: bool,
) -> Self {
Self {
- foreign: RwLock::new(Vec::new()),
+ foreign: RwLock::new(Vec::new().into()),
fallback,
state,
dynamic_object_store,
@@ -60,12 +62,13 @@ impl SedonaCatalogRegistry {
/// Add a foreign catalog list. Later registrations take precedence.
pub fn register_foreign(&self, catalogs: Arc<dyn SedonaCatalogList>) {
- self.foreign.write().push(catalogs);
+ let mut foreign = self.foreign.write();
+ *foreign = foreign.iter().cloned().chain([catalogs]).collect();
}
/// Resolve catalog ownership asynchronously, newest registration first.
pub async fn foreign_catalog(&self, name: &str) -> Result<Option<Arc<dyn SedonaCatalogList>>> {
- let foreign = self.foreign.read().clone();
+ let foreign = self.foreign_lists();
for catalogs in foreign.iter().rev() {
if catalogs
.list_identifiers(&[name], Some(0))
@@ -81,10 +84,17 @@ impl SedonaCatalogRegistry {
Ok(None)
}
- /// Destination for new top-level catalogs.
+ /// The most recently registered foreign catalog list.
pub fn latest_foreign(&self) -> Option<Arc<dyn SedonaCatalogList>> {
self.foreign.read().last().cloned()
}
+
+ /// The registered foreign catalog lists in registration order, oldest
+ /// first. Later registrations take precedence, so callers that pick an
+ /// owner iterate this slice in reverse.
+ pub fn foreign_lists(&self) -> Arc<[Arc<dyn SedonaCatalogList>]> {
+ self.foreign.read().clone()
+ }
}
impl CatalogProviderList for SedonaCatalogRegistry {
diff --git a/rust/sedona/src/exec.rs b/rust/sedona/src/exec.rs
index b9aad826f..d0488c514 100644
--- a/rust/sedona/src/exec.rs
+++ b/rust/sedona/src/exec.rs
@@ -16,7 +16,7 @@
// under the License.
use crate::{context::SedonaContext, object_storage::register_object_store_and_config_extensions};
use datafusion::{error::Result, sql::parser::Statement};
-use datafusion_common::{exec_err, SchemaReference, TableReference};
+use datafusion_common::{exec_err, DataFusionError, SchemaReference, TableReference};
use datafusion_expr::{DdlStatement, LogicalPlan};
use datafusion_physical_plan::ExecutionPlan;
use sedona_catalog::{CatalogObjectType, CreateMode, CreateObjectOptions, DropObjectOptions};
@@ -195,15 +195,29 @@ pub(crate) async fn resolve_sedona_catalog_ddl(
}
let registry = ctx.catalog_registry();
let owner = registry.foreign_catalog(&identifier[0]).await?;
- let owner = if matches!(ddl, DdlStatement::CreateCatalog(_)) && owner.is_none() {
+ if matches!(ddl, DdlStatement::CreateCatalog(_)) && owner.is_none() {
// Existing built-in catalogs remain owned by DataFusion.
if state.catalog_list().catalog(&identifier[0]).is_some() {
return Ok(None);
}
- registry.latest_foreign()
- } else {
- owner
- };
+ // A new catalog goes to the newest registration that supports creating
+ // one. A catalog list that does not returns a NotImplemented error, and the
+ // next older registration is tried, then DataFusion's built-in list.
+ reject_unsupported_create_metadata(ddl)?;
+ let identifier: Vec<&str> = identifier.iter().map(String::as_str).collect();
+ for candidate in registry.foreign_lists().iter().rev() {
+ match candidate
+ .create_object(&state, &identifier, &create, None)
+ .await
+ {
+ Err(err) if matches!(err.find_root(), DataFusionError::NotImplemented(_)) => {
+ continue
+ }
+ result => return result.map(Some),
+ }
+ }
+ return Ok(None);
+ }
let Some(owner) = owner else {
return Ok(None);
};
@@ -383,6 +397,11 @@ mod tests {
drops: Mutex<Vec<(Vec<String>, DropObjectOptions)>>,
ddl_output: Option<Arc<dyn ExecutionPlan>>,
fail: bool,
+ /// Refuse to create catalogs with NotImplemented, as a catalog list
+ /// without top-level catalog creation would.
+ no_catalogs: bool,
+ /// Fail to create catalogs with an ordinary error.
+ catalog_error: bool,
}
fn owned(parts: &[&str]) -> Vec<String> {
@@ -443,6 +462,14 @@ mod tests {
input: Option<Arc<dyn ExecutionPlan>>,
) -> Result<Arc<dyn ExecutionPlan>> {
tokio::task::yield_now().await;
+ if options.object_type == CatalogObjectType::Catalog {
+ if self.no_catalogs {
+ return datafusion_common::not_impl_err!("test catalog cannot create catalogs");
+ }
+ if self.catalog_error {
+ return exec_err!("catalog creation failed");
+ }
+ }
let identifier = owned(identifier);
self.creates.lock().unwrap().push((
identifier.clone(),
@@ -517,6 +544,74 @@ mod tests {
(ctx, catalog)
}
+ #[tokio::test]
+ async fn create_database_skips_catalog_lists_that_cannot_create_one() -> Result<()> {
+ let (ctx, older) = test_context();
+ let newer = Arc::new(TestCatalog {
+ no_catalogs: true,
+ ..Default::default()
+ });
+ ctx.register_catalog_list(newer.clone());
+
+ ctx.sql("CREATE DATABASE new_catalog")
+ .await?
+ .collect()
+ .await?;
+ assert_eq!(
+ older.objects.lock().unwrap().get(&owned(&["new_catalog"])),
+ Some(&CatalogObjectType::Catalog)
+ );
+ assert!(newer.objects.lock().unwrap().is_empty());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn create_database_falls_back_to_builtin_when_no_catalog_list_can() -> Result<()> {
+ let ctx = SedonaContext::new();
+ let catalog = Arc::new(TestCatalog {
+ no_catalogs: true,
+ ..Default::default()
+ });
+ ctx.register_catalog_list(catalog.clone());
+
+ ctx.sql("CREATE DATABASE new_catalog")
+ .await?
+ .collect()
+ .await?;
+ assert!(ctx.ctx.catalog("new_catalog").is_some());
+ assert!(catalog.objects.lock().unwrap().is_empty());
+ ctx.sql("CREATE SCHEMA new_catalog.s")
+ .await?
+ .collect()
+ .await?;
+ assert!(ctx
+ .ctx
+ .catalog("new_catalog")
+ .unwrap()
+ .schema("s")
+ .is_some());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn create_database_does_not_fall_back_on_other_errors() -> Result<()> {
+ let (ctx, older) = test_context();
+ ctx.register_catalog_list(Arc::new(TestCatalog {
+ catalog_error: true,
+ ..Default::default()
+ }));
+
+ let err = ctx.sql("CREATE DATABASE new_catalog").await.unwrap_err();
+ assert!(err.to_string().contains("catalog creation failed"), "{err}");
+ assert!(!older
+ .objects
+ .lock()
+ .unwrap()
+ .contains_key(&owned(&["new_catalog"])));
+ assert!(ctx.ctx.catalog("new_catalog").is_none());
+ Ok(())
+ }
+
#[tokio::test]
async fn foreign_create_routes_all_object_kinds_and_executes_eagerly() -> Result<()> {
let (ctx, catalog) = test_context();