You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/10/09 23:02:47 UTC

(sedona-db) branch main updated: fix(rust/sedona): let CREATE DATABASE fall back past catalog lists that cannot create one (#1410)

This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git


The following commit(s) were added to refs/heads/main by this push:
     new ae4a8e378 fix(rust/sedona): let CREATE DATABASE fall back past catalog lists that cannot create one (#1410)
ae4a8e378 is described below

commit ae4a8e3784500a6d3a670759e547ba9fdd410145
Author: Animesh Kumar <61...@users.noreply.github.com>
AuthorDate: Sat Oct 10 04:32:41 2026 +0530

    fix(rust/sedona): let CREATE DATABASE fall back past catalog lists that cannot create one (#1410)
---
 c/sedona-extension/src/catalog.rs         |  60 ++++++++++++++++-
 c/sedona-extension/src/sedona_extension.h |   4 +-
 rust/sedona-catalog/src/lib.rs            |   6 ++
 rust/sedona/src/catalog.rs                |  20 ++++--
 rust/sedona/src/exec.rs                   | 107 ++++++++++++++++++++++++++++--
 5 files changed, 184 insertions(+), 13 deletions(-)

diff --git a/c/sedona-extension/src/catalog.rs b/c/sedona-extension/src/catalog.rs
index bb8ca718c..edbe3eb48 100644
--- a/c/sedona-extension/src/catalog.rs
+++ b/c/sedona-extension/src/catalog.rs
@@ -274,7 +274,13 @@ unsafe fn write_plan(
         }
         Err(error) => {
             set_ffi_error!(err, "{}", error);
-            libc::EINVAL
+            // ENOTSUP tells the caller that this catalog list doesn't support the
+            // operation, so CREATE DATABASE can try the next one.
+            if matches!(error.find_root(), DataFusionError::NotImplemented(_)) {
+                libc::ENOTSUP
+            } else {
+                libc::EINVAL
+            }
         }
     }
 }
@@ -465,6 +471,8 @@ impl SedonaCatalogList for ImportedCatalogProviderList {
 fn check_error(code: c_int, err: &SedonaCError) -> Result<()> {
     if code == ERRNO_OK {
         Ok(())
+    } else if code == libc::ENOTSUP {
+        datafusion_common::not_impl_err!("Catalog callback is not supported ({code}): {err}")
     } else {
         datafusion_common::exec_err!("Catalog callback failed ({code}): {err}")
     }
@@ -565,6 +573,7 @@ mod tests {
         calls: Mutex<Vec<String>>,
         mutations: Arc<AtomicUsize>,
         fail: bool,
+        unsupported: bool,
     }
 
     #[async_trait]
@@ -616,6 +625,9 @@ mod tests {
             if self.fail {
                 return datafusion_common::exec_err!("create failed");
             }
+            if self.unsupported {
+                return datafusion_common::not_impl_err!("create not supported");
+            }
             self.calls.lock().unwrap().push(format!(
                 "create:{identifier:?}:{options:?}:{}",
                 input.is_some()
@@ -777,6 +789,52 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn unsupported_create_returns_enotsup() {
+        let raw = raw(Arc::new(TestCatalog {
+            unsupported: true,
+            ..Default::default()
+        }));
+        let args = CString::new(r#"{"identifier":["catalog"],"options":{}}"#).unwrap();
+        let mut out = SedonaCExecutionPlan::default();
+        let mut err = SedonaCError::default();
+        let code = unsafe {
+            raw.create_object.unwrap()(&raw, args.as_ptr(), null_mut(), &mut out, &mut err)
+        };
+        assert_eq!(code, libc::ENOTSUP);
+        assert!(out.release.is_none());
+        assert!(err.to_string().contains("create not supported"));
+    }
+
+    #[tokio::test]
+    async fn imported_unsupported_create_is_not_implemented() -> Result<()> {
+        // A NotImplemented error survives the round trip through the C interface, so an
+        // imported catalog list can decline CREATE DATABASE and let the next one handle it.
+        let catalog = Arc::new(TestCatalog {
+            unsupported: true,
+            ..Default::default()
+        });
+        let imported = ImportedCatalogProviderList::try_new(raw(catalog), runtime())?;
+        let error = imported
+            .create_object(
+                &SessionContext::new().state(),
+                &["catalog"],
+                &CreateObjectOptions::default(),
+                None,
+            )
+            .await
+            .unwrap_err();
+        assert!(
+            matches!(error.find_root(), DataFusionError::NotImplemented(_)),
+            "{error}"
+        );
+        assert!(
+            error.to_string().contains("create not supported"),
+            "{error}"
+        );
+        Ok(())
+    }
+
     #[test]
     fn callback_transfers_input_ownership_and_rejects_invalid_json() {
         let raw = raw(Arc::new(TestCatalog::default()));
diff --git a/c/sedona-extension/src/sedona_extension.h b/c/sedona-extension/src/sedona_extension.h
index 69b521275..5f200d66b 100644
--- a/c/sedona-extension/src/sedona_extension.h
+++ b/c/sedona-extension/src/sedona_extension.h
@@ -526,7 +526,9 @@ struct SedonaCCatalogProviderList {
   /// "definition": null|string}}. Missing options fields use their Rust defaults.
   /// A NULL input means no plan; otherwise move it and clear its release callback
   /// when taking ownership. out is required on success and defers the mutation
-  /// (including existence/conflict checks) until execution.
+  /// (including existence/conflict checks) until execution. Return ENOTSUP if this
+  /// catalog list cannot create the object: for a new top-level catalog (CREATE
+  /// DATABASE), SedonaDB then offers it to the next registered catalog list.
   int (*create_object)(const struct SedonaCCatalogProviderList* self, const char* args,
                        struct SedonaCExecutionPlan* input,
                        struct SedonaCExecutionPlan* out, struct SedonaCError* err);
diff --git a/rust/sedona-catalog/src/lib.rs b/rust/sedona-catalog/src/lib.rs
index 8083b5d29..9fcedddbe 100644
--- a/rust/sedona-catalog/src/lib.rs
+++ b/rust/sedona-catalog/src/lib.rs
@@ -134,6 +134,12 @@ pub trait SedonaCatalogList: Debug + Send + Sync {
     /// external tables, namespaces, and indexes. `session` belongs to
     /// this call and must not be retained by the catalog. The returned plan applies
     /// conflict behavior and validates the parent namespace when executed.
+    ///
+    /// A new top-level catalog (`CREATE DATABASE`) is offered to the newest
+    /// registration first. An implementation that cannot create one should return a
+    /// `NotImplemented` error, so that the next older registration, and then the
+    /// built-in catalog list, can create it instead. Any other error is returned to
+    /// the caller.
     async fn create_object(
         &self,
         session: &dyn Session,
diff --git a/rust/sedona/src/catalog.rs b/rust/sedona/src/catalog.rs
index 6d6e8e8bb..2d0b342b6 100644
--- a/rust/sedona/src/catalog.rs
+++ b/rust/sedona/src/catalog.rs
@@ -37,7 +37,9 @@ use sedona_catalog::{CatalogObjectType, SedonaCatalogList};
 /// register object stores dynamically for file locations.
 #[derive(Debug)]
 pub struct SedonaCatalogRegistry {
-    foreign: RwLock<Vec<Arc<dyn SedonaCatalogList>>>,
+    /// Registered foreign catalog lists, oldest first. Registration replaces the
+    /// slice so readers can take a snapshot without copying it.
+    foreign: RwLock<Arc<[Arc<dyn SedonaCatalogList>]>>,
     fallback: Arc<dyn CatalogProviderList>,
     state: Weak<RwLock<SessionState>>,
     dynamic_object_store: bool,
@@ -51,7 +53,7 @@ impl SedonaCatalogRegistry {
         dynamic_object_store: bool,
     ) -> Self {
         Self {
-            foreign: RwLock::new(Vec::new()),
+            foreign: RwLock::new(Vec::new().into()),
             fallback,
             state,
             dynamic_object_store,
@@ -60,12 +62,13 @@ impl SedonaCatalogRegistry {
 
     /// Add a foreign catalog list. Later registrations take precedence.
     pub fn register_foreign(&self, catalogs: Arc<dyn SedonaCatalogList>) {
-        self.foreign.write().push(catalogs);
+        let mut foreign = self.foreign.write();
+        *foreign = foreign.iter().cloned().chain([catalogs]).collect();
     }
 
     /// Resolve catalog ownership asynchronously, newest registration first.
     pub async fn foreign_catalog(&self, name: &str) -> Result<Option<Arc<dyn SedonaCatalogList>>> {
-        let foreign = self.foreign.read().clone();
+        let foreign = self.foreign_lists();
         for catalogs in foreign.iter().rev() {
             if catalogs
                 .list_identifiers(&[name], Some(0))
@@ -81,10 +84,17 @@ impl SedonaCatalogRegistry {
         Ok(None)
     }
 
-    /// Destination for new top-level catalogs.
+    /// The most recently registered foreign catalog list.
     pub fn latest_foreign(&self) -> Option<Arc<dyn SedonaCatalogList>> {
         self.foreign.read().last().cloned()
     }
+
+    /// The registered foreign catalog lists in registration order, oldest
+    /// first. Later registrations take precedence, so callers that pick an
+    /// owner iterate this slice in reverse.
+    pub fn foreign_lists(&self) -> Arc<[Arc<dyn SedonaCatalogList>]> {
+        self.foreign.read().clone()
+    }
 }
 
 impl CatalogProviderList for SedonaCatalogRegistry {
diff --git a/rust/sedona/src/exec.rs b/rust/sedona/src/exec.rs
index b9aad826f..d0488c514 100644
--- a/rust/sedona/src/exec.rs
+++ b/rust/sedona/src/exec.rs
@@ -16,7 +16,7 @@
 // under the License.
 use crate::{context::SedonaContext, object_storage::register_object_store_and_config_extensions};
 use datafusion::{error::Result, sql::parser::Statement};
-use datafusion_common::{exec_err, SchemaReference, TableReference};
+use datafusion_common::{exec_err, DataFusionError, SchemaReference, TableReference};
 use datafusion_expr::{DdlStatement, LogicalPlan};
 use datafusion_physical_plan::ExecutionPlan;
 use sedona_catalog::{CatalogObjectType, CreateMode, CreateObjectOptions, DropObjectOptions};
@@ -195,15 +195,29 @@ pub(crate) async fn resolve_sedona_catalog_ddl(
     }
     let registry = ctx.catalog_registry();
     let owner = registry.foreign_catalog(&identifier[0]).await?;
-    let owner = if matches!(ddl, DdlStatement::CreateCatalog(_)) && owner.is_none() {
+    if matches!(ddl, DdlStatement::CreateCatalog(_)) && owner.is_none() {
         // Existing built-in catalogs remain owned by DataFusion.
         if state.catalog_list().catalog(&identifier[0]).is_some() {
             return Ok(None);
         }
-        registry.latest_foreign()
-    } else {
-        owner
-    };
+        // A new catalog goes to the newest registration that supports creating
+        // one. A catalog list that does not returns a NotImplemented error, and the
+        // next older registration is tried, then DataFusion's built-in list.
+        reject_unsupported_create_metadata(ddl)?;
+        let identifier: Vec<&str> = identifier.iter().map(String::as_str).collect();
+        for candidate in registry.foreign_lists().iter().rev() {
+            match candidate
+                .create_object(&state, &identifier, &create, None)
+                .await
+            {
+                Err(err) if matches!(err.find_root(), DataFusionError::NotImplemented(_)) => {
+                    continue
+                }
+                result => return result.map(Some),
+            }
+        }
+        return Ok(None);
+    }
     let Some(owner) = owner else {
         return Ok(None);
     };
@@ -383,6 +397,11 @@ mod tests {
         drops: Mutex<Vec<(Vec<String>, DropObjectOptions)>>,
         ddl_output: Option<Arc<dyn ExecutionPlan>>,
         fail: bool,
+        /// Refuse to create catalogs with NotImplemented, as a catalog list
+        /// without top-level catalog creation would.
+        no_catalogs: bool,
+        /// Fail to create catalogs with an ordinary error.
+        catalog_error: bool,
     }
 
     fn owned(parts: &[&str]) -> Vec<String> {
@@ -443,6 +462,14 @@ mod tests {
             input: Option<Arc<dyn ExecutionPlan>>,
         ) -> Result<Arc<dyn ExecutionPlan>> {
             tokio::task::yield_now().await;
+            if options.object_type == CatalogObjectType::Catalog {
+                if self.no_catalogs {
+                    return datafusion_common::not_impl_err!("test catalog cannot create catalogs");
+                }
+                if self.catalog_error {
+                    return exec_err!("catalog creation failed");
+                }
+            }
             let identifier = owned(identifier);
             self.creates.lock().unwrap().push((
                 identifier.clone(),
@@ -517,6 +544,74 @@ mod tests {
         (ctx, catalog)
     }
 
+    #[tokio::test]
+    async fn create_database_skips_catalog_lists_that_cannot_create_one() -> Result<()> {
+        let (ctx, older) = test_context();
+        let newer = Arc::new(TestCatalog {
+            no_catalogs: true,
+            ..Default::default()
+        });
+        ctx.register_catalog_list(newer.clone());
+
+        ctx.sql("CREATE DATABASE new_catalog")
+            .await?
+            .collect()
+            .await?;
+        assert_eq!(
+            older.objects.lock().unwrap().get(&owned(&["new_catalog"])),
+            Some(&CatalogObjectType::Catalog)
+        );
+        assert!(newer.objects.lock().unwrap().is_empty());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn create_database_falls_back_to_builtin_when_no_catalog_list_can() -> Result<()> {
+        let ctx = SedonaContext::new();
+        let catalog = Arc::new(TestCatalog {
+            no_catalogs: true,
+            ..Default::default()
+        });
+        ctx.register_catalog_list(catalog.clone());
+
+        ctx.sql("CREATE DATABASE new_catalog")
+            .await?
+            .collect()
+            .await?;
+        assert!(ctx.ctx.catalog("new_catalog").is_some());
+        assert!(catalog.objects.lock().unwrap().is_empty());
+        ctx.sql("CREATE SCHEMA new_catalog.s")
+            .await?
+            .collect()
+            .await?;
+        assert!(ctx
+            .ctx
+            .catalog("new_catalog")
+            .unwrap()
+            .schema("s")
+            .is_some());
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn create_database_does_not_fall_back_on_other_errors() -> Result<()> {
+        let (ctx, older) = test_context();
+        ctx.register_catalog_list(Arc::new(TestCatalog {
+            catalog_error: true,
+            ..Default::default()
+        }));
+
+        let err = ctx.sql("CREATE DATABASE new_catalog").await.unwrap_err();
+        assert!(err.to_string().contains("catalog creation failed"), "{err}");
+        assert!(!older
+            .objects
+            .lock()
+            .unwrap()
+            .contains_key(&owned(&["new_catalog"])));
+        assert!(ctx.ctx.catalog("new_catalog").is_none());
+        Ok(())
+    }
+
     #[tokio::test]
     async fn foreign_create_routes_all_object_kinds_and_executes_eagerly() -> Result<()> {
         let (ctx, catalog) = test_context();