You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ti...@apache.org on 2025/07/02 12:08:58 UTC

(datafusion-python) branch main updated: feat: python based catalog and schema provider (#1156)

This is an automated email from the ASF dual-hosted git repository.

timsaucer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git


The following commit(s) were added to refs/heads/main by this push:
     new 9362f531 feat: python based catalog and schema provider (#1156)
9362f531 is described below

commit 9362f53150e5423581757ed56883b3ca2c95b8a2
Author: Tim Saucer <ti...@gmail.com>
AuthorDate: Wed Jul 2 08:08:53 2025 -0400

    feat: python based catalog and schema provider (#1156)
    
    * Exposing FFI to python
    
    * Exposing FFI to python
    
    * Workin progress on python catalog
    
    * Flushing out schema and catalog providers
    
    * Adding implementation of python based catalog and schema providers
    
    * Small updates after rebase
    
    * Add default in memory options for adding schema and catalogs
    
    * Add support for creating in memory catalog and schema
    
    * Update from database to schema in unit tests
    
    * xfailed label no longer applies to these unit tests
    
    * Defining abstract methods for catalog and schema providers
    
    * Working through issues between custom catalog and build in schema
    
    * Check types on schema provider to return
    
    * Add docstring
    
    * Add documentation about how to use catalog and schema providers
    
    * Re-add module to all after rebase
    
    * Minor bugfix
    
    * Clippy updates from the new rust version
    
    ---------
    
    Co-authored-by: renato2099 <re...@gmail.com>
---
 Cargo.lock                                         |  19 +
 Cargo.toml                                         |   2 +
 docs/source/user-guide/data-sources.rst            |  56 +++
 examples/datafusion-ffi-example/Cargo.lock         |   1 +
 examples/datafusion-ffi-example/Cargo.toml         |   1 +
 .../python/tests/_test_catalog_provider.py         |  60 +++
 .../datafusion-ffi-example/src/catalog_provider.rs | 179 ++++++++
 examples/datafusion-ffi-example/src/lib.rs         |   3 +
 python/datafusion/__init__.py                      |   1 +
 python/datafusion/catalog.py                       | 195 +++++++-
 python/datafusion/context.py                       |  24 +-
 python/datafusion/dataframe.py                     |   9 +-
 python/tests/test_catalog.py                       | 173 +++++++-
 python/tests/test_context.py                       |  40 +-
 python/tests/test_sql.py                           |  30 +-
 python/tests/test_substrait.py                     |   4 +-
 src/catalog.rs                                     | 490 +++++++++++++++++++--
 src/common/data_type.rs                            | 120 +++--
 src/context.rs                                     |  61 ++-
 src/expr.rs                                        |  15 +-
 src/expr/aggregate.rs                              |   2 +-
 src/expr/aggregate_expr.rs                         |   2 +-
 src/expr/alias.rs                                  |   2 +-
 src/expr/analyze.rs                                |   2 +-
 src/expr/between.rs                                |   2 +-
 src/expr/column.rs                                 |   2 +-
 src/expr/copy_to.rs                                |   4 +-
 src/expr/create_catalog.rs                         |   2 +-
 src/expr/create_catalog_schema.rs                  |   2 +-
 src/expr/create_external_table.rs                  |   2 +-
 src/expr/create_function.rs                        |   2 +-
 src/expr/create_index.rs                           |   2 +-
 src/expr/create_memory_table.rs                    |   2 +-
 src/expr/create_view.rs                            |   2 +-
 src/expr/describe_table.rs                         |   2 +-
 src/expr/distinct.rs                               |   5 +-
 src/expr/drop_catalog_schema.rs                    |   2 +-
 src/expr/drop_function.rs                          |   2 +-
 src/expr/drop_table.rs                             |   2 +-
 src/expr/drop_view.rs                              |   2 +-
 src/expr/empty_relation.rs                         |   2 +-
 src/expr/filter.rs                                 |   2 +-
 src/expr/join.rs                                   |   2 +-
 src/expr/like.rs                                   |   6 +-
 src/expr/limit.rs                                  |   2 +-
 src/expr/projection.rs                             |   2 +-
 src/expr/recursive_query.rs                        |   2 +-
 src/expr/repartition.rs                            |   2 +-
 src/expr/sort.rs                                   |   2 +-
 src/expr/sort_expr.rs                              |   2 +-
 src/expr/subquery.rs                               |   2 +-
 src/expr/subquery_alias.rs                         |   2 +-
 src/expr/table_scan.rs                             |   2 +-
 src/expr/union.rs                                  |   2 +-
 src/expr/unnest.rs                                 |   2 +-
 src/expr/unnest_expr.rs                            |   2 +-
 src/expr/window.rs                                 |  11 +-
 src/functions.rs                                   |   2 +-
 src/lib.rs                                         |  10 +-
 src/physical_plan.rs                               |   3 +-
 src/sql/logical.rs                                 |   3 +-
 src/utils.rs                                       |   5 +-
 62 files changed, 1340 insertions(+), 258 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 112167cb..a3e9336c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -165,6 +165,12 @@ dependencies = [
  "zstd",
 ]
 
+[[package]]
+name = "arc-swap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+
 [[package]]
 name = "arrayref"
 version = "0.3.9"
@@ -1503,6 +1509,7 @@ dependencies = [
  "datafusion-proto",
  "datafusion-substrait",
  "futures",
+ "log",
  "mimalloc",
  "object_store",
  "prost",
@@ -1510,6 +1517,7 @@ dependencies = [
  "pyo3",
  "pyo3-async-runtimes",
  "pyo3-build-config",
+ "pyo3-log",
  "tokio",
  "url",
  "uuid",
@@ -2953,6 +2961,17 @@ dependencies = [
  "pyo3-build-config",
 ]
 
+[[package]]
+name = "pyo3-log"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45192e5e4a4d2505587e27806c7b710c231c40c56f3bfc19535d0bb25df52264"
+dependencies = [
+ "arc-swap",
+ "log",
+ "pyo3",
+]
+
 [[package]]
 name = "pyo3-macros"
 version = "0.24.2"
diff --git a/Cargo.toml b/Cargo.toml
index 4135e64e..1f7895a5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,6 +37,7 @@ substrait = ["dep:datafusion-substrait"]
 tokio = { version = "1.45", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 pyo3 = { version = "0.24", features = ["extension-module", "abi3", "abi3-py39"] }
 pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"]}
+pyo3-log = "0.12.4"
 arrow = { version = "55.1.0", features = ["pyarrow"] }
 datafusion = { version = "48.0.0", features = ["avro", "unicode_expressions"] }
 datafusion-substrait = { version = "48.0.0", optional = true }
@@ -49,6 +50,7 @@ async-trait = "0.1.88"
 futures = "0.3"
 object_store = { version = "0.12.1", features = ["aws", "gcp", "azure", "http"] }
 url = "2"
+log = "0.4.27"
 
 [build-dependencies]
 prost-types = "0.13.1" # keep in line with `datafusion-substrait`
diff --git a/docs/source/user-guide/data-sources.rst b/docs/source/user-guide/data-sources.rst
index ba5967c9..9c95d58e 100644
--- a/docs/source/user-guide/data-sources.rst
+++ b/docs/source/user-guide/data-sources.rst
@@ -185,3 +185,59 @@ the interface as describe in the :ref:`Custom Table Provider <io_custom_table_pr
 section. This is an advanced topic, but a
 `user example <https://github.com/apache/datafusion-python/tree/main/examples/ffi-table-provider>`_
 is provided in the DataFusion repository.
+
+Catalog
+=======
+
+A common technique for organizing tables is using a three level hierarchical approach. DataFusion
+supports this form of organizing using the :py:class:`~datafusion.catalog.Catalog`,
+:py:class:`~datafusion.catalog.Schema`, and :py:class:`~datafusion.catalog.Table`. By default,
+a :py:class:`~datafusion.context.SessionContext` comes with a single Catalog and a single Schema
+with the names ``datafusion`` and ``default``, respectively.
+
+The default implementation uses an in-memory approach to the catalog and schema. We have support
+for adding additional in-memory catalogs and schemas. This can be done like in the following
+example:
+
+.. code-block:: python
+
+    from datafusion.catalog import Catalog, Schema
+
+    my_catalog = Catalog.memory_catalog()
+    my_schema = Schema.memory_schema()
+
+    my_catalog.register_schema("my_schema_name", my_schema)
+
+    ctx.register_catalog("my_catalog_name", my_catalog)
+
+You could then register tables in ``my_schema`` and access them either through the DataFrame
+API or via sql commands such as ``"SELECT * from my_catalog_name.my_schema_name.my_table"``.
+
+User Defined Catalog and Schema
+-------------------------------
+
+If the in-memory catalogs are insufficient for your uses, there are two approaches you can take
+to implementing a custom catalog and/or schema. In the below discussion, we describe how to
+implement these for a Catalog, but the approach to implementing for a Schema is nearly
+identical.
+
+DataFusion supports Catalogs written in either Rust or Python. If you write a Catalog in Rust,
+you will need to export it as a Python library via PyO3. There is a complete example of a
+catalog implemented this way in the
+`examples folder <https://github.com/apache/datafusion-python/tree/main/examples/>`_
+of our repository. Writing catalog providers in Rust provides typically can lead to significant
+performance improvements over the Python based approach.
+
+To implement a Catalog in Python, you will need to inherit from the abstract base class
+:py:class:`~datafusion.catalog.CatalogProvider`. There are examples in the
+`unit tests <https://github.com/apache/datafusion-python/tree/main/python/tests>`_ of
+implementing a basic Catalog in Python where we simply keep a dictionary of the
+registered Schemas.
+
+One important note for developers is that when we have a Catalog defined in Python, we have
+two different ways of accessing this Catalog. First, we register the catalog with a Rust
+wrapper. This allows for any rust based code to call the Python functions as necessary.
+Second, if the user access the Catalog via the Python API, we identify this and return back
+the original Python object that implements the Catalog. This is an important distinction
+for developers because we do *not* return a Python wrapper around the Rust wrapper of the
+original Python object.
diff --git a/examples/datafusion-ffi-example/Cargo.lock b/examples/datafusion-ffi-example/Cargo.lock
index 075ebd5a..e5a1ca8d 100644
--- a/examples/datafusion-ffi-example/Cargo.lock
+++ b/examples/datafusion-ffi-example/Cargo.lock
@@ -1448,6 +1448,7 @@ dependencies = [
  "arrow",
  "arrow-array",
  "arrow-schema",
+ "async-trait",
  "datafusion",
  "datafusion-ffi",
  "pyo3",
diff --git a/examples/datafusion-ffi-example/Cargo.toml b/examples/datafusion-ffi-example/Cargo.toml
index 0e17567b..31916355 100644
--- a/examples/datafusion-ffi-example/Cargo.toml
+++ b/examples/datafusion-ffi-example/Cargo.toml
@@ -27,6 +27,7 @@ pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"]
 arrow = { version = "55.0.0" }
 arrow-array = { version = "55.0.0" }
 arrow-schema = { version = "55.0.0" }
+async-trait = "0.1.88"
 
 [build-dependencies]
 pyo3-build-config = "0.23"
diff --git a/examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py b/examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py
new file mode 100644
index 00000000..72aadf64
--- /dev/null
+++ b/examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import pyarrow as pa
+from datafusion import SessionContext
+from datafusion_ffi_example import MyCatalogProvider
+
+
+def test_catalog_provider():
+    ctx = SessionContext()
+
+    my_catalog_name = "my_catalog"
+    expected_schema_name = "my_schema"
+    expected_table_name = "my_table"
+    expected_table_columns = ["units", "price"]
+
+    catalog_provider = MyCatalogProvider()
+    ctx.register_catalog_provider(my_catalog_name, catalog_provider)
+    my_catalog = ctx.catalog(my_catalog_name)
+
+    my_catalog_schemas = my_catalog.names()
+    assert expected_schema_name in my_catalog_schemas
+    my_database = my_catalog.database(expected_schema_name)
+    assert expected_table_name in my_database.names()
+    my_table = my_database.table(expected_table_name)
+    assert expected_table_columns == my_table.schema.names
+
+    result = ctx.table(
+        f"{my_catalog_name}.{expected_schema_name}.{expected_table_name}"
+    ).collect()
+    assert len(result) == 2
+
+    col0_result = [r.column(0) for r in result]
+    col1_result = [r.column(1) for r in result]
+    expected_col0 = [
+        pa.array([10, 20, 30], type=pa.int32()),
+        pa.array([5, 7], type=pa.int32()),
+    ]
+    expected_col1 = [
+        pa.array([1, 2, 5], type=pa.float64()),
+        pa.array([1.5, 2.5], type=pa.float64()),
+    ]
+    assert col0_result == expected_col0
+    assert col1_result == expected_col1
diff --git a/examples/datafusion-ffi-example/src/catalog_provider.rs b/examples/datafusion-ffi-example/src/catalog_provider.rs
new file mode 100644
index 00000000..54e61cf3
--- /dev/null
+++ b/examples/datafusion-ffi-example/src/catalog_provider.rs
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use pyo3::{pyclass, pymethods, Bound, PyResult, Python};
+use std::{any::Any, fmt::Debug, sync::Arc};
+
+use arrow::datatypes::Schema;
+use async_trait::async_trait;
+use datafusion::{
+    catalog::{
+        CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider, TableProvider,
+    },
+    common::exec_err,
+    datasource::MemTable,
+    error::{DataFusionError, Result},
+};
+use datafusion_ffi::catalog_provider::FFI_CatalogProvider;
+use pyo3::types::PyCapsule;
+
+pub fn my_table() -> Arc<dyn TableProvider + 'static> {
+    use arrow::datatypes::{DataType, Field};
+    use datafusion::common::record_batch;
+
+    let schema = Arc::new(Schema::new(vec![
+        Field::new("units", DataType::Int32, true),
+        Field::new("price", DataType::Float64, true),
+    ]));
+
+    let partitions = vec![
+        record_batch!(
+            ("units", Int32, vec![10, 20, 30]),
+            ("price", Float64, vec![1.0, 2.0, 5.0])
+        )
+        .unwrap(),
+        record_batch!(
+            ("units", Int32, vec![5, 7]),
+            ("price", Float64, vec![1.5, 2.5])
+        )
+        .unwrap(),
+    ];
+
+    Arc::new(MemTable::try_new(schema, vec![partitions]).unwrap())
+}
+
+#[derive(Debug)]
+pub struct FixedSchemaProvider {
+    inner: MemorySchemaProvider,
+}
+
+impl Default for FixedSchemaProvider {
+    fn default() -> Self {
+        let inner = MemorySchemaProvider::new();
+
+        let table = my_table();
+
+        let _ = inner.register_table("my_table".to_string(), table).unwrap();
+
+        Self { inner }
+    }
+}
+
+#[async_trait]
+impl SchemaProvider for FixedSchemaProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Vec<String> {
+        self.inner.table_names()
+    }
+
+    async fn table(&self, name: &str) -> Result<Option<Arc<dyn TableProvider>>, DataFusionError> {
+        self.inner.table(name).await
+    }
+
+    fn register_table(
+        &self,
+        name: String,
+        table: Arc<dyn TableProvider>,
+    ) -> Result<Option<Arc<dyn TableProvider>>> {
+        self.inner.register_table(name, table)
+    }
+
+    fn deregister_table(&self, name: &str) -> Result<Option<Arc<dyn TableProvider>>> {
+        self.inner.deregister_table(name)
+    }
+
+    fn table_exist(&self, name: &str) -> bool {
+        self.inner.table_exist(name)
+    }
+}
+
+/// This catalog provider is intended only for unit tests. It prepopulates with one
+/// schema and only allows for schemas named after four types of fruit.
+#[pyclass(
+    name = "MyCatalogProvider",
+    module = "datafusion_ffi_example",
+    subclass
+)]
+#[derive(Debug)]
+pub(crate) struct MyCatalogProvider {
+    inner: MemoryCatalogProvider,
+}
+
+impl Default for MyCatalogProvider {
+    fn default() -> Self {
+        let inner = MemoryCatalogProvider::new();
+
+        let schema_name: &str = "my_schema";
+        let _ = inner.register_schema(schema_name, Arc::new(FixedSchemaProvider::default()));
+
+        Self { inner }
+    }
+}
+
+impl CatalogProvider for MyCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Vec<String> {
+        self.inner.schema_names()
+    }
+
+    fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {
+        self.inner.schema(name)
+    }
+
+    fn register_schema(
+        &self,
+        name: &str,
+        schema: Arc<dyn SchemaProvider>,
+    ) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        self.inner.register_schema(name, schema)
+    }
+
+    fn deregister_schema(
+        &self,
+        name: &str,
+        cascade: bool,
+    ) -> Result<Option<Arc<dyn SchemaProvider>>> {
+        self.inner.deregister_schema(name, cascade)
+    }
+}
+
+#[pymethods]
+impl MyCatalogProvider {
+    #[new]
+    pub fn new() -> Self {
+        Self {
+            inner: Default::default(),
+        }
+    }
+
+    pub fn __datafusion_catalog_provider__<'py>(
+        &self,
+        py: Python<'py>,
+    ) -> PyResult<Bound<'py, PyCapsule>> {
+        let name = cr"datafusion_catalog_provider".into();
+        let catalog_provider =
+            FFI_CatalogProvider::new(Arc::new(MyCatalogProvider::default()), None);
+
+        PyCapsule::new(py, catalog_provider, Some(name))
+    }
+}
diff --git a/examples/datafusion-ffi-example/src/lib.rs b/examples/datafusion-ffi-example/src/lib.rs
index ae08c3b6..3a4cf224 100644
--- a/examples/datafusion-ffi-example/src/lib.rs
+++ b/examples/datafusion-ffi-example/src/lib.rs
@@ -15,10 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use crate::catalog_provider::MyCatalogProvider;
 use crate::table_function::MyTableFunction;
 use crate::table_provider::MyTableProvider;
 use pyo3::prelude::*;
 
+pub(crate) mod catalog_provider;
 pub(crate) mod table_function;
 pub(crate) mod table_provider;
 
@@ -26,5 +28,6 @@ pub(crate) mod table_provider;
 fn datafusion_ffi_example(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<MyTableProvider>()?;
     m.add_class::<MyTableFunction>()?;
+    m.add_class::<MyCatalogProvider>()?;
     Ok(())
 }
diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py
index fd7f4fc0..e9d2dba7 100644
--- a/python/datafusion/__init__.py
+++ b/python/datafusion/__init__.py
@@ -92,6 +92,7 @@ __all__ = [
     "TableFunction",
     "WindowFrame",
     "WindowUDF",
+    "catalog",
     "col",
     "column",
     "common",
diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py
index 67ab3ead..536b3a79 100644
--- a/python/datafusion/catalog.py
+++ b/python/datafusion/catalog.py
@@ -19,18 +19,33 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Protocol
 
 import datafusion._internal as df_internal
 
 if TYPE_CHECKING:
     import pyarrow as pa
 
+try:
+    from warnings import deprecated  # Python 3.13+
+except ImportError:
+    from typing_extensions import deprecated  # Python 3.12
+
+
+__all__ = [
+    "Catalog",
+    "CatalogProvider",
+    "Schema",
+    "SchemaProvider",
+    "Table",
+]
+
 
 class Catalog:
     """DataFusion data catalog."""
 
-    def __init__(self, catalog: df_internal.Catalog) -> None:
+    def __init__(self, catalog: df_internal.catalog.RawCatalog) -> None:
         """This constructor is not typically called by the end user."""
         self.catalog = catalog
 
@@ -38,39 +53,95 @@ class Catalog:
         """Print a string representation of the catalog."""
         return self.catalog.__repr__()
 
-    def names(self) -> list[str]:
-        """Returns the list of databases in this catalog."""
-        return self.catalog.names()
+    def names(self) -> set[str]:
+        """This is an alias for `schema_names`."""
+        return self.schema_names()
+
+    def schema_names(self) -> set[str]:
+        """Returns the list of schemas in this catalog."""
+        return self.catalog.schema_names()
+
+    @staticmethod
+    def memory_catalog() -> Catalog:
+        """Create an in-memory catalog provider."""
+        catalog = df_internal.catalog.RawCatalog.memory_catalog()
+        return Catalog(catalog)
 
-    def database(self, name: str = "public") -> Database:
+    def schema(self, name: str = "public") -> Schema:
         """Returns the database with the given ``name`` from this catalog."""
-        return Database(self.catalog.database(name))
+        schema = self.catalog.schema(name)
+
+        return (
+            Schema(schema)
+            if isinstance(schema, df_internal.catalog.RawSchema)
+            else schema
+        )
+
+    @deprecated("Use `schema` instead.")
+    def database(self, name: str = "public") -> Schema:
+        """Returns the database with the given ``name`` from this catalog."""
+        return self.schema(name)
+
+    def register_schema(self, name, schema) -> Schema | None:
+        """Register a schema with this catalog."""
+        if isinstance(schema, Schema):
+            return self.catalog.register_schema(name, schema._raw_schema)
+        return self.catalog.register_schema(name, schema)
+
+    def deregister_schema(self, name: str, cascade: bool = True) -> Schema | None:
+        """Deregister a schema from this catalog."""
+        return self.catalog.deregister_schema(name, cascade)
 
 
-class Database:
-    """DataFusion Database."""
+class Schema:
+    """DataFusion Schema."""
 
-    def __init__(self, db: df_internal.Database) -> None:
+    def __init__(self, schema: df_internal.catalog.RawSchema) -> None:
         """This constructor is not typically called by the end user."""
-        self.db = db
+        self._raw_schema = schema
 
     def __repr__(self) -> str:
-        """Print a string representation of the database."""
-        return self.db.__repr__()
+        """Print a string representation of the schema."""
+        return self._raw_schema.__repr__()
+
+    @staticmethod
+    def memory_schema() -> Schema:
+        """Create an in-memory schema provider."""
+        schema = df_internal.catalog.RawSchema.memory_schema()
+        return Schema(schema)
 
     def names(self) -> set[str]:
-        """Returns the list of all tables in this database."""
-        return self.db.names()
+        """This is an alias for `table_names`."""
+        return self.table_names()
+
+    def table_names(self) -> set[str]:
+        """Returns the list of all tables in this schema."""
+        return self._raw_schema.table_names
 
     def table(self, name: str) -> Table:
-        """Return the table with the given ``name`` from this database."""
-        return Table(self.db.table(name))
+        """Return the table with the given ``name`` from this schema."""
+        return Table(self._raw_schema.table(name))
+
+    def register_table(self, name, table) -> None:
+        """Register a table provider in this schema."""
+        if isinstance(table, Table):
+            return self._raw_schema.register_table(name, table.table)
+        return self._raw_schema.register_table(name, table)
+
+    def deregister_table(self, name: str) -> None:
+        """Deregister a table provider from this schema."""
+        return self._raw_schema.deregister_table(name)
+
+
+@deprecated("Use `Schema` instead.")
+class Database(Schema):
+    """See `Schema`."""
 
 
 class Table:
     """DataFusion table."""
 
-    def __init__(self, table: df_internal.Table) -> None:
+    def __init__(self, table: df_internal.catalog.RawTable) -> None:
         """This constructor is not typically called by the end user."""
         self.table = table
 
@@ -78,6 +149,11 @@ class Table:
         """Print a string representation of the table."""
         return self.table.__repr__()
 
+    @staticmethod
+    def from_dataset(dataset: pa.dataset.Dataset) -> Table:
+        """Turn a pyarrow Dataset into a Table."""
+        return Table(df_internal.catalog.RawTable.from_dataset(dataset))
+
     @property
     def schema(self) -> pa.Schema:
         """Returns the schema associated with this table."""
@@ -87,3 +163,86 @@ class Table:
     def kind(self) -> str:
         """Returns the kind of table."""
         return self.table.kind
+
+
+class CatalogProvider(ABC):
+    """Abstract class for defining a Python based Catalog Provider."""
+
+    @abstractmethod
+    def schema_names(self) -> set[str]:
+        """Set of the names of all schemas in this catalog."""
+        ...
+
+    @abstractmethod
+    def schema(self, name: str) -> Schema | None:
+        """Retrieve a specific schema from this catalog."""
+        ...
+
+    def register_schema(  # noqa: B027
+        self, name: str, schema: SchemaProviderExportable | SchemaProvider | Schema
+    ) -> None:
+        """Add a schema to this catalog.
+
+        This method is optional. If your catalog provides a fixed list of schemas, you
+        do not need to implement this method.
+        """
+
+    def deregister_schema(self, name: str, cascade: bool) -> None:  # noqa: B027
+        """Remove a schema from this catalog.
+
+        This method is optional. If your catalog provides a fixed list of schemas, you
+        do not need to implement this method.
+
+        Args:
+            name: The name of the schema to remove.
+            cascade: If true, deregister the tables within the schema.
+        """
+
+
+class SchemaProvider(ABC):
+    """Abstract class for defining a Python based Schema Provider."""
+
+    def owner_name(self) -> str | None:
+        """Returns the owner of the schema.
+
+        This is an optional method. The default return is None.
+        """
+        return None
+
+    @abstractmethod
+    def table_names(self) -> set[str]:
+        """Set of the names of all tables in this schema."""
+        ...
+
+    @abstractmethod
+    def table(self, name: str) -> Table | None:
+        """Retrieve a specific table from this schema."""
+        ...
+
+    def register_table(self, name: str, table: Table) -> None:  # noqa: B027
+        """Add a table from this schema.
+
+        This method is optional. If your schema provides a fixed list of tables, you do
+        not need to implement this method.
+        """
+
+    def deregister_table(self, name, cascade: bool) -> None:  # noqa: B027
+        """Remove a table from this schema.
+
+        This method is optional. If your schema provides a fixed list of tables, you do
+        not need to implement this method.
+        """
+
+    @abstractmethod
+    def table_exist(self, name: str) -> bool:
+        """Returns true if the table exists in this schema."""
+        ...
+
+
+class SchemaProviderExportable(Protocol):
+    """Type hint for object that has __datafusion_schema_provider__ PyCapsule.
+
+    https://docs.rs/datafusion/latest/datafusion/catalog/trait.SchemaProvider.html
+    """
+
+    def __datafusion_schema_provider__(self) -> object: ...
diff --git a/python/datafusion/context.py b/python/datafusion/context.py
index 5b99b0d2..bce51d64 100644
--- a/python/datafusion/context.py
+++ b/python/datafusion/context.py
@@ -29,7 +29,7 @@ try:
 except ImportError:
     from typing_extensions import deprecated  # Python 3.12
 
-from datafusion.catalog import Catalog, Table
+from datafusion.catalog import Catalog, CatalogProvider, Table
 from datafusion.dataframe import DataFrame
 from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list
 from datafusion.record_batch import RecordBatchStream
@@ -80,6 +80,15 @@ class TableProviderExportable(Protocol):
     def __datafusion_table_provider__(self) -> object: ...  # noqa: D105
 
 
+class CatalogProviderExportable(Protocol):
+    """Type hint for object that has __datafusion_catalog_provider__ PyCapsule.
+
+    https://docs.rs/datafusion/latest/datafusion/catalog/trait.CatalogProvider.html
+    """
+
+    def __datafusion_catalog_provider__(self) -> object: ...  # noqa: D105
+
+
 class SessionConfig:
     """Session configuration options."""
 
@@ -749,6 +758,19 @@ class SessionContext:
         """Remove a table from the session."""
         self.ctx.deregister_table(name)
 
+    def catalog_names(self) -> set[str]:
+        """Returns the list of catalogs in this context."""
+        return self.ctx.catalog_names()
+
+    def register_catalog_provider(
+        self, name: str, provider: CatalogProviderExportable | CatalogProvider | Catalog
+    ) -> None:
+        """Register a catalog provider."""
+        if isinstance(provider, Catalog):
+            self.ctx.register_catalog_provider(name, provider.catalog)
+        else:
+            self.ctx.register_catalog_provider(name, provider)
+
     def register_table_provider(
         self, name: str, provider: TableProviderExportable
     ) -> None:
diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py
index 991e6875..61cb0943 100644
--- a/python/datafusion/dataframe.py
+++ b/python/datafusion/dataframe.py
@@ -760,19 +760,16 @@ class DataFrame:
         exprs = [expr.expr for expr in on_exprs]
         return DataFrame(self.df.join_on(right.df, exprs, how))
 
-    def explain(self, verbose: bool = False, analyze: bool = False) -> DataFrame:
-        """Return a DataFrame with the explanation of its plan so far.
+    def explain(self, verbose: bool = False, analyze: bool = False) -> None:
+        """Print an explanation of the DataFrame's plan so far.
 
         If ``analyze`` is specified, runs the plan and reports metrics.
 
         Args:
             verbose: If ``True``, more details will be included.
             analyze: If ``Tru`e``, the plan will run and metrics reported.
-
-        Returns:
-            DataFrame with the explanation of its plan.
         """
-        return DataFrame(self.df.explain(verbose, analyze))
+        self.df.explain(verbose, analyze)
 
     def logical_plan(self) -> LogicalPlan:
         """Return the unoptimized ``LogicalPlan``.
diff --git a/python/tests/test_catalog.py b/python/tests/test_catalog.py
index 23b32845..1f9ecbfc 100644
--- a/python/tests/test_catalog.py
+++ b/python/tests/test_catalog.py
@@ -14,9 +14,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from __future__ import annotations
 
+import datafusion as dfn
 import pyarrow as pa
+import pyarrow.dataset as ds
 import pytest
+from datafusion import SessionContext, Table
 
 
 # Note we take in `database` as a variable even though we don't use
@@ -27,9 +31,9 @@ def test_basic(ctx, database):
         ctx.catalog("non-existent")
 
     default = ctx.catalog()
-    assert default.names() == ["public"]
+    assert default.names() == {"public"}
 
-    for db in [default.database("public"), default.database()]:
+    for db in [default.schema("public"), default.schema()]:
         assert db.names() == {"csv1", "csv", "csv2"}
 
     table = db.table("csv")
@@ -41,3 +45,168 @@ def test_basic(ctx, database):
             pa.field("float", pa.float64(), nullable=True),
         ]
     )
+
+
+def create_dataset() -> Table:
+    batch = pa.RecordBatch.from_arrays(
+        [pa.array([1, 2, 3]), pa.array([4, 5, 6])],
+        names=["a", "b"],
+    )
+    dataset = ds.dataset([batch])
+    return Table.from_dataset(dataset)
+
+
+class CustomSchemaProvider(dfn.catalog.SchemaProvider):
+    def __init__(self):
+        self.tables = {"table1": create_dataset()}
+
+    def table_names(self) -> set[str]:
+        return set(self.tables.keys())
+
+    def register_table(self, name: str, table: Table):
+        self.tables[name] = table
+
+    def deregister_table(self, name, cascade: bool = True):
+        del self.tables[name]
+
+    def table(self, name: str) -> Table | None:
+        return self.tables[name]
+
+    def table_exist(self, name: str) -> bool:
+        return name in self.tables
+
+
+class CustomCatalogProvider(dfn.catalog.CatalogProvider):
+    def __init__(self):
+        self.schemas = {"my_schema": CustomSchemaProvider()}
+
+    def schema_names(self) -> set[str]:
+        return set(self.schemas.keys())
+
+    def schema(self, name: str):
+        return self.schemas[name]
+
+    def register_schema(self, name: str, schema: dfn.catalog.Schema):
+        self.schemas[name] = schema
+
+    def deregister_schema(self, name, cascade: bool):
+        del self.schemas[name]
+
+
+def test_python_catalog_provider(ctx: SessionContext):
+    ctx.register_catalog_provider("my_catalog", CustomCatalogProvider())
+
+    # Check the default catalog provider
+    assert ctx.catalog("datafusion").names() == {"public"}
+
+    my_catalog = ctx.catalog("my_catalog")
+    assert my_catalog.names() == {"my_schema"}
+
+    my_catalog.register_schema("second_schema", CustomSchemaProvider())
+    assert my_catalog.schema_names() == {"my_schema", "second_schema"}
+
+    my_catalog.deregister_schema("my_schema")
+    assert my_catalog.schema_names() == {"second_schema"}
+
+
+def test_in_memory_providers(ctx: SessionContext):
+    catalog = dfn.catalog.Catalog.memory_catalog()
+    ctx.register_catalog_provider("in_mem_catalog", catalog)
+
+    assert ctx.catalog_names() == {"datafusion", "in_mem_catalog"}
+
+    schema = dfn.catalog.Schema.memory_schema()
+    catalog.register_schema("in_mem_schema", schema)
+
+    schema.register_table("my_table", create_dataset())
+
+    batches = ctx.sql("select * from in_mem_catalog.in_mem_schema.my_table").collect()
+
+    assert len(batches) == 1
+    assert batches[0].column(0) == pa.array([1, 2, 3])
+    assert batches[0].column(1) == pa.array([4, 5, 6])
+
+
+def test_python_schema_provider(ctx: SessionContext):
+    catalog = ctx.catalog()
+
+    catalog.deregister_schema("public")
+
+    catalog.register_schema("test_schema1", CustomSchemaProvider())
+    assert catalog.names() == {"test_schema1"}
+
+    catalog.register_schema("test_schema2", CustomSchemaProvider())
+    catalog.deregister_schema("test_schema1")
+    assert catalog.names() == {"test_schema2"}
+
+
+def test_python_table_provider(ctx: SessionContext):
+    catalog = ctx.catalog()
+
+    catalog.register_schema("custom_schema", CustomSchemaProvider())
+    schema = catalog.schema("custom_schema")
+
+    assert schema.table_names() == {"table1"}
+
+    schema.deregister_table("table1")
+    schema.register_table("table2", create_dataset())
+    assert schema.table_names() == {"table2"}
+
+    # Use the default schema instead of our custom schema
+
+    schema = catalog.schema()
+
+    schema.register_table("table3", create_dataset())
+    assert schema.table_names() == {"table3"}
+
+    schema.deregister_table("table3")
+    schema.register_table("table4", create_dataset())
+    assert schema.table_names() == {"table4"}
+
+
+def test_in_end_to_end_python_providers(ctx: SessionContext):
+    """Test registering all python providers and running a query against them."""
+
+    all_catalog_names = [
+        "datafusion",
+        "custom_catalog",
+        "in_mem_catalog",
+    ]
+
+    all_schema_names = [
+        "custom_schema",
+        "in_mem_schema",
+    ]
+
+    ctx.register_catalog_provider(all_catalog_names[1], CustomCatalogProvider())
+    ctx.register_catalog_provider(
+        all_catalog_names[2], dfn.catalog.Catalog.memory_catalog()
+    )
+
+    for catalog_name in all_catalog_names:
+        catalog = ctx.catalog(catalog_name)
+
+        # Clean out previous schemas if they exist so we can start clean
+        for schema_name in catalog.schema_names():
+            catalog.deregister_schema(schema_name, cascade=False)
+
+        catalog.register_schema(all_schema_names[0], CustomSchemaProvider())
+        catalog.register_schema(all_schema_names[1], dfn.catalog.Schema.memory_schema())
+
+        for schema_name in all_schema_names:
+            schema = catalog.schema(schema_name)
+
+            for table_name in schema.table_names():
+                schema.deregister_table(table_name)
+
+            schema.register_table("test_table", create_dataset())
+
+    for catalog_name in all_catalog_names:
+        for schema_name in all_schema_names:
+            table_full_name = f"{catalog_name}.{schema_name}.test_table"
+
+            batches = ctx.sql(f"select * from {table_full_name}").collect()
+
+            assert len(batches) == 1
+            assert batches[0].column(0) == pa.array([1, 2, 3])
+            assert batches[0].column(1) == pa.array([4, 5, 6])
diff --git a/python/tests/test_context.py b/python/tests/test_context.py
index 4a15ac9c..6dbcc0d5 100644
--- a/python/tests/test_context.py
+++ b/python/tests/test_context.py
@@ -57,7 +57,7 @@ def test_runtime_configs(tmp_path, path_to_str):
     ctx = SessionContext(config, runtime)
     assert ctx is not None
 
-    db = ctx.catalog("foo").database("bar")
+    db = ctx.catalog("foo").schema("bar")
     assert db is not None
 
 
@@ -70,7 +70,7 @@ def test_temporary_files(tmp_path, path_to_str):
     ctx = SessionContext(config, runtime)
     assert ctx is not None
 
-    db = ctx.catalog("foo").database("bar")
+    db = ctx.catalog("foo").schema("bar")
     assert db is not None
 
 
@@ -91,7 +91,7 @@ def test_create_context_with_all_valid_args():
     ctx = SessionContext(config, runtime)
 
     # verify that at least some of the arguments worked
-    ctx.catalog("foo").database("bar")
+    ctx.catalog("foo").schema("bar")
     with pytest.raises(KeyError):
         ctx.catalog("datafusion")
 
@@ -105,7 +105,7 @@ def test_register_record_batches(ctx):
 
     ctx.register_record_batches("t", [[batch]])
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     result = ctx.sql("SELECT a+b, a-b FROM t").collect()
 
@@ -121,7 +121,7 @@ def test_create_dataframe_registers_unique_table_name(ctx):
     )
 
     df = ctx.create_dataframe([[batch]])
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -141,7 +141,7 @@ def test_create_dataframe_registers_with_defined_table_name(ctx):
     )
 
     df = ctx.create_dataframe([[batch]], name="tbl")
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -155,7 +155,7 @@ def test_from_arrow_table(ctx):
 
     # convert to DataFrame
     df = ctx.from_arrow(table)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -200,7 +200,7 @@ def test_from_arrow_table_with_name(ctx):
 
     # convert to DataFrame with optional name
     df = ctx.from_arrow(table, name="tbl")
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert tables[0] == "tbl"
@@ -213,7 +213,7 @@ def test_from_arrow_table_empty(ctx):
 
     # convert to DataFrame
     df = ctx.from_arrow(table)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -228,7 +228,7 @@ def test_from_arrow_table_empty_no_schema(ctx):
 
     # convert to DataFrame
     df = ctx.from_arrow(table)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -246,7 +246,7 @@ def test_from_pylist(ctx):
     ]
 
     df = ctx.from_pylist(data)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -260,7 +260,7 @@ def test_from_pydict(ctx):
     data = {"a": [1, 2, 3], "b": [4, 5, 6]}
 
     df = ctx.from_pydict(data)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -276,7 +276,7 @@ def test_from_pandas(ctx):
     pandas_df = pd.DataFrame(data)
 
     df = ctx.from_pandas(pandas_df)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -292,7 +292,7 @@ def test_from_polars(ctx):
     polars_df = pd.DataFrame(data)
 
     df = ctx.from_polars(polars_df)
-    tables = list(ctx.catalog().database().names())
+    tables = list(ctx.catalog().schema().names())
 
     assert df
     assert len(tables) == 1
@@ -303,7 +303,7 @@ def test_from_polars(ctx):
 
 def test_register_table(ctx, database):
     default = ctx.catalog()
-    public = default.database("public")
+    public = default.schema("public")
     assert public.names() == {"csv", "csv1", "csv2"}
     table = public.table("csv")
 
@@ -313,7 +313,7 @@ def test_register_table(ctx, database):
 
 def test_read_table(ctx, database):
     default = ctx.catalog()
-    public = default.database("public")
+    public = default.schema("public")
     assert public.names() == {"csv", "csv1", "csv2"}
 
     table = public.table("csv")
@@ -323,7 +323,7 @@ def test_read_table(ctx, database):
 
 def test_deregister_table(ctx, database):
     default = ctx.catalog()
-    public = default.database("public")
+    public = default.schema("public")
     assert public.names() == {"csv", "csv1", "csv2"}
 
     ctx.deregister_table("csv")
@@ -339,7 +339,7 @@ def test_register_dataset(ctx):
     dataset = ds.dataset([batch])
     ctx.register_dataset("t", dataset)
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     result = ctx.sql("SELECT a+b, a-b FROM t").collect()
 
@@ -356,7 +356,7 @@ def test_dataset_filter(ctx, capfd):
     dataset = ds.dataset([batch])
     ctx.register_dataset("t", dataset)
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
     df = ctx.sql("SELECT a+b, a-b FROM t WHERE a BETWEEN 2 and 3 AND b > 5")
 
     # Make sure the filter was pushed down in Physical Plan
@@ -455,7 +455,7 @@ def test_dataset_filter_nested_data(ctx):
     dataset = ds.dataset([batch])
     ctx.register_dataset("t", dataset)
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     df = ctx.table("t")
 
diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py
index 41cee4ef..c383edc6 100644
--- a/python/tests/test_sql.py
+++ b/python/tests/test_sql.py
@@ -75,7 +75,7 @@ def test_register_csv(ctx, tmp_path):
     )
     ctx.register_csv("csv3", path, schema=alternative_schema)
 
-    assert ctx.catalog().database().names() == {
+    assert ctx.catalog().schema().names() == {
         "csv",
         "csv1",
         "csv2",
@@ -150,7 +150,7 @@ def test_register_parquet(ctx, tmp_path):
     path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data())
     ctx.register_parquet("t", path)
     ctx.register_parquet("t1", str(path))
-    assert ctx.catalog().database().names() == {"t", "t1"}
+    assert ctx.catalog().schema().names() == {"t", "t1"}
 
     result = ctx.sql("SELECT COUNT(a) AS cnt FROM t").collect()
     result = pa.Table.from_batches(result)
@@ -188,7 +188,7 @@ def test_register_parquet_partitioned(ctx, tmp_path, path_to_str, legacy_data_ty
         parquet_pruning=True,
         file_extension=".parquet",
     )
-    assert ctx.catalog().database().names() == {"datapp"}
+    assert ctx.catalog().schema().names() == {"datapp"}
 
     result = ctx.sql("SELECT grp, COUNT(*) AS cnt FROM datapp GROUP BY grp").collect()
     result = pa.Table.from_batches(result)
@@ -204,7 +204,7 @@ def test_register_dataset(ctx, tmp_path, path_to_str):
     dataset = ds.dataset(path, format="parquet")
 
     ctx.register_dataset("t", dataset)
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     result = ctx.sql("SELECT COUNT(a) AS cnt FROM t").collect()
     result = pa.Table.from_batches(result)
@@ -251,7 +251,7 @@ def test_register_json(ctx, tmp_path):
     )
     ctx.register_json("json3", path, schema=alternative_schema)
 
-    assert ctx.catalog().database().names() == {
+    assert ctx.catalog().schema().names() == {
         "json",
         "json1",
         "json2",
@@ -308,7 +308,7 @@ def test_execute(ctx, tmp_path):
     path = helpers.write_parquet(tmp_path / "a.parquet", pa.array(data))
     ctx.register_parquet("t", path)
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     # count
     result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").collect()
@@ -451,18 +451,10 @@ _null_mask = np.array([False, True, False])
             id="datetime_ns",
         ),
         # Not writtable to parquet
-        pytest.param(
-            helpers.data_timedelta("s"), id="timedelta_s", marks=pytest.mark.xfail
-        ),
-        pytest.param(
-            helpers.data_timedelta("ms"), id="timedelta_ms", marks=pytest.mark.xfail
-        ),
-        pytest.param(
-            helpers.data_timedelta("us"), id="timedelta_us", marks=pytest.mark.xfail
-        ),
-        pytest.param(
-            helpers.data_timedelta("ns"), id="timedelta_ns", marks=pytest.mark.xfail
-        ),
+        pytest.param(helpers.data_timedelta("s"), id="timedelta_s"),
+        pytest.param(helpers.data_timedelta("ms"), id="timedelta_ms"),
+        pytest.param(helpers.data_timedelta("us"), id="timedelta_us"),
+        pytest.param(helpers.data_timedelta("ns"), id="timedelta_ns"),
     ],
 )
 def test_simple_select(ctx, tmp_path, arr):
@@ -524,7 +516,7 @@ def test_register_listing_table(
         schema=table.schema if pass_schema else None,
         file_sort_order=file_sort_order,
     )
-    assert ctx.catalog().database().names() == {"my_table"}
+    assert ctx.catalog().schema().names() == {"my_table"}
 
     result = ctx.sql(
         "SELECT grp, COUNT(*) AS count FROM my_table GROUP BY grp"
diff --git a/python/tests/test_substrait.py b/python/tests/test_substrait.py
index f367a447..43aa327d 100644
--- a/python/tests/test_substrait.py
+++ b/python/tests/test_substrait.py
@@ -34,7 +34,7 @@ def test_substrait_serialization(ctx):
 
     ctx.register_record_batches("t", [[batch]])
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     # For now just make sure the method calls blow up
     substrait_plan = ss.Serde.serialize_to_plan("SELECT * FROM t", ctx)
@@ -59,7 +59,7 @@ def test_substrait_file_serialization(ctx, tmp_path, path_to_str):
 
     ctx.register_record_batches("t", [[batch]])
 
-    assert ctx.catalog().database().names() == {"t"}
+    assert ctx.catalog().schema().names() == {"t"}
 
     path = tmp_path / "substrait_plan"
     path = str(path) if path_to_str else path
diff --git a/src/catalog.rs b/src/catalog.rs
index 83f8d08c..17d4ec3b 100644
--- a/src/catalog.rs
+++ b/src/catalog.rs
@@ -15,44 +15,54 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use std::collections::HashSet;
-use std::sync::Arc;
-
-use pyo3::exceptions::PyKeyError;
-use pyo3::prelude::*;
-
-use crate::errors::{PyDataFusionError, PyDataFusionResult};
-use crate::utils::wait_for_future;
+use crate::dataset::Dataset;
+use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionError, PyDataFusionResult};
+use crate::utils::{validate_pycapsule, wait_for_future};
+use async_trait::async_trait;
+use datafusion::catalog::{MemoryCatalogProvider, MemorySchemaProvider};
+use datafusion::common::DataFusionError;
 use datafusion::{
     arrow::pyarrow::ToPyArrow,
     catalog::{CatalogProvider, SchemaProvider},
     datasource::{TableProvider, TableType},
 };
+use datafusion_ffi::schema_provider::{FFI_SchemaProvider, ForeignSchemaProvider};
+use datafusion_ffi::table_provider::{FFI_TableProvider, ForeignTableProvider};
+use pyo3::exceptions::PyKeyError;
+use pyo3::prelude::*;
+use pyo3::types::PyCapsule;
+use pyo3::IntoPyObjectExt;
+use std::any::Any;
+use std::collections::HashSet;
+use std::sync::Arc;
 
-#[pyclass(name = "Catalog", module = "datafusion", subclass)]
+#[pyclass(name = "RawCatalog", module = "datafusion.catalog", subclass)]
+#[derive(Clone)]
 pub struct PyCatalog {
     pub catalog: Arc<dyn CatalogProvider>,
 }
 
-#[pyclass(name = "Database", module = "datafusion", subclass)]
-pub struct PyDatabase {
-    pub database: Arc<dyn SchemaProvider>,
+#[pyclass(name = "RawSchema", module = "datafusion.catalog", subclass)]
+#[derive(Clone)]
+pub struct PySchema {
+    pub schema: Arc<dyn SchemaProvider>,
 }
 
-#[pyclass(name = "Table", module = "datafusion", subclass)]
+#[pyclass(name = "RawTable", module = "datafusion.catalog", subclass)]
+#[derive(Clone)]
 pub struct PyTable {
     pub table: Arc<dyn TableProvider>,
 }
 
-impl PyCatalog {
-    pub fn new(catalog: Arc<dyn CatalogProvider>) -> Self {
+impl From<Arc<dyn CatalogProvider>> for PyCatalog {
+    fn from(catalog: Arc<dyn CatalogProvider>) -> Self {
         Self { catalog }
     }
 }
 
-impl PyDatabase {
-    pub fn new(database: Arc<dyn SchemaProvider>) -> Self {
-        Self { database }
+impl From<Arc<dyn SchemaProvider>> for PySchema {
+    fn from(schema: Arc<dyn SchemaProvider>) -> Self {
+        Self { schema }
     }
 }
 
@@ -68,36 +78,109 @@ impl PyTable {
 
 #[pymethods]
 impl PyCatalog {
-    fn names(&self) -> Vec<String> {
-        self.catalog.schema_names()
+    #[new]
+    fn new(catalog: PyObject) -> Self {
+        let catalog_provider =
+            Arc::new(RustWrappedPyCatalogProvider::new(catalog)) as Arc<dyn CatalogProvider>;
+        catalog_provider.into()
+    }
+
+    #[staticmethod]
+    fn memory_catalog() -> Self {
+        let catalog_provider =
+            Arc::new(MemoryCatalogProvider::default()) as Arc<dyn CatalogProvider>;
+        catalog_provider.into()
+    }
+
+    fn schema_names(&self) -> HashSet<String> {
+        self.catalog.schema_names().into_iter().collect()
     }
 
     #[pyo3(signature = (name="public"))]
-    fn database(&self, name: &str) -> PyResult<PyDatabase> {
-        match self.catalog.schema(name) {
-            Some(database) => Ok(PyDatabase::new(database)),
-            None => Err(PyKeyError::new_err(format!(
-                "Database with name {name} doesn't exist."
-            ))),
-        }
+    fn schema(&self, name: &str) -> PyResult<PyObject> {
+        let schema = self
+            .catalog
+            .schema(name)
+            .ok_or(PyKeyError::new_err(format!(
+                "Schema with name {name} doesn't exist."
+            )))?;
+
+        Python::with_gil(|py| {
+            match schema
+                .as_any()
+                .downcast_ref::<RustWrappedPySchemaProvider>()
+            {
+                Some(wrapped_schema) => Ok(wrapped_schema.schema_provider.clone_ref(py)),
+                None => PySchema::from(schema).into_py_any(py),
+            }
+        })
+    }
+
+    fn register_schema(&self, name: &str, schema_provider: Bound<'_, PyAny>) -> PyResult<()> {
+        let provider = if schema_provider.hasattr("__datafusion_schema_provider__")? {
+            let capsule = schema_provider
+                .getattr("__datafusion_schema_provider__")?
+                .call0()?;
+            let capsule = capsule.downcast::<PyCapsule>().map_err(py_datafusion_err)?;
+            validate_pycapsule(capsule, "datafusion_schema_provider")?;
+
+            let provider = unsafe { capsule.reference::<FFI_SchemaProvider>() };
+            let provider: ForeignSchemaProvider = provider.into();
+            Arc::new(provider) as Arc<dyn SchemaProvider>
+        } else {
+            match schema_provider.extract::<PySchema>() {
+                Ok(py_schema) => py_schema.schema,
+                Err(_) => Arc::new(RustWrappedPySchemaProvider::new(schema_provider.into()))
+                    as Arc<dyn SchemaProvider>,
+            }
+        };
+
+        let _ = self
+            .catalog
+            .register_schema(name, provider)
+            .map_err(py_datafusion_err)?;
+
+        Ok(())
+    }
+
+    fn deregister_schema(&self, name: &str, cascade: bool) -> PyResult<()> {
+        let _ = self
+            .catalog
+            .deregister_schema(name, cascade)
+            .map_err(py_datafusion_err)?;
+
+        Ok(())
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!(
-            "Catalog(schema_names=[{}])",
-            self.names().join(";")
-        ))
+        let mut names: Vec<String> = self.schema_names().into_iter().collect();
+        names.sort();
+        Ok(format!("Catalog(schema_names=[{}])", names.join(", ")))
     }
 }
 
 #[pymethods]
-impl PyDatabase {
-    fn names(&self) -> HashSet<String> {
-        self.database.table_names().into_iter().collect()
+impl PySchema {
+    #[new]
+    fn new(schema_provider: PyObject) -> Self {
+        let schema_provider =
+            Arc::new(RustWrappedPySchemaProvider::new(schema_provider)) as Arc<dyn SchemaProvider>;
+        schema_provider.into()
+    }
+
+    #[staticmethod]
+    fn memory_schema() -> Self {
+        let schema_provider = Arc::new(MemorySchemaProvider::default()) as Arc<dyn SchemaProvider>;
+        schema_provider.into()
+    }
+
+    #[getter]
+    fn table_names(&self) -> HashSet<String> {
+        self.schema.table_names().into_iter().collect()
     }
 
     fn table(&self, name: &str, py: Python) -> PyDataFusionResult<PyTable> {
-        if let Some(table) = wait_for_future(py, self.database.table(name))?? {
+        if let Some(table) = wait_for_future(py, self.schema.table(name))?? {
             Ok(PyTable::new(table))
         } else {
             Err(PyDataFusionError::Common(format!(
@@ -107,14 +190,49 @@ impl PyDatabase {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!(
-            "Database(table_names=[{}])",
-            Vec::from_iter(self.names()).join(";")
-        ))
+        let mut names: Vec<String> = self.table_names().into_iter().collect();
+        names.sort();
+        Ok(format!("Schema(table_names=[{}])", names.join(";")))
     }
 
-    // register_table
-    // deregister_table
+    fn register_table(&self, name: &str, table_provider: Bound<'_, PyAny>) -> PyResult<()> {
+        let provider = if table_provider.hasattr("__datafusion_table_provider__")? {
+            let capsule = table_provider
+                .getattr("__datafusion_table_provider__")?
+                .call0()?;
+            let capsule = capsule.downcast::<PyCapsule>().map_err(py_datafusion_err)?;
+            validate_pycapsule(capsule, "datafusion_table_provider")?;
+
+            let provider = unsafe { capsule.reference::<FFI_TableProvider>() };
+            let provider: ForeignTableProvider = provider.into();
+            Arc::new(provider) as Arc<dyn TableProvider>
+        } else {
+            match table_provider.extract::<PyTable>() {
+                Ok(py_table) => py_table.table,
+                Err(_) => {
+                    let py = table_provider.py();
+                    let provider = Dataset::new(&table_provider, py)?;
+                    Arc::new(provider) as Arc<dyn TableProvider>
+                }
+            }
+        };
+
+        let _ = self
+            .schema
+            .register_table(name.to_string(), provider)
+            .map_err(py_datafusion_err)?;
+
+        Ok(())
+    }
+
+    fn deregister_table(&self, name: &str) -> PyResult<()> {
+        let _ = self
+            .schema
+            .deregister_table(name)
+            .map_err(py_datafusion_err)?;
+
+        Ok(())
+    }
 }
 
 #[pymethods]
@@ -125,6 +243,14 @@ impl PyTable {
         self.table.schema().to_pyarrow(py)
     }
 
+    #[staticmethod]
+    fn from_dataset(py: Python<'_>, dataset: &Bound<'_, PyAny>) -> PyResult<Self> {
+        let ds = Arc::new(Dataset::new(dataset, py).map_err(py_datafusion_err)?)
+            as Arc<dyn TableProvider>;
+
+        Ok(Self::new(ds))
+    }
+
     /// Get the type of this table for metadata/catalog purposes.
     #[getter]
     fn kind(&self) -> &str {
@@ -145,3 +271,285 @@ impl PyTable {
     // fn has_exact_statistics
     // fn supports_filter_pushdown
 }
+
+#[derive(Debug)]
+pub(crate) struct RustWrappedPySchemaProvider {
+    schema_provider: PyObject,
+    owner_name: Option<String>,
+}
+
+impl RustWrappedPySchemaProvider {
+    pub fn new(schema_provider: PyObject) -> Self {
+        let owner_name = Python::with_gil(|py| {
+            schema_provider
+                .bind(py)
+                .getattr("owner_name")
+                .ok()
+                .map(|name| name.to_string())
+        });
+
+        Self {
+            schema_provider,
+            owner_name,
+        }
+    }
+
+    fn table_inner(&self, name: &str) -> PyResult<Option<Arc<dyn TableProvider>>> {
+        Python::with_gil(|py| {
+            let provider = self.schema_provider.bind(py);
+            let py_table_method = provider.getattr("table")?;
+
+            let py_table = py_table_method.call((name,), None)?;
+            if py_table.is_none() {
+                return Ok(None);
+            }
+
+            if py_table.hasattr("__datafusion_table_provider__")? {
+                let capsule = provider.getattr("__datafusion_table_provider__")?.call0()?;
+                let capsule = capsule.downcast::<PyCapsule>().map_err(py_datafusion_err)?;
+                validate_pycapsule(capsule, "datafusion_table_provider")?;
+
+                let provider = unsafe { capsule.reference::<FFI_TableProvider>() };
+                let provider: ForeignTableProvider = provider.into();
+
+                Ok(Some(Arc::new(provider) as Arc<dyn TableProvider>))
+            } else {
+                if let Ok(inner_table) = py_table.getattr("table") {
+                    if let Ok(inner_table) = inner_table.extract::<PyTable>() {
+                        return Ok(Some(inner_table.table));
+                    }
+                }
+
+                match py_table.extract::<PyTable>() {
+                    Ok(py_table) => Ok(Some(py_table.table)),
+                    Err(_) => {
+                        let ds = Dataset::new(&py_table, py).map_err(py_datafusion_err)?;
+                        Ok(Some(Arc::new(ds) as Arc<dyn TableProvider>))
+                    }
+                }
+            }
+        })
+    }
+}
+
+#[async_trait]
+impl SchemaProvider for RustWrappedPySchemaProvider {
+    fn owner_name(&self) -> Option<&str> {
+        self.owner_name.as_deref()
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn table_names(&self) -> Vec<String> {
+        Python::with_gil(|py| {
+            let provider = self.schema_provider.bind(py);
+
+            provider
+                .getattr("table_names")
+                .and_then(|names| names.extract::<Vec<String>>())
+                .unwrap_or_else(|err| {
+                    log::error!("Unable to get table_names: {err}");
+                    Vec::default()
+                })
+        })
+    }
+
+    async fn table(
+        &self,
+        name: &str,
+    ) -> datafusion::common::Result<Option<Arc<dyn TableProvider>>, DataFusionError> {
+        self.table_inner(name).map_err(to_datafusion_err)
+    }
+
+    fn register_table(
+        &self,
+        name: String,
+        table: Arc<dyn TableProvider>,
+    ) -> datafusion::common::Result<Option<Arc<dyn TableProvider>>> {
+        let py_table = PyTable::new(table);
+        Python::with_gil(|py| {
+            let provider = self.schema_provider.bind(py);
+            let _ = provider
+                .call_method1("register_table", (name, py_table))
+                .map_err(to_datafusion_err)?;
+            // Since the definition of `register_table` says that an error
+            // will be returned if the table already exists, there is no
+            // case where we want to return a table provider as output.
+            Ok(None)
+        })
+    }
+
+    fn deregister_table(
+        &self,
+        name: &str,
+    ) -> datafusion::common::Result<Option<Arc<dyn TableProvider>>> {
+        Python::with_gil(|py| {
+            let provider = self.schema_provider.bind(py);
+            let table = provider
+                .call_method1("deregister_table", (name,))
+                .map_err(to_datafusion_err)?;
+            if table.is_none() {
+                return Ok(None);
+            }
+
+            // If we can turn this table provider into a `Dataset`, return it.
+            // Otherwise, return None.
+            let dataset = match Dataset::new(&table, py) {
+                Ok(dataset) => Some(Arc::new(dataset) as Arc<dyn TableProvider>),
+                Err(_) => None,
+            };
+
+            Ok(dataset)
+        })
+    }
+
+    fn table_exist(&self, name: &str) -> bool {
+        Python::with_gil(|py| {
+            let provider = self.schema_provider.bind(py);
+            provider
+                .call_method1("table_exist", (name,))
+                .and_then(|pyobj| pyobj.extract())
+                .unwrap_or(false)
+        })
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct RustWrappedPyCatalogProvider {
+    pub(crate) catalog_provider: PyObject,
+}
+
+impl RustWrappedPyCatalogProvider {
+    pub fn new(catalog_provider: PyObject) -> Self {
+        Self { catalog_provider }
+    }
+
+    fn schema_inner(&self, name: &str) -> PyResult<Option<Arc<dyn SchemaProvider>>> {
+        Python::with_gil(|py| {
+            let provider = self.catalog_provider.bind(py);
+
+            let py_schema = provider.call_method1("schema", (name,))?;
+            if py_schema.is_none() {
+                return Ok(None);
+            }
+
+            if py_schema.hasattr("__datafusion_schema_provider__")? {
+                let capsule = provider
+                    .getattr("__datafusion_schema_provider__")?
+                    .call0()?;
+                let capsule = capsule.downcast::<PyCapsule>().map_err(py_datafusion_err)?;
+                validate_pycapsule(capsule, "datafusion_schema_provider")?;
+
+                let provider = unsafe { capsule.reference::<FFI_SchemaProvider>() };
+                let provider: ForeignSchemaProvider = provider.into();
+
+                Ok(Some(Arc::new(provider) as Arc<dyn SchemaProvider>))
+            } else {
+                if let Ok(inner_schema) = py_schema.getattr("schema") {
+                    if let Ok(inner_schema) = inner_schema.extract::<PySchema>() {
+                        return Ok(Some(inner_schema.schema));
+                    }
+                }
+                match py_schema.extract::<PySchema>() {
+                    Ok(inner_schema) => Ok(Some(inner_schema.schema)),
+                    Err(_) => {
+                        let py_schema = RustWrappedPySchemaProvider::new(py_schema.into());
+
+                        Ok(Some(Arc::new(py_schema) as Arc<dyn SchemaProvider>))
+                    }
+                }
+            }
+        })
+    }
+}
+
+#[async_trait]
+impl CatalogProvider for RustWrappedPyCatalogProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema_names(&self) -> Vec<String> {
+        Python::with_gil(|py| {
+            let provider = self.catalog_provider.bind(py);
+            provider
+                .getattr("schema_names")
+                .and_then(|names| names.extract::<Vec<String>>())
+                .unwrap_or_else(|err| {
+                    log::error!("Unable to get schema_names: {err}");
+                    Vec::default()
+                })
+        })
+    }
+
+    fn schema(&self, name: &str) -> Option<Arc<dyn SchemaProvider>> {
+        self.schema_inner(name).unwrap_or_else(|err| {
+            log::error!("CatalogProvider schema returned error: {err}");
+            None
+        })
+    }
+
+    fn register_schema(
+        &self,
+        name: &str,
+        schema: Arc<dyn SchemaProvider>,
+    ) -> datafusion::common::Result<Option<Arc<dyn SchemaProvider>>> {
+        // JRIGHT HERE
+        // let py_schema: PySchema = schema.into();
+        Python::with_gil(|py| {
+            let py_schema = match schema
+                .as_any()
+                .downcast_ref::<RustWrappedPySchemaProvider>()
+            {
+                Some(wrapped_schema) => wrapped_schema.schema_provider.as_any(),
+                None => &PySchema::from(schema)
+                    .into_py_any(py)
+                    .map_err(to_datafusion_err)?,
+            };
+
+            let provider = self.catalog_provider.bind(py);
+            let schema = provider
+                .call_method1("register_schema", (name, py_schema))
+                .map_err(to_datafusion_err)?;
+            if schema.is_none() {
+                return Ok(None);
+            }
+
+            let schema = Arc::new(RustWrappedPySchemaProvider::new(schema.into()))
+                as Arc<dyn SchemaProvider>;
+
+            Ok(Some(schema))
+        })
+    }
+
+    fn deregister_schema(
+        &self,
+        name: &str,
+        cascade: bool,
+    ) -> datafusion::common::Result<Option<Arc<dyn SchemaProvider>>> {
+        Python::with_gil(|py| {
+            let provider = self.catalog_provider.bind(py);
+            let schema = provider
+                .call_method1("deregister_schema", (name, cascade))
+                .map_err(to_datafusion_err)?;
+            if schema.is_none() {
+                return Ok(None);
+            }
+
+            let schema = Arc::new(RustWrappedPySchemaProvider::new(schema.into()))
+                as Arc<dyn SchemaProvider>;
+
+            Ok(Some(schema))
+        })
+    }
+}
+
+pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<PyCatalog>()?;
+    m.add_class::<PySchema>()?;
+    m.add_class::<PyTable>()?;
+
+    Ok(())
+}
diff --git a/src/common/data_type.rs b/src/common/data_type.rs
index f5f8a6b0..5cf9d6e9 100644
--- a/src/common/data_type.rs
+++ b/src/common/data_type.rs
@@ -172,7 +172,7 @@ impl DataTypeMap {
                 SqlType::DATE,
             )),
             DataType::Duration(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::Interval(interval_unit) => Ok(DataTypeMap::new(
                 DataType::Interval(*interval_unit),
@@ -189,7 +189,7 @@ impl DataTypeMap {
                 SqlType::BINARY,
             )),
             DataType::FixedSizeBinary(_) => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", arrow_type)),
+                DataFusionError::NotImplemented(format!("{arrow_type:?}")),
             )),
             DataType::LargeBinary => Ok(DataTypeMap::new(
                 DataType::LargeBinary,
@@ -207,23 +207,22 @@ impl DataTypeMap {
                 SqlType::VARCHAR,
             )),
             DataType::List(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                arrow_type
+                "{arrow_type:?}"
             )))),
             DataType::FixedSizeList(_, _) => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", arrow_type)),
+                DataFusionError::NotImplemented(format!("{arrow_type:?}")),
             )),
             DataType::LargeList(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::Struct(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::Union(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::Dictionary(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::Decimal128(precision, scale) => Ok(DataTypeMap::new(
                 DataType::Decimal128(*precision, *scale),
@@ -236,23 +235,22 @@ impl DataTypeMap {
                 SqlType::DECIMAL,
             )),
             DataType::Map(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::RunEndEncoded(_, _) => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", arrow_type)),
+                DataFusionError::NotImplemented(format!("{arrow_type:?}")),
             )),
             DataType::BinaryView => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::Utf8View => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                arrow_type
+                "{arrow_type:?}"
             )))),
             DataType::ListView(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
             DataType::LargeListView(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", arrow_type),
+                format!("{arrow_type:?}"),
             ))),
         }
     }
@@ -379,8 +377,7 @@ impl DataTypeMap {
             "double" => Ok(DataType::Float64),
             "byte_array" => Ok(DataType::Utf8),
             _ => Err(PyValueError::new_err(format!(
-                "Unable to determine Arrow Data Type from Parquet String type: {:?}",
-                parquet_str_type
+                "Unable to determine Arrow Data Type from Parquet String type: {parquet_str_type:?}"
             ))),
         };
         DataTypeMap::map_from_arrow_type(&arrow_dtype?)
@@ -404,12 +401,10 @@ impl DataTypeMap {
     pub fn py_map_from_sql_type(sql_type: &SqlType) -> PyResult<DataTypeMap> {
         match sql_type {
             SqlType::ANY => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::ARRAY => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::BIGINT => Ok(DataTypeMap::new(
                 DataType::Int64,
@@ -432,11 +427,10 @@ impl DataTypeMap {
                 SqlType::CHAR,
             )),
             SqlType::COLUMN_LIST => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::CURSOR => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::DATE => Ok(DataTypeMap::new(
                 DataType::Date64,
@@ -449,8 +443,7 @@ impl DataTypeMap {
                 SqlType::DECIMAL,
             )),
             SqlType::DISTINCT => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::DOUBLE => Ok(DataTypeMap::new(
                 DataType::Decimal256(1, 1),
@@ -458,7 +451,7 @@ impl DataTypeMap {
                 SqlType::DOUBLE,
             )),
             SqlType::DYNAMIC_STAR => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::FLOAT => Ok(DataTypeMap::new(
                 DataType::Decimal128(1, 1),
@@ -466,8 +459,7 @@ impl DataTypeMap {
                 SqlType::FLOAT,
             )),
             SqlType::GEOMETRY => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::INTEGER => Ok(DataTypeMap::new(
                 DataType::Int8,
@@ -475,55 +467,52 @@ impl DataTypeMap {
                 SqlType::INTEGER,
             )),
             SqlType::INTERVAL => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::INTERVAL_DAY => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_DAY_HOUR => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_DAY_MINUTE => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::INTERVAL_DAY_SECOND => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::INTERVAL_HOUR => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_HOUR_MINUTE => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::INTERVAL_HOUR_SECOND => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::INTERVAL_MINUTE => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_MINUTE_SECOND => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::INTERVAL_MONTH => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_SECOND => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_YEAR => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::INTERVAL_YEAR_MONTH => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::MAP => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::MULTISET => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::NULL => Ok(DataTypeMap::new(
                 DataType::Null,
@@ -531,20 +520,16 @@ impl DataTypeMap {
                 SqlType::NULL,
             )),
             SqlType::OTHER => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::REAL => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::ROW => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::SARG => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::SMALLINT => Ok(DataTypeMap::new(
                 DataType::Int16,
@@ -552,25 +537,22 @@ impl DataTypeMap {
                 SqlType::SMALLINT,
             )),
             SqlType::STRUCTURED => Err(py_datafusion_err(DataFusionError::NotImplemented(
-                format!("{:?}", sql_type),
+                format!("{sql_type:?}"),
             ))),
             SqlType::SYMBOL => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::TIME => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::TIME_WITH_LOCAL_TIME_ZONE => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::TIMESTAMP => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::TIMESTAMP_WITH_LOCAL_TIME_ZONE => Err(py_datafusion_err(
-                DataFusionError::NotImplemented(format!("{:?}", sql_type)),
+                DataFusionError::NotImplemented(format!("{sql_type:?}")),
             )),
             SqlType::TINYINT => Ok(DataTypeMap::new(
                 DataType::Int8,
@@ -578,8 +560,7 @@ impl DataTypeMap {
                 SqlType::TINYINT,
             )),
             SqlType::UNKNOWN => Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                "{:?}",
-                sql_type
+                "{sql_type:?}"
             )))),
             SqlType::VARBINARY => Ok(DataTypeMap::new(
                 DataType::LargeBinary,
@@ -682,8 +663,7 @@ impl PyDataType {
             "datetime64" => Ok(DataType::Date64),
             "object" => Ok(DataType::Utf8),
             _ => Err(PyValueError::new_err(format!(
-                "Unable to determine Arrow Data Type from Arrow String type: {:?}",
-                arrow_str_type
+                "Unable to determine Arrow Data Type from Arrow String type: {arrow_str_type:?}"
             ))),
         };
         Ok(PyDataType {
diff --git a/src/context.rs b/src/context.rs
index 6ce1f12b..36133a33 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -31,7 +31,7 @@ use uuid::Uuid;
 use pyo3::exceptions::{PyKeyError, PyValueError};
 use pyo3::prelude::*;
 
-use crate::catalog::{PyCatalog, PyTable};
+use crate::catalog::{PyCatalog, PyTable, RustWrappedPyCatalogProvider};
 use crate::dataframe::PyDataFrame;
 use crate::dataset::Dataset;
 use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionResult};
@@ -49,6 +49,7 @@ use crate::utils::{get_global_ctx, get_tokio_runtime, validate_pycapsule, wait_f
 use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef};
 use datafusion::arrow::pyarrow::PyArrowType;
 use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::catalog::CatalogProvider;
 use datafusion::common::TableReference;
 use datafusion::common::{exec_err, ScalarValue};
 use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
@@ -69,8 +70,10 @@ use datafusion::physical_plan::SendableRecordBatchStream;
 use datafusion::prelude::{
     AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions,
 };
+use datafusion_ffi::catalog_provider::{FFI_CatalogProvider, ForeignCatalogProvider};
 use datafusion_ffi::table_provider::{FFI_TableProvider, ForeignTableProvider};
 use pyo3::types::{PyCapsule, PyDict, PyList, PyTuple, PyType};
+use pyo3::IntoPyObjectExt;
 use tokio::task::JoinHandle;
 
 /// Configuration options for a SessionContext
@@ -365,7 +368,7 @@ impl PySessionContext {
         } else {
             &upstream_host
         };
-        let url_string = format!("{}{}", scheme, derived_host);
+        let url_string = format!("{scheme}{derived_host}");
         let url = Url::parse(&url_string).unwrap();
         self.ctx.runtime_env().register_object_store(&url, store);
         Ok(())
@@ -614,6 +617,34 @@ impl PySessionContext {
         Ok(())
     }
 
+    pub fn register_catalog_provider(
+        &mut self,
+        name: &str,
+        provider: Bound<'_, PyAny>,
+    ) -> PyDataFusionResult<()> {
+        let provider = if provider.hasattr("__datafusion_catalog_provider__")? {
+            let capsule = provider
+                .getattr("__datafusion_catalog_provider__")?
+                .call0()?;
+            let capsule = capsule.downcast::<PyCapsule>().map_err(py_datafusion_err)?;
+            validate_pycapsule(capsule, "datafusion_catalog_provider")?;
+
+            let provider = unsafe { capsule.reference::<FFI_CatalogProvider>() };
+            let provider: ForeignCatalogProvider = provider.into();
+            Arc::new(provider) as Arc<dyn CatalogProvider>
+        } else {
+            match provider.extract::<PyCatalog>() {
+                Ok(py_catalog) => py_catalog.catalog,
+                Err(_) => Arc::new(RustWrappedPyCatalogProvider::new(provider.into()))
+                    as Arc<dyn CatalogProvider>,
+            }
+        };
+
+        let _ = self.ctx.register_catalog(name, provider);
+
+        Ok(())
+    }
+
     /// Construct datafusion dataframe from Arrow Table
     pub fn register_table_provider(
         &mut self,
@@ -845,14 +876,24 @@ impl PySessionContext {
     }
 
     #[pyo3(signature = (name="datafusion"))]
-    pub fn catalog(&self, name: &str) -> PyResult<PyCatalog> {
-        match self.ctx.catalog(name) {
-            Some(catalog) => Ok(PyCatalog::new(catalog)),
-            None => Err(PyKeyError::new_err(format!(
-                "Catalog with name {} doesn't exist.",
-                &name,
-            ))),
-        }
+    pub fn catalog(&self, name: &str) -> PyResult<PyObject> {
+        let catalog = self.ctx.catalog(name).ok_or(PyKeyError::new_err(format!(
+            "Catalog with name {name} doesn't exist."
+        )))?;
+
+        Python::with_gil(|py| {
+            match catalog
+                .as_any()
+                .downcast_ref::<RustWrappedPyCatalogProvider>()
+            {
+                Some(wrapped_schema) => Ok(wrapped_schema.catalog_provider.clone_ref(py)),
+                None => PyCatalog::from(catalog).into_py_any(py),
+            }
+        })
+    }
+
+    pub fn catalog_names(&self) -> HashSet<String> {
+        self.ctx.catalog_names().into_iter().collect()
     }
 
     pub fn tables(&self) -> HashSet<String> {
diff --git a/src/expr.rs b/src/expr.rs
index 6b1d01d6..f1e00236 100644
--- a/src/expr.rs
+++ b/src/expr.rs
@@ -171,12 +171,10 @@ impl PyExpr {
             Expr::Cast(value) => Ok(cast::PyCast::from(value.clone()).into_bound_py_any(py)?),
             Expr::TryCast(value) => Ok(cast::PyTryCast::from(value.clone()).into_bound_py_any(py)?),
             Expr::ScalarFunction(value) => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::ScalarFunction to a Python object is not implemented: {:?}",
-                value
+                "Converting Expr::ScalarFunction to a Python object is not implemented: {value:?}"
             ))),
             Expr::WindowFunction(value) => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::WindowFunction to a Python object is not implemented: {:?}",
-                value
+                "Converting Expr::WindowFunction to a Python object is not implemented: {value:?}"
             ))),
             Expr::InList(value) => Ok(in_list::PyInList::from(value.clone()).into_bound_py_any(py)?),
             Expr::Exists(value) => Ok(exists::PyExists::from(value.clone()).into_bound_py_any(py)?),
@@ -188,8 +186,7 @@ impl PyExpr {
             }
             #[allow(deprecated)]
             Expr::Wildcard { qualifier, options } => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::Wildcard to a Python object is not implemented : {:?} {:?}",
-                qualifier, options
+                "Converting Expr::Wildcard to a Python object is not implemented : {qualifier:?} {options:?}"
             ))),
             Expr::GroupingSet(value) => {
                 Ok(grouping_set::PyGroupingSet::from(value.clone()).into_bound_py_any(py)?)
@@ -198,8 +195,7 @@ impl PyExpr {
                 Ok(placeholder::PyPlaceholder::from(value.clone()).into_bound_py_any(py)?)
             }
             Expr::OuterReferenceColumn(data_type, column) => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::OuterReferenceColumn to a Python object is not implemented: {:?} - {:?}",
-                data_type, column
+                "Converting Expr::OuterReferenceColumn to a Python object is not implemented: {data_type:?} - {column:?}"
             ))),
             Expr::Unnest(value) => Ok(unnest_expr::PyUnnestExpr::from(value.clone()).into_bound_py_any(py)?),
         }
@@ -755,8 +751,7 @@ impl PyExpr {
             Expr::Cast(Cast { expr: _, data_type }) => DataTypeMap::map_from_arrow_type(data_type),
             Expr::Literal(scalar_value, _) => DataTypeMap::map_from_scalar_value(scalar_value),
             _ => Err(py_type_err(format!(
-                "Non Expr::Literal encountered in types: {:?}",
-                expr
+                "Non Expr::Literal encountered in types: {expr:?}"
             ))),
         }
     }
diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs
index a99d83d2..fd439327 100644
--- a/src/expr/aggregate.rs
+++ b/src/expr/aggregate.rs
@@ -116,7 +116,7 @@ impl PyAggregate {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Aggregate({})", self))
+        Ok(format!("Aggregate({self})"))
     }
 }
 
diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs
index c09f116e..7c5d3d31 100644
--- a/src/expr/aggregate_expr.rs
+++ b/src/expr/aggregate_expr.rs
@@ -75,6 +75,6 @@ impl PyAggregateFunction {
 
     /// Get a String representation of this column
     fn __repr__(&self) -> String {
-        format!("{}", self)
+        format!("{self}")
     }
 }
diff --git a/src/expr/alias.rs b/src/expr/alias.rs
index e8e03cfa..40746f20 100644
--- a/src/expr/alias.rs
+++ b/src/expr/alias.rs
@@ -64,6 +64,6 @@ impl PyAlias {
 
     /// Get a String representation of this column
     fn __repr__(&self) -> String {
-        format!("{}", self)
+        format!("{self}")
     }
 }
diff --git a/src/expr/analyze.rs b/src/expr/analyze.rs
index 62f93cd2..e8081e95 100644
--- a/src/expr/analyze.rs
+++ b/src/expr/analyze.rs
@@ -69,7 +69,7 @@ impl PyAnalyze {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Analyze({})", self))
+        Ok(format!("Analyze({self})"))
     }
 }
 
diff --git a/src/expr/between.rs b/src/expr/between.rs
index a2cac144..817f1baa 100644
--- a/src/expr/between.rs
+++ b/src/expr/between.rs
@@ -71,6 +71,6 @@ impl PyBetween {
     }
 
     fn __repr__(&self) -> String {
-        format!("{}", self)
+        format!("{self}")
     }
 }
diff --git a/src/expr/column.rs b/src/expr/column.rs
index 365dbc0d..50f316f1 100644
--- a/src/expr/column.rs
+++ b/src/expr/column.rs
@@ -45,7 +45,7 @@ impl PyColumn {
 
     /// Get the column relation
     fn relation(&self) -> Option<String> {
-        self.col.relation.as_ref().map(|r| format!("{}", r))
+        self.col.relation.as_ref().map(|r| format!("{r}"))
     }
 
     /// Get the fully-qualified column name
diff --git a/src/expr/copy_to.rs b/src/expr/copy_to.rs
index ebfcb8eb..473dabfe 100644
--- a/src/expr/copy_to.rs
+++ b/src/expr/copy_to.rs
@@ -106,7 +106,7 @@ impl PyCopyTo {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CopyTo({})", self))
+        Ok(format!("CopyTo({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
@@ -129,7 +129,7 @@ impl Display for PyFileType {
 #[pymethods]
 impl PyFileType {
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("FileType({})", self))
+        Ok(format!("FileType({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_catalog.rs b/src/expr/create_catalog.rs
index f4ea0f51..d2d2ee8f 100644
--- a/src/expr/create_catalog.rs
+++ b/src/expr/create_catalog.rs
@@ -81,7 +81,7 @@ impl PyCreateCatalog {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateCatalog({})", self))
+        Ok(format!("CreateCatalog({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_catalog_schema.rs b/src/expr/create_catalog_schema.rs
index 85f447e1..e794962f 100644
--- a/src/expr/create_catalog_schema.rs
+++ b/src/expr/create_catalog_schema.rs
@@ -81,7 +81,7 @@ impl PyCreateCatalogSchema {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateCatalogSchema({})", self))
+        Ok(format!("CreateCatalogSchema({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_external_table.rs b/src/expr/create_external_table.rs
index 01ce7d0c..3e35af00 100644
--- a/src/expr/create_external_table.rs
+++ b/src/expr/create_external_table.rs
@@ -164,7 +164,7 @@ impl PyCreateExternalTable {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateExternalTable({})", self))
+        Ok(format!("CreateExternalTable({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_function.rs b/src/expr/create_function.rs
index 6f3c3f0f..c02ceebb 100644
--- a/src/expr/create_function.rs
+++ b/src/expr/create_function.rs
@@ -163,7 +163,7 @@ impl PyCreateFunction {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateFunction({})", self))
+        Ok(format!("CreateFunction({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_index.rs b/src/expr/create_index.rs
index 13dadbc3..0f4b5011 100644
--- a/src/expr/create_index.rs
+++ b/src/expr/create_index.rs
@@ -110,7 +110,7 @@ impl PyCreateIndex {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateIndex({})", self))
+        Ok(format!("CreateIndex({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_memory_table.rs b/src/expr/create_memory_table.rs
index 8872b2d4..37f4d342 100644
--- a/src/expr/create_memory_table.rs
+++ b/src/expr/create_memory_table.rs
@@ -78,7 +78,7 @@ impl PyCreateMemoryTable {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateMemoryTable({})", self))
+        Ok(format!("CreateMemoryTable({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/create_view.rs b/src/expr/create_view.rs
index 87bb7687..718e404d 100644
--- a/src/expr/create_view.rs
+++ b/src/expr/create_view.rs
@@ -75,7 +75,7 @@ impl PyCreateView {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("CreateView({})", self))
+        Ok(format!("CreateView({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/describe_table.rs b/src/expr/describe_table.rs
index 5658a13f..6c48f3c7 100644
--- a/src/expr/describe_table.rs
+++ b/src/expr/describe_table.rs
@@ -61,7 +61,7 @@ impl PyDescribeTable {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("DescribeTable({})", self))
+        Ok(format!("DescribeTable({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs
index b62b776f..889e7099 100644
--- a/src/expr/distinct.rs
+++ b/src/expr/distinct.rs
@@ -48,8 +48,7 @@ impl Display for PyDistinct {
             Distinct::All(input) => write!(
                 f,
                 "Distinct ALL
-            \nInput: {:?}",
-                input,
+            \nInput: {input:?}",
             ),
             Distinct::On(distinct_on) => {
                 write!(
@@ -71,7 +70,7 @@ impl PyDistinct {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Distinct({})", self))
+        Ok(format!("Distinct({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/drop_catalog_schema.rs b/src/expr/drop_catalog_schema.rs
index b7420a99..b4a4c521 100644
--- a/src/expr/drop_catalog_schema.rs
+++ b/src/expr/drop_catalog_schema.rs
@@ -101,7 +101,7 @@ impl PyDropCatalogSchema {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("DropCatalogSchema({})", self))
+        Ok(format!("DropCatalogSchema({self})"))
     }
 }
 
diff --git a/src/expr/drop_function.rs b/src/expr/drop_function.rs
index 9fbd78fd..fca9eb94 100644
--- a/src/expr/drop_function.rs
+++ b/src/expr/drop_function.rs
@@ -76,7 +76,7 @@ impl PyDropFunction {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("DropFunction({})", self))
+        Ok(format!("DropFunction({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/drop_table.rs b/src/expr/drop_table.rs
index 96983c1c..3f442539 100644
--- a/src/expr/drop_table.rs
+++ b/src/expr/drop_table.rs
@@ -70,7 +70,7 @@ impl PyDropTable {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("DropTable({})", self))
+        Ok(format!("DropTable({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/drop_view.rs b/src/expr/drop_view.rs
index 1d1ab1e5..6196c8bb 100644
--- a/src/expr/drop_view.rs
+++ b/src/expr/drop_view.rs
@@ -83,7 +83,7 @@ impl PyDropView {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("DropView({})", self))
+        Ok(format!("DropView({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/empty_relation.rs b/src/expr/empty_relation.rs
index a1534ac1..75821342 100644
--- a/src/expr/empty_relation.rs
+++ b/src/expr/empty_relation.rs
@@ -65,7 +65,7 @@ impl PyEmptyRelation {
 
     /// Get a String representation of this column
     fn __repr__(&self) -> String {
-        format!("{}", self)
+        format!("{self}")
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/filter.rs b/src/expr/filter.rs
index 9bdb667c..4fcb600c 100644
--- a/src/expr/filter.rs
+++ b/src/expr/filter.rs
@@ -72,7 +72,7 @@ impl PyFilter {
     }
 
     fn __repr__(&self) -> String {
-        format!("Filter({})", self)
+        format!("Filter({self})")
     }
 }
 
diff --git a/src/expr/join.rs b/src/expr/join.rs
index 76ec532e..b8d1d9da 100644
--- a/src/expr/join.rs
+++ b/src/expr/join.rs
@@ -177,7 +177,7 @@ impl PyJoin {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Join({})", self))
+        Ok(format!("Join({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/like.rs b/src/expr/like.rs
index 2e1f060b..f180f5d4 100644
--- a/src/expr/like.rs
+++ b/src/expr/like.rs
@@ -75,7 +75,7 @@ impl PyLike {
     }
 
     fn __repr__(&self) -> String {
-        format!("Like({})", self)
+        format!("Like({self})")
     }
 }
 
@@ -133,7 +133,7 @@ impl PyILike {
     }
 
     fn __repr__(&self) -> String {
-        format!("Like({})", self)
+        format!("Like({self})")
     }
 }
 
@@ -191,6 +191,6 @@ impl PySimilarTo {
     }
 
     fn __repr__(&self) -> String {
-        format!("Like({})", self)
+        format!("Like({self})")
     }
 }
diff --git a/src/expr/limit.rs b/src/expr/limit.rs
index c2a33ff8..92552814 100644
--- a/src/expr/limit.rs
+++ b/src/expr/limit.rs
@@ -81,7 +81,7 @@ impl PyLimit {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Limit({})", self))
+        Ok(format!("Limit({self})"))
     }
 }
 
diff --git a/src/expr/projection.rs b/src/expr/projection.rs
index dc7e5e3c..b5a9ef34 100644
--- a/src/expr/projection.rs
+++ b/src/expr/projection.rs
@@ -85,7 +85,7 @@ impl PyProjection {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Projection({})", self))
+        Ok(format!("Projection({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/recursive_query.rs b/src/expr/recursive_query.rs
index 65181f7d..2517b741 100644
--- a/src/expr/recursive_query.rs
+++ b/src/expr/recursive_query.rs
@@ -89,7 +89,7 @@ impl PyRecursiveQuery {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("RecursiveQuery({})", self))
+        Ok(format!("RecursiveQuery({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/repartition.rs b/src/expr/repartition.rs
index 3e782d6a..48b5e704 100644
--- a/src/expr/repartition.rs
+++ b/src/expr/repartition.rs
@@ -108,7 +108,7 @@ impl PyRepartition {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Repartition({})", self))
+        Ok(format!("Repartition({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/sort.rs b/src/expr/sort.rs
index ed494759..79a8aee5 100644
--- a/src/expr/sort.rs
+++ b/src/expr/sort.rs
@@ -87,7 +87,7 @@ impl PySort {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Sort({})", self))
+        Ok(format!("Sort({self})"))
     }
 }
 
diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs
index 12f74e4d..79e35d97 100644
--- a/src/expr/sort_expr.rs
+++ b/src/expr/sort_expr.rs
@@ -85,6 +85,6 @@ impl PySortExpr {
     }
 
     fn __repr__(&self) -> String {
-        format!("{}", self)
+        format!("{self}")
     }
 }
diff --git a/src/expr/subquery.rs b/src/expr/subquery.rs
index 5ebfe692..77f56f9a 100644
--- a/src/expr/subquery.rs
+++ b/src/expr/subquery.rs
@@ -62,7 +62,7 @@ impl PySubquery {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Subquery({})", self))
+        Ok(format!("Subquery({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/subquery_alias.rs b/src/expr/subquery_alias.rs
index 267a4d48..3302e7f2 100644
--- a/src/expr/subquery_alias.rs
+++ b/src/expr/subquery_alias.rs
@@ -72,7 +72,7 @@ impl PySubqueryAlias {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("SubqueryAlias({})", self))
+        Ok(format!("SubqueryAlias({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs
index 6a0d53f0..32996468 100644
--- a/src/expr/table_scan.rs
+++ b/src/expr/table_scan.rs
@@ -136,7 +136,7 @@ impl PyTableScan {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("TableScan({})", self))
+        Ok(format!("TableScan({self})"))
     }
 }
 
diff --git a/src/expr/union.rs b/src/expr/union.rs
index 5a08ccc1..e0b22139 100644
--- a/src/expr/union.rs
+++ b/src/expr/union.rs
@@ -66,7 +66,7 @@ impl PyUnion {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Union({})", self))
+        Ok(format!("Union({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/unnest.rs b/src/expr/unnest.rs
index 8e70e099..c8833347 100644
--- a/src/expr/unnest.rs
+++ b/src/expr/unnest.rs
@@ -66,7 +66,7 @@ impl PyUnnest {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("Unnest({})", self))
+        Ok(format!("Unnest({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/unnest_expr.rs b/src/expr/unnest_expr.rs
index 2234d24b..634186ed 100644
--- a/src/expr/unnest_expr.rs
+++ b/src/expr/unnest_expr.rs
@@ -58,7 +58,7 @@ impl PyUnnestExpr {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("UnnestExpr({})", self))
+        Ok(format!("UnnestExpr({self})"))
     }
 
     fn __name__(&self) -> PyResult<String> {
diff --git a/src/expr/window.rs b/src/expr/window.rs
index 052d9eeb..a408731c 100644
--- a/src/expr/window.rs
+++ b/src/expr/window.rs
@@ -185,8 +185,7 @@ impl PyWindowFrame {
             "groups" => WindowFrameUnits::Groups,
             _ => {
                 return Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                    "{:?}",
-                    units,
+                    "{units:?}",
                 ))));
             }
         };
@@ -197,8 +196,7 @@ impl PyWindowFrame {
                 WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)),
                 WindowFrameUnits::Groups => {
                     return Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                        "{:?}",
-                        units,
+                        "{units:?}",
                     ))));
                 }
             },
@@ -210,8 +208,7 @@ impl PyWindowFrame {
                 WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)),
                 WindowFrameUnits::Groups => {
                     return Err(py_datafusion_err(DataFusionError::NotImplemented(format!(
-                        "{:?}",
-                        units,
+                        "{units:?}",
                     ))));
                 }
             },
@@ -236,7 +233,7 @@ impl PyWindowFrame {
 
     /// Get a String representation of this window frame
     fn __repr__(&self) -> String {
-        format!("{}", self)
+        format!("{self}")
     }
 }
 
diff --git a/src/functions.rs b/src/functions.rs
index b2bafcb6..b40500b8 100644
--- a/src/functions.rs
+++ b/src/functions.rs
@@ -937,7 +937,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(left))?;
     m.add_wrapped(wrap_pyfunction!(length))?;
     m.add_wrapped(wrap_pyfunction!(ln))?;
-    m.add_wrapped(wrap_pyfunction!(log))?;
+    m.add_wrapped(wrap_pyfunction!(self::log))?;
     m.add_wrapped(wrap_pyfunction!(log10))?;
     m.add_wrapped(wrap_pyfunction!(log2))?;
     m.add_wrapped(wrap_pyfunction!(lower))?;
diff --git a/src/lib.rs b/src/lib.rs
index 1293eee3..29d3f41d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -77,10 +77,10 @@ pub(crate) struct TokioRuntime(tokio::runtime::Runtime);
 /// datafusion directory.
 #[pymodule]
 fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
+    // Initialize logging
+    pyo3_log::init();
+
     // Register the python classes
-    m.add_class::<catalog::PyCatalog>()?;
-    m.add_class::<catalog::PyDatabase>()?;
-    m.add_class::<catalog::PyTable>()?;
     m.add_class::<context::PyRuntimeEnvBuilder>()?;
     m.add_class::<context::PySessionConfig>()?;
     m.add_class::<context::PySessionContext>()?;
@@ -98,6 +98,10 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<record_batch::PyRecordBatch>()?;
     m.add_class::<record_batch::PyRecordBatchStream>()?;
 
+    let catalog = PyModule::new(py, "catalog")?;
+    catalog::init_module(&catalog)?;
+    m.add_submodule(&catalog)?;
+
     // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/
     let common = PyModule::new(py, "common")?;
     common::init_module(&common)?;
diff --git a/src/physical_plan.rs b/src/physical_plan.rs
index f0be45c6..49db643e 100644
--- a/src/physical_plan.rs
+++ b/src/physical_plan.rs
@@ -78,8 +78,7 @@ impl PyExecutionPlan {
         let proto_plan =
             datafusion_proto::protobuf::PhysicalPlanNode::decode(bytes).map_err(|e| {
                 PyRuntimeError::new_err(format!(
-                    "Unable to decode logical node from serialized bytes: {}",
-                    e
+                    "Unable to decode logical node from serialized bytes: {e}"
                 ))
             })?;
 
diff --git a/src/sql/logical.rs b/src/sql/logical.rs
index 198d68bd..97d32047 100644
--- a/src/sql/logical.rs
+++ b/src/sql/logical.rs
@@ -201,8 +201,7 @@ impl PyLogicalPlan {
         let proto_plan =
             datafusion_proto::protobuf::LogicalPlanNode::decode(bytes).map_err(|e| {
                 PyRuntimeError::new_err(format!(
-                    "Unable to decode logical node from serialized bytes: {}",
-                    e
+                    "Unable to decode logical node from serialized bytes: {e}"
                 ))
             })?;
 
diff --git a/src/utils.rs b/src/utils.rs
index f4e121fd..3b30de5d 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -109,8 +109,7 @@ pub(crate) fn validate_pycapsule(capsule: &Bound<PyCapsule>, name: &str) -> PyRe
     let capsule_name = capsule_name.unwrap().to_str()?;
     if capsule_name != name {
         return Err(PyValueError::new_err(format!(
-            "Expected name '{}' in PyCapsule, instead got '{}'",
-            name, capsule_name
+            "Expected name '{name}' in PyCapsule, instead got '{capsule_name}'"
         )));
     }
 
@@ -127,7 +126,7 @@ pub(crate) fn py_obj_to_scalar_value(py: Python, obj: PyObject) -> PyResult<Scal
 
     // Convert PyArrow scalar to PyScalarValue
     let py_scalar = PyScalarValue::extract_bound(scalar.as_ref())
-        .map_err(|e| PyValueError::new_err(format!("Failed to extract PyScalarValue: {}", e)))?;
+        .map_err(|e| PyValueError::new_err(format!("Failed to extract PyScalarValue: {e}")))?;
 
     // Convert PyScalarValue to ScalarValue
     Ok(py_scalar.into())


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org