You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ho...@apache.org on 2024/03/05 09:02:39 UTC
(iceberg-python) branch main updated: add support for glue.id (#490)
This is an automated email from the ASF dual-hosted git repository.
honahx pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new be815299 add support for glue.id (#490)
be815299 is described below
commit be8152994efc148021d6c6d5bca150e97c1676c0
Author: Michel Rouly <mi...@rouly.net>
AuthorDate: Tue Mar 5 04:02:33 2024 -0500
add support for glue.id (#490)
---
pyiceberg/catalog/glue.py | 26 ++++++++++++++++++++++++++
tests/catalog/test_glue.py | 21 +++++++++++++++++++++
2 files changed, 47 insertions(+)
diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py
index 089a30ba..bd902cd1 100644
--- a/pyiceberg/catalog/glue.py
+++ b/pyiceberg/catalog/glue.py
@@ -93,6 +93,13 @@ from pyiceberg.types import (
if TYPE_CHECKING:
import pyarrow as pa
+
+# There is a unique Glue metastore in each AWS account and each AWS region. By default, GlueCatalog chooses the Glue
+# metastore to use based on the user's default AWS client credential and region setup. You can specify the Glue catalog
+# ID through glue.id catalog property to point to a Glue catalog in a different AWS account. The Glue catalog ID is your
+# numeric AWS account ID.
+GLUE_ID = "glue.id"
+
# If Glue should skip archiving an old table version when creating a new version in a commit. By
# default, Glue archives all old table versions after an UpdateTable call, but Glue has a default
# max number of archived table versions (can be increased). So for streaming use case with lots
@@ -252,6 +259,22 @@ def _construct_database_input(database_name: str, properties: Properties) -> Dat
return database_input
+def _register_glue_catalog_id_with_glue_client(glue: GlueClient, glue_catalog_id: str) -> None:
+ """
+ Register the Glue Catalog ID (AWS Account ID) as a parameter on all Glue client methods.
+
+ It's more ergonomic to do this than to pass the CatalogId as a parameter to every client call since it's an optional
+ parameter and boto3 does not support 'None' values for missing parameters.
+ """
+ event_system = glue.meta.events
+
+ def add_glue_catalog_id(params: Dict[str, str], **kwargs: Any) -> None:
+ if "CatalogId" not in params:
+ params["CatalogId"] = glue_catalog_id
+
+ event_system.register("provide-client-params.glue", add_glue_catalog_id)
+
+
class GlueCatalog(Catalog):
def __init__(self, name: str, **properties: Any):
super().__init__(name, **properties)
@@ -266,6 +289,9 @@ class GlueCatalog(Catalog):
)
self.glue: GlueClient = session.client("glue")
+ if glue_catalog_id := properties.get(GLUE_ID):
+ _register_glue_catalog_id_with_glue_client(self.glue, glue_catalog_id)
+
def _convert_glue_to_iceberg(self, glue_table: TableTypeDef) -> Table:
properties: Properties = glue_table["Parameters"]
diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py
index 6e0196c1..6d44d927 100644
--- a/tests/catalog/test_glue.py
+++ b/tests/catalog/test_glue.py
@@ -206,6 +206,27 @@ def test_create_table_with_no_database(
test_catalog.create_table(identifier=identifier, schema=table_schema_nested)
+@mock_aws
+def test_create_table_with_glue_catalog_id(
+ _bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str
+) -> None:
+ catalog_name = "glue"
+ catalog_id = "444444444444"
+ identifier = (database_name, table_name)
+ test_catalog = GlueCatalog(
+ catalog_name, **{"s3.endpoint": moto_endpoint_url, "warehouse": f"s3://{BUCKET_NAME}", "glue.id": catalog_id}
+ )
+ test_catalog.create_namespace(namespace=database_name)
+ table = test_catalog.create_table(identifier, table_schema_nested)
+ assert table.identifier == (catalog_name,) + identifier
+ assert TABLE_METADATA_LOCATION_REGEX.match(table.metadata_location)
+ assert test_catalog._parse_metadata_version(table.metadata_location) == 0
+
+ glue = boto3.client("glue")
+ databases = glue.get_databases()
+ assert databases["DatabaseList"][0]["CatalogId"] == catalog_id
+
+
@mock_aws
def test_create_duplicated_table(
_bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str