You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2023/10/15 13:07:34 UTC

[iceberg-python] 05/10: Fix the TableIdentifier (#44)

This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch pyiceberg-0.5.x
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git

commit 724e3911a67ed31e7c24d53c9bed01299b796b4b
Author: Fokko Driesprong <fo...@tabular.io>
AuthorDate: Sat Oct 7 15:16:51 2023 +0200

    Fix the TableIdentifier (#44)
    
    * Fix the TableIdentifier
    
    * Make the rest of the tests happy
    
    * Add default properties
---
 pyiceberg/catalog/rest.py   |  6 +++++-
 pyiceberg/table/__init__.py | 26 ++++++++++++++++++++++++--
 tests/catalog/test_base.py  |  8 +++++---
 tests/test_integration.py   | 12 +++++++-----
 4 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py
index 0023e18..ad2c510 100644
--- a/pyiceberg/catalog/rest.py
+++ b/pyiceberg/catalog/rest.py
@@ -62,6 +62,7 @@ from pyiceberg.table import (
     CommitTableRequest,
     CommitTableResponse,
     Table,
+    TableIdentifier,
     TableMetadata,
 )
 from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder
@@ -301,7 +302,10 @@ class RestCatalog(Catalog):
         # Update URI based on overrides
         self.uri = config[URI]
 
-    def _split_identifier_for_path(self, identifier: Union[str, Identifier]) -> Properties:
+    def _split_identifier_for_path(self, identifier: Union[str, Identifier, TableIdentifier]) -> Properties:
+        if isinstance(identifier, TableIdentifier):
+            return {"namespace": NAMESPACE_SEPARATOR.join(identifier.namespace.root[1:]), "table": identifier.name}
+
         identifier_tuple = self.identifier_to_tuple(identifier)
         if len(identifier_tuple) <= 1:
             raise NoSuchTableError(f"Missing namespace or invalid identifier: {'.'.join(identifier_tuple)}")
diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index b905c95..9f77dff 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -74,6 +74,7 @@ from pyiceberg.table.sorting import SortOrder
 from pyiceberg.typedef import (
     EMPTY_DICT,
     IcebergBaseModel,
+    IcebergRootModel,
     Identifier,
     KeyDefaultDict,
     Properties,
@@ -402,8 +403,25 @@ class AssertDefaultSortOrderId(TableRequirement):
     default_sort_order_id: int = Field(..., alias="default-sort-order-id")
 
 
+class Namespace(IcebergRootModel[List[str]]):
+    """Reference to one or more levels of a namespace."""
+
+    root: List[str] = Field(
+        ...,
+        description='Reference to one or more levels of a namespace',
+        example=['accounting', 'tax'],
+    )
+
+
+class TableIdentifier(IcebergBaseModel):
+    """Fully Qualified identifier to a table."""
+
+    namespace: Namespace
+    name: str
+
+
 class CommitTableRequest(IcebergBaseModel):
-    identifier: Identifier = Field()
+    identifier: TableIdentifier = Field()
     requirements: Tuple[SerializeAsAny[TableRequirement], ...] = Field(default_factory=tuple)
     updates: Tuple[SerializeAsAny[TableUpdate], ...] = Field(default_factory=tuple)
 
@@ -526,7 +544,11 @@ class Table:
 
     def _do_commit(self, updates: Tuple[TableUpdate, ...], requirements: Tuple[TableRequirement, ...]) -> None:
         response = self.catalog._commit_table(  # pylint: disable=W0212
-            CommitTableRequest(identifier=self.identifier[1:], updates=updates, requirements=requirements)
+            CommitTableRequest(
+                identifier=TableIdentifier(namespace=self.identifier[:-1], name=self.identifier[-1]),
+                updates=updates,
+                requirements=requirements,
+            )
         )  # pylint: disable=W0212
         self.metadata = response.metadata
         self.metadata_location = response.metadata_location
diff --git a/tests/catalog/test_base.py b/tests/catalog/test_base.py
index d1d7d63..fbec2cb 100644
--- a/tests/catalog/test_base.py
+++ b/tests/catalog/test_base.py
@@ -46,8 +46,10 @@ from pyiceberg.table import (
     AddSchemaUpdate,
     CommitTableRequest,
     CommitTableResponse,
+    Namespace,
     SetCurrentSchemaUpdate,
     Table,
+    TableIdentifier,
 )
 from pyiceberg.table.metadata import TableMetadata, TableMetadataV1, new_table_metadata
 from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder
@@ -119,8 +121,8 @@ class InMemoryCatalog(Catalog):
         for update in table_request.updates:
             if isinstance(update, AddSchemaUpdate):
                 add_schema_update: AddSchemaUpdate = update
-                identifier = Catalog.identifier_to_tuple(table_request.identifier)
-                table = self.__tables[("com", *identifier)]
+                identifier = tuple(table_request.identifier.namespace.root) + (table_request.identifier.name,)
+                table = self.__tables[identifier]
                 new_metadata = new_table_metadata(
                     add_schema_update.schema_,
                     table.metadata.partition_specs[0],
@@ -528,7 +530,7 @@ def test_commit_table(catalog: InMemoryCatalog) -> None:
     # When
     response = given_table.catalog._commit_table(  # pylint: disable=W0212
         CommitTableRequest(
-            identifier=given_table.identifier[1:],
+            identifier=TableIdentifier(namespace=Namespace(given_table.identifier[:-1]), name=given_table.identifier[-1]),
             updates=[
                 AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
                 SetCurrentSchemaUpdate(schema_id=-1),
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 297749b..6d777cf 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -46,6 +46,8 @@ from pyiceberg.types import (
     TimestampType,
 )
 
+DEFAULT_PROPERTIES = {'write.parquet.compression-codec': 'zstd'}
+
 
 @pytest.fixture()
 def catalog() -> Catalog:
@@ -104,25 +106,25 @@ def table(catalog: Catalog) -> Table:
 
 @pytest.mark.integration
 def test_table_properties(table: Table) -> None:
-    assert table.properties == {}
+    assert table.properties == DEFAULT_PROPERTIES
 
     with table.transaction() as transaction:
         transaction.set_properties(abc="🤪")
 
-    assert table.properties == {"abc": "🤪"}
+    assert table.properties == dict(**{"abc": "🤪"}, **DEFAULT_PROPERTIES)
 
     with table.transaction() as transaction:
         transaction.remove_properties("abc")
 
-    assert table.properties == {}
+    assert table.properties == DEFAULT_PROPERTIES
 
     table = table.transaction().set_properties(abc="def").commit_transaction()
 
-    assert table.properties == {"abc": "def"}
+    assert table.properties == dict(**{"abc": "def"}, **DEFAULT_PROPERTIES)
 
     table = table.transaction().remove_properties("abc").commit_transaction()
 
-    assert table.properties == {}
+    assert table.properties == DEFAULT_PROPERTIES
 
 
 @pytest.fixture()