You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "Fokko (via GitHub)" <gi...@apache.org> on 2023/08/22 18:54:57 UTC
[GitHub] [iceberg] Fokko opened a new pull request, #8374: Python: Follow up on update schema
Fokko opened a new pull request, #8374:
URL: https://github.com/apache/iceberg/pull/8374
(no comment)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305005295
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
Review Comment:
This could be smaller if you used an `_internal_move(self, path, reference, op)` method.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#issuecomment-1703531967
Awesome work, @Fokko! This is really close.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313617130
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
Review Comment:
Agreed. I guess it's established already though.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314246799
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
Review Comment:
I'm missing the point here :(
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313614871
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
+ tbl.metadata_location,
+ properties={
+ "s3.endpoint": "http://127.0.0.1:9000",
Review Comment:
Probably not needed
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
Review Comment:
`s3a`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314245215
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
Review Comment:
Good one, thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304798205
##########
python/pyiceberg/table/__init__.py:
##########
@@ -527,8 +519,13 @@ def history(self) -> List[SnapshotLogEntry]:
"""Get the snapshot history of this table."""
return self.metadata.snapshot_log
- def update_schema(self) -> UpdateSchema:
- return UpdateSchema(self.schema(), self)
+ def update_schema(self, allow_incompatible_changes: bool = False, case_sensitive: bool = True) -> UpdateSchema:
+ return UpdateSchema(self, allow_incompatible_changes=allow_incompatible_changes, case_sensitive=case_sensitive)
+
+ def _do_commit(self, request: CommitTableRequest) -> None:
Review Comment:
I think it would make more sense to pass requirements and updates. The identifier is already in the table and this can create the request. Or maybe the request should be created in `catalog._commit_table`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306714287
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
Review Comment:
My IDEA sets this:
<img width="592" alt="image" src="https://github.com/apache/iceberg/assets/1134248/8c0886df-3b87-436b-aa1c-2a2ca5ddc47c">
But I agree that `kwargs > args`, thanks for pointing this out
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313641169
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
Review Comment:
duplication?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306718688
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
Review Comment:
It looks like we also need this in the exceptions, but I don't think we need the paths. Cleaned this up a bit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306717284
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
Review Comment:
I just copied it from Java without any modifications, but I like it!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314249767
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
Review Comment:
For me it adds another layer of testing since the REST catalog will also reject certain things (for example, the requirements on the identifier fields), this caught already a few bugs. I also like to follow the same path that the end user would take
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313643901
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
Review Comment:
Changing the scale of a decimal is not an allowed promotion. I think if you remove `DecimalType(18,2)` this should pass right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313638947
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
Review Comment:
Why are all of these tests done in integration tests? In Java, we test the schema update API by calling `apply` to produce the schema that will be committed. Only the commit needs to be done in an integration test that way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1315084709
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
Review Comment:
Okay, I get it now. Seems alright, although the REST catalog really doesn't check much. It would probably be good to move most of the schema evolution behavior tests to unit tests, but the important thing is that they're happening somewhere.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314244440
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
+ tbl.metadata_location,
+ properties={
+ "s3.endpoint": "http://127.0.0.1:9000",
Review Comment:
This was for running the docs as a test, this would then connect to the minio instance :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314244546
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
Review Comment:
Historical reasons: https://github.com/tabular-io/docker-spark-iceberg/pull/91
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314246858
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
Review Comment:
This move is on the top level indeed, added a comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] danielcweeks closed pull request #8374: Python: Add updates, moves and deletes
Posted by "danielcweeks (via GitHub)" <gi...@apache.org>.
danielcweeks closed pull request #8374: Python: Add updates, moves and deletes
URL: https://github.com/apache/iceberg/pull/8374
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Follow up on update schema
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1302507284
##########
python/mkdocs/docs/api.md:
##########
@@ -150,32 +150,25 @@ catalog.create_table(
Add new columns through the `Transaction` or `UpdateSchema` API:
-Use the Transaction API:
-
```python
-with table.transaction() as transaction:
- transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
+with table.update_schema() as update:
+ update.add_column("x", IntegerType(), "doc")
```
-Or, without a context manager:
+Or, without a context manager by calling the `.commit()` explicitly:
```python
-transaction = table.transaction()
-transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
-transaction.commit_transaction()
+table.update_schema().add_column("x", IntegerType(), "doc").commit()
```
-Or, use the UpdateSchema API directly:
+Use the Transaction API:
Review Comment:
```suggestion
Alternatively, use the transaction API to combine changes from multiple operations:
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305000187
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
Review Comment:
Ah, I see why you wanted to parse `name_from`. The problem is that we can never parse full names using `split` because we don't know whether `.` is a delimiter or a character in a name. In this case, you should look up the parent's name by finding the parent's ID and looking it up in the ID to name index.
By the way, I think this fixes a problem in the Java implementation. The `newName` is a leaf name, not a full name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305008713
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
+ """Allow incompatible changes to the schema.
+
+ Returns:
+ This for method chaining
+ """
+ self._allow_incompatible_changes = True
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._schema.highest_field_id, new_schema.highest_field_id)
+ updates = [
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ ]
+ requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ else:
+ self._table._do_commit( # pylint: disable=W0212
+ CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ )
+
+ def _apply(self) -> Schema:
+ """Apply the pending changes to the original schema and returns the result.
+
+ Returns:
+ the result Schema when all pending updates are applied
+ """
+ struct = visit(self._schema, _ApplyChanges(self._adds, self._updates, self._deletes, self._moves))
+ if struct is None:
+ # Should never happen
+ raise ValueError("Could not apply changes")
+
+ # @TODO: This differs still a bit from the Java side,
+ # The validate identifier field is missing
+ field_ids = set()
+ for name in self._identifier_field_names:
+ field = self._schema.find_field(name, self._case_sensitive)
+ field_ids.add(field.field_id)
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot delete identifier field {name}. To force deletion, update the identifier fields first.")
+
+ # If it nested, also check if the parents aren't deleted
+ column_name = self._id_to_parent.get(field.field_id)
+ while column_name is not None:
+ parent = self._schema.find_field(column_name)
+ if parent.field_id in self._deletes:
+ raise ValueError(
+ f"Cannot delete field {parent.field_id} as it will delete nested identifier field {name}",
+ )
+ column_name = self._id_to_parent.get(parent.field_id)
+
+ return Schema(*struct.fields, identifier_field_ids=field_ids)
def assign_new_column_id(self) -> int:
return next(self._last_column_id)
-def _apply_changes(schema_: Schema, adds: Dict[int, List[NestedField]], identifier_field_names: List[str]) -> Schema:
- struct = visit(schema_, _ApplyChanges(adds))
- name_to_id: Dict[str, int] = index_by_name(struct)
- for name in identifier_field_names:
- if name not in name_to_id:
- raise ValueError(f"Cannot add field {name} as an identifier field: not found in current schema or added columns")
-
- return Schema(*struct.fields)
+class _ApplyChanges(SchemaVisitor[Optional[IcebergType]]):
+ _adds: Dict[int, List[NestedField]]
+ _updates: Dict[int, NestedField]
+ _deletes: Set[int]
+ _moves: Dict[int, List[Move]]
+ def __init__(
+ self, adds: Dict[int, List[NestedField]], updates: Dict[int, NestedField], deletes: Set[int], moves: Dict[int, List[Move]]
+ ) -> None:
+ self._adds = adds
+ self._updates = updates
+ self._deletes = deletes
+ self._moves = moves
-class _ApplyChanges(SchemaVisitor[IcebergType]):
- def __init__(self, adds: Dict[int, List[NestedField]]):
- self.adds = adds
+ def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ if new_fields := _add_fields(struct_result.fields if struct_result else [], self._adds.get(TABLE_ROOT_ID)):
+ return StructType(*new_fields)
+ else:
+ return struct_result
- def schema(self, schema: Schema, struct_result: IcebergType) -> IcebergType:
- fields = _ApplyChanges.add_fields(schema.as_struct().fields, self.adds.get(TABLE_ROOT_ID))
- if len(fields) > 0:
- return StructType(*fields)
+ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]:
+ has_changes = False
+ new_fields = []
- return struct_result
+ for idx, result_type in enumerate(field_results):
+ result_type = field_results[idx]
- def struct(self, struct: StructType, field_results: List[IcebergType]) -> IcebergType:
- has_change = False
- new_fields: List[NestedField] = []
- for i in range(len(field_results)):
- type_: Optional[IcebergType] = field_results[i]
- if type_ is None:
- has_change = True
+ # Has been deleted
+ if result_type is None:
+ has_changes = True
continue
- field: NestedField = struct.fields[i]
- new_fields.append(field)
+ field = struct.fields[idx]
+
+ name = field.name
+ doc = field.doc
+ required = field.required
- if has_change:
+ # There is an update
+ if update := self._updates.get(field.field_id):
+ name = update.name
+ doc = update.doc
+ required = update.required
+
+ if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+ new_fields.append(field)
+ else:
+ has_changes = True
+ new_fields.append(
+ NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+ )
+
+ if has_changes:
return StructType(*new_fields)
return struct
- def field(self, field: NestedField, field_result: IcebergType) -> IcebergType:
- field_id: int = field.field_id
- if field_id in self.adds:
- new_fields = self.adds[field_id]
- if len(new_fields) > 0:
- fields = _ApplyChanges.add_fields(field_result.fields, new_fields)
- if len(fields) > 0:
- return StructType(*fields)
+ def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ # the API validates deletes, updates, and additions don't conflict handle deletes
+ if field.field_id in self._deletes:
+ return None
- return field_result
+ # handle updates
+ if (update := self._updates.get(field.field_id)) and field.field_type != update.field_type:
+ return update.field_type
- def list(self, list_type: ListType, element_result: IcebergType) -> IcebergType:
- element_field: NestedField = list_type.element_field
- element_type = self.field(element_field, element_result)
- if element_type is None:
- raise ValueError(f"Cannot delete element type from list: {element_field}")
+ # handle add & moves
+ added = self._adds.get(field.field_id)
+ moves = self._moves.get(field.field_id)
+ if added is not None or moves is not None:
+ if not isinstance(field.field_type, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {field}")
- is_element_optional = not list_type.element_required
+ if new_fields := _add_and_move_fields(field.field_type.fields, added or [], moves or []):
+ return StructType(*new_fields)
- if is_element_optional == element_field.required and list_type.element_type == element_type:
- return list_type
+ return field_result
+
+ def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ element_type = self.field(list_type.element_field, element_result)
+ if element_type is None:
+ raise ValueError(f"Cannot delete element type from list: {element_result}")
- return ListType(list_type.element_id, element_type, is_element_optional)
+ return ListType(element_id=list_type.element_id, element=element_type, element_required=list_type.element_required)
- def map(self, map_type: MapType, key_result: IcebergType, value_result: IcebergType) -> IcebergType:
+ def map(
+ self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType]
+ ) -> Optional[IcebergType]:
key_id: int = map_type.key_field.field_id
- if key_id in self.adds:
+ if key_id in self._adds:
raise ValueError(f"Cannot add fields to map keys: {map_type}")
value_field: NestedField = map_type.value_field
value_type = self.field(value_field, value_result)
if value_type is None:
raise ValueError(f"Cannot delete value type from map: {value_field}")
- is_value_optional = not map_type.value_required
+ return MapType(
+ key_id=map_type.key_id,
+ key_type=map_type.key_type,
+ value_id=map_type.value_id,
+ value_type=value_type,
+ value_required=map_type.value_required,
+ )
- if is_value_optional != value_field.required and map_type.value_type == value_type:
- return map_type
+ def primitive(self, primitive: PrimitiveType) -> Optional[IcebergType]:
+ return primitive
- return MapType(map_type.key_id, map_type.key_field, map_type.value_id, value_type, not is_value_optional)
- def primitive(self, primitive: PrimitiveType) -> IcebergType:
- return primitive
+def _add_fields(fields: Tuple[NestedField, ...], adds: Optional[List[NestedField]]) -> Optional[Tuple[NestedField, ...]]:
+ adds = adds or []
+ return None if len(adds) == 0 else fields + tuple(adds)
Review Comment:
The way this is called, do we want to return None? `_add_and_move_fields` doesn't check for it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306711893
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
Review Comment:
It throws an error indeed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306713188
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
Review Comment:
I'm open to making it more readable, but that doesn't work on my machine:
```python
➜ python git:(fd-followup) python3
Python 3.11.4 (main, Jul 25 2023, 17:36:13) [Clang 14.0.3 (clang-1403.0.22.14.1)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> tuple(*[1, 2], 3)
```
What does work is:
```python
self._adds[parent_id] = self._adds.get(parent_id, tuple()), + (field,)
```
But then we end up with the same thing as before, but then a tuple. I would suggest refactoring it as:
```python
if parent_id in self._adds:
self._adds[parent_id].append(field)
else:
self._adds[parent_id] = [field]
```
Then we don't recreate the collections all the time (but this isn't the hottest path, probably)
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
Review Comment:
I'm open to making it more readable, but that doesn't work on my machine:
```python
➜ python git:(fd-followup) python3
Python 3.11.4 (main, Jul 25 2023, 17:36:13) [Clang 14.0.3 (clang-1403.0.22.14.1)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> tuple(*[1, 2], 3)
```
What does work is:
```python
self._adds[parent_id] = self._adds.get(parent_id, tuple()), + (field,)
```
But then we end up with the same thing as before, but then a tuple. I would suggest refactoring it as:
```python
if parent_id in self._adds:
self._adds[parent_id].append(field)
else:
self._adds[parent_id] = [field]
```
Then we don't recreate the collections all the time (but this isn't the hottest path, probably)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306714078
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
Review Comment:
This was for updating the identifier fields. I've updated that to:
```python
if path_from in self._identifier_field_names:
self._identifier_field_names.remove(path_from)
self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313627337
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
Review Comment:
Nit: in `move_before` the arg name is `before_path` but it is `after_name` here. Probably should use a consistent name: `after_path`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313622941
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
Review Comment:
Ah, I see that this is using the same internals. Looks good to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306718999
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
Review Comment:
I agree, should we do the same for case sensitivity?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306718793
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
Review Comment:
I played around with this, but it gets messy because it also requires the `reference`, which also uses `_find_for_move`. I cleaned it up abit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Follow up on update schema
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1302222390
##########
python/mkdocs/docs/api.md:
##########
@@ -150,32 +150,25 @@ catalog.create_table(
Add new columns through the `Transaction` or `UpdateSchema` API:
-Use the Transaction API:
-
```python
-with table.transaction() as transaction:
- transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
+with table.update_schema() as update:
+ update.add_column("x", IntegerType(), "doc")
```
-Or, without a context manager:
+Or, without a context manager by calling the `.commit()` explicitly:
```python
-transaction = table.transaction()
-transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
-transaction.commit_transaction()
+table.update_schema().add_column("x", IntegerType(), "doc").commit()
```
-Or, use the UpdateSchema API directly:
+Use the Transaction API:
Review Comment:
What about "Alternatively, use the transaction API to combine changes from multiple operations"? Otherwise it isn't clear why you'd use a transaction.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314245090
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
Review Comment:
I like that, thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314244350
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
+ tbl.metadata_location,
Review Comment:
This is the location of a table that we created earlier.
Some context: I was also trying to run tests on the docs, so we know that they are up to date. I will split that out to a different PR since that was much more work than anticipated, and this PR has grown quite big already. The `s3a://` location in the comment above will change on each run, so just plugging in `tbl.metadata_location` created in an earlier example would fix this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#issuecomment-1704404610
> Schema ID is not being assigned in commit, which corrupts metadata
This was a great catch, updated this! The REST Catalog actually handled this correctly, but now it is also fixed and tested outside of the rest catalog.
> Tests are run as integration tests, but most could be converted to unit tests that call _apply
I ported some tests outside of the integration tests as well. As [mentioned above](https://github.com/apache/iceberg/pull/8374#discussion_r1314249767) I really like testing against the REST catalog, since uses the same public API that the customer uses (which is a good thing from an OOP perspective), but we're testing much more code than we actually need. I also ported some test back to avoid having only integration tests, but the majority is still IT tests because I prefer them. Let me know if you want me to port the rest back as well.
> update_column always creates an entry in changes even if the change should be a noop
Added and tested as well, also removed the `make_required` and `set_doc` operations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314247173
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
Review Comment:
I agree, updated it! 👍🏻
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
Review Comment:
I agree, updated it! 👍🏻
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314246550
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
Review Comment:
I agree, I removed the other methods
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304994873
##########
python/pyiceberg/table/__init__.py:
##########
@@ -889,28 +888,58 @@ def to_ray(self) -> ray.data.dataset.Dataset:
return ray.data.from_arrow(self.to_arrow())
+class MoveOperation(Enum):
+ First = 1
+ Before = 2
+ After = 3
+
+
+@dataclass
+class Move:
+ field_id: int
+ op: MoveOperation
+ other_field_id: Optional[int] = None
+
+
class UpdateSchema:
_table: Table
_schema: Schema
_last_column_id: itertools.count[int]
- _identifier_field_names: List[str]
- _adds: Dict[int, List[NestedField]]
- _added_name_to_id: Dict[str, int]
- _id_to_parent: Dict[int, str]
+ _identifier_field_names: Set[str]
+
+ _adds: Dict[int, List[NestedField]] = {}
+ _updates: Dict[int, NestedField] = {}
+ _deletes: Set[int] = set()
+ _moves: Dict[int, List[Move]] = {}
+
+ _added_name_to_id: Dict[str, int] = {}
+ _id_to_parent: Dict[int, str] = {}
_allow_incompatible_changes: bool
_case_sensitive: bool
_transaction: Optional[Transaction]
- def __init__(self, schema: Schema, table: Table, transaction: Optional[Transaction] = None):
+ def __init__(
+ self,
+ table: Table,
+ transaction: Optional[Transaction] = None,
+ allow_incompatible_changes: bool = False,
+ case_sensitive: bool = True,
+ ) -> None:
self._table = table
- self._schema = schema
- self._last_column_id = itertools.count(schema.highest_field_id + 1)
- self._identifier_field_names = schema.column_names
+ self._schema = table.schema()
+ self._last_column_id = itertools.count(self._schema.highest_field_id + 1)
Review Comment:
The highest field ID needs to come from the table metadata, not from the current schema. Otherwise, an ID could be reassigned if an ID is assigned, the schema is rolled back, and another column is added.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304807399
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +961,608 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
Review Comment:
This isn't quite correct. It's actually fine for the name to contain a `.` if the path was a tuple. If it was a tuple, then we know exactly what part is the parent. So this check is only needed if path is a str:
```python
if isinstance(path, str):
if '.' in path:
raise ValueError(...)
path = (path,)
...
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305005731
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
+ """Allow incompatible changes to the schema.
+
+ Returns:
+ This for method chaining
+ """
+ self._allow_incompatible_changes = True
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._schema.highest_field_id, new_schema.highest_field_id)
Review Comment:
This should also be the table's last_assigned_field_id.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306712586
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
Review Comment:
This was in there before any PyIceberg release. This follows the _Easier to Ask Forgiveness Than Permission_ style, which is quite popular in Python. I personally like to avoid this pattern and avoid the exceptions. The more now we have the walrus `:=` operator.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305435166
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +961,608 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
Review Comment:
Thanks, also ran into this when porting the tests
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313616847
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
Review Comment:
Nit: the second interpolation could be replaced with `{parent_full_path}`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314249375
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
Review Comment:
Schema evolution catches this:
```
Cannot change column type: foo: struct<2: bar: optional string> -> string
```
I've added a check to make the error more specific error:
```
Cannot change column type: struct<2: bar: optional string> is not a primitive
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313626530
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
Review Comment:
Above in `add_column` you use an `if` for a very similar case, but I'm good either way.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313624828
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
Review Comment:
It's fine to have this and the methods to set the requirement for a field, but it doesn't seem very pythonic to have `update_column_doc` and `require_column`, as well as `update_column` because it creates more than one way to do the same task. Plus it's a bit more code. Definitely up to you whether you think it's a good idea to keep these.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314310307
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
+ # DecimalType(9, 2),
+ # DecimalType(9, 3),
+ # DecimalType(18, 2)
+]
+
+
+@pytest.mark.parametrize("from_type", disallowed_promotions_types, ids=str)
+@pytest.mark.parametrize("to_type", disallowed_promotions_types, ids=str)
+@pytest.mark.integration
+def test_disallowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ if from_type != to_type and (from_type, to_type) not in allowed_promotions:
+ with pytest.raises(ValidationError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert str(exc_info.value).startswith("Cannot change column type: bar:")
+ else:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=to_type, required=True),
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.rename_column("foo", "vo")
+
+ assert simple_table.schema() == Schema(
+ NestedField(field_id=1, name="vo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple_nested(catalog: Catalog) -> None:
Review Comment:
Added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314317358
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
Review Comment:
Also added a check to make the noop possible, and added a test
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314246550
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
Review Comment:
Removed the other methods
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue merged pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue merged PR #8374:
URL: https://github.com/apache/iceberg/pull/8374
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304996719
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
Review Comment:
Maybe it's me, but using + makes me wonder about the argument types. I find this more clear:
```python
self._adds[parent_id] = tuple(*self._adds.get(parent_id, []), field)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304995596
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
Review Comment:
Why would `find_field` throw a `ValueError`? Shouldn't it return None instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306711870
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
Review Comment:
```python
def find_field(self, name_or_id: Union[str, int], case_sensitive: bool = True) -> NestedField:
"""Find a field using a field name or field ID.
Args:
name_or_id (Union[str, int]): Either a field name or a field ID.
case_sensitive (bool, optional): Whether to perform a case-sensitive lookup using a field name. Defaults to True.
Raises:
ValueError: When the value cannot be found.
Returns:
NestedField: The matched NestedField.
"""
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306711434
##########
python/pyiceberg/table/__init__.py:
##########
@@ -889,28 +888,58 @@ def to_ray(self) -> ray.data.dataset.Dataset:
return ray.data.from_arrow(self.to_arrow())
+class MoveOperation(Enum):
+ First = 1
+ Before = 2
+ After = 3
+
+
+@dataclass
+class Move:
+ field_id: int
+ op: MoveOperation
+ other_field_id: Optional[int] = None
+
+
class UpdateSchema:
_table: Table
_schema: Schema
_last_column_id: itertools.count[int]
- _identifier_field_names: List[str]
- _adds: Dict[int, List[NestedField]]
- _added_name_to_id: Dict[str, int]
- _id_to_parent: Dict[int, str]
+ _identifier_field_names: Set[str]
+
+ _adds: Dict[int, List[NestedField]] = {}
+ _updates: Dict[int, NestedField] = {}
+ _deletes: Set[int] = set()
+ _moves: Dict[int, List[Move]] = {}
+
+ _added_name_to_id: Dict[str, int] = {}
+ _id_to_parent: Dict[int, str] = {}
_allow_incompatible_changes: bool
_case_sensitive: bool
_transaction: Optional[Transaction]
- def __init__(self, schema: Schema, table: Table, transaction: Optional[Transaction] = None):
+ def __init__(
+ self,
+ table: Table,
+ transaction: Optional[Transaction] = None,
+ allow_incompatible_changes: bool = False,
+ case_sensitive: bool = True,
+ ) -> None:
self._table = table
- self._schema = schema
- self._last_column_id = itertools.count(schema.highest_field_id + 1)
- self._identifier_field_names = schema.column_names
+ self._schema = table.schema()
+ self._last_column_id = itertools.count(self._schema.highest_field_id + 1)
Review Comment:
Thanks, I also ran into this in a test :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306719848
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
+ """Allow incompatible changes to the schema.
+
+ Returns:
+ This for method chaining
+ """
+ self._allow_incompatible_changes = True
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._schema.highest_field_id, new_schema.highest_field_id)
+ updates = [
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ ]
+ requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ else:
+ self._table._do_commit( # pylint: disable=W0212
+ CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ )
+
+ def _apply(self) -> Schema:
+ """Apply the pending changes to the original schema and returns the result.
+
+ Returns:
+ the result Schema when all pending updates are applied
+ """
+ struct = visit(self._schema, _ApplyChanges(self._adds, self._updates, self._deletes, self._moves))
+ if struct is None:
+ # Should never happen
+ raise ValueError("Could not apply changes")
+
+ # @TODO: This differs still a bit from the Java side,
+ # The validate identifier field is missing
+ field_ids = set()
+ for name in self._identifier_field_names:
+ field = self._schema.find_field(name, self._case_sensitive)
+ field_ids.add(field.field_id)
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot delete identifier field {name}. To force deletion, update the identifier fields first.")
+
+ # If it nested, also check if the parents aren't deleted
+ column_name = self._id_to_parent.get(field.field_id)
+ while column_name is not None:
+ parent = self._schema.find_field(column_name)
+ if parent.field_id in self._deletes:
+ raise ValueError(
+ f"Cannot delete field {parent.field_id} as it will delete nested identifier field {name}",
+ )
+ column_name = self._id_to_parent.get(parent.field_id)
+
+ return Schema(*struct.fields, identifier_field_ids=field_ids)
def assign_new_column_id(self) -> int:
return next(self._last_column_id)
-def _apply_changes(schema_: Schema, adds: Dict[int, List[NestedField]], identifier_field_names: List[str]) -> Schema:
- struct = visit(schema_, _ApplyChanges(adds))
- name_to_id: Dict[str, int] = index_by_name(struct)
- for name in identifier_field_names:
- if name not in name_to_id:
- raise ValueError(f"Cannot add field {name} as an identifier field: not found in current schema or added columns")
-
- return Schema(*struct.fields)
+class _ApplyChanges(SchemaVisitor[Optional[IcebergType]]):
+ _adds: Dict[int, List[NestedField]]
+ _updates: Dict[int, NestedField]
+ _deletes: Set[int]
+ _moves: Dict[int, List[Move]]
+ def __init__(
+ self, adds: Dict[int, List[NestedField]], updates: Dict[int, NestedField], deletes: Set[int], moves: Dict[int, List[Move]]
+ ) -> None:
+ self._adds = adds
+ self._updates = updates
+ self._deletes = deletes
+ self._moves = moves
-class _ApplyChanges(SchemaVisitor[IcebergType]):
- def __init__(self, adds: Dict[int, List[NestedField]]):
- self.adds = adds
+ def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ if new_fields := _add_fields(struct_result.fields if struct_result else [], self._adds.get(TABLE_ROOT_ID)):
+ return StructType(*new_fields)
+ else:
+ return struct_result
- def schema(self, schema: Schema, struct_result: IcebergType) -> IcebergType:
- fields = _ApplyChanges.add_fields(schema.as_struct().fields, self.adds.get(TABLE_ROOT_ID))
- if len(fields) > 0:
- return StructType(*fields)
+ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]:
+ has_changes = False
+ new_fields = []
- return struct_result
+ for idx, result_type in enumerate(field_results):
+ result_type = field_results[idx]
- def struct(self, struct: StructType, field_results: List[IcebergType]) -> IcebergType:
- has_change = False
- new_fields: List[NestedField] = []
- for i in range(len(field_results)):
- type_: Optional[IcebergType] = field_results[i]
- if type_ is None:
- has_change = True
+ # Has been deleted
+ if result_type is None:
+ has_changes = True
continue
- field: NestedField = struct.fields[i]
- new_fields.append(field)
+ field = struct.fields[idx]
+
+ name = field.name
+ doc = field.doc
+ required = field.required
- if has_change:
+ # There is an update
+ if update := self._updates.get(field.field_id):
+ name = update.name
+ doc = update.doc
+ required = update.required
+
+ if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+ new_fields.append(field)
+ else:
+ has_changes = True
+ new_fields.append(
+ NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+ )
+
+ if has_changes:
return StructType(*new_fields)
return struct
- def field(self, field: NestedField, field_result: IcebergType) -> IcebergType:
- field_id: int = field.field_id
- if field_id in self.adds:
- new_fields = self.adds[field_id]
- if len(new_fields) > 0:
- fields = _ApplyChanges.add_fields(field_result.fields, new_fields)
- if len(fields) > 0:
- return StructType(*fields)
+ def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ # the API validates deletes, updates, and additions don't conflict handle deletes
+ if field.field_id in self._deletes:
+ return None
- return field_result
+ # handle updates
+ if (update := self._updates.get(field.field_id)) and field.field_type != update.field_type:
+ return update.field_type
- def list(self, list_type: ListType, element_result: IcebergType) -> IcebergType:
- element_field: NestedField = list_type.element_field
- element_type = self.field(element_field, element_result)
- if element_type is None:
- raise ValueError(f"Cannot delete element type from list: {element_field}")
+ # handle add & moves
+ added = self._adds.get(field.field_id)
+ moves = self._moves.get(field.field_id)
+ if added is not None or moves is not None:
+ if not isinstance(field.field_type, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {field}")
- is_element_optional = not list_type.element_required
+ if new_fields := _add_and_move_fields(field.field_type.fields, added or [], moves or []):
+ return StructType(*new_fields)
- if is_element_optional == element_field.required and list_type.element_type == element_type:
- return list_type
+ return field_result
+
+ def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ element_type = self.field(list_type.element_field, element_result)
+ if element_type is None:
+ raise ValueError(f"Cannot delete element type from list: {element_result}")
- return ListType(list_type.element_id, element_type, is_element_optional)
+ return ListType(element_id=list_type.element_id, element=element_type, element_required=list_type.element_required)
- def map(self, map_type: MapType, key_result: IcebergType, value_result: IcebergType) -> IcebergType:
+ def map(
+ self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType]
+ ) -> Optional[IcebergType]:
key_id: int = map_type.key_field.field_id
- if key_id in self.adds:
+ if key_id in self._adds:
raise ValueError(f"Cannot add fields to map keys: {map_type}")
value_field: NestedField = map_type.value_field
value_type = self.field(value_field, value_result)
if value_type is None:
raise ValueError(f"Cannot delete value type from map: {value_field}")
- is_value_optional = not map_type.value_required
+ return MapType(
+ key_id=map_type.key_id,
+ key_type=map_type.key_type,
+ value_id=map_type.value_id,
+ value_type=value_type,
+ value_required=map_type.value_required,
+ )
- if is_value_optional != value_field.required and map_type.value_type == value_type:
- return map_type
+ def primitive(self, primitive: PrimitiveType) -> Optional[IcebergType]:
+ return primitive
- return MapType(map_type.key_id, map_type.key_field, map_type.value_id, value_type, not is_value_optional)
- def primitive(self, primitive: PrimitiveType) -> IcebergType:
- return primitive
+def _add_fields(fields: Tuple[NestedField, ...], adds: Optional[List[NestedField]]) -> Optional[Tuple[NestedField, ...]]:
+ adds = adds or []
+ return None if len(adds) == 0 else fields + tuple(adds)
Review Comment:
No, I think we can drop the `None`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313636444
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move {full_name} after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._table.metadata.last_column_id, new_schema.highest_field_id)
+ updates = (
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ )
+ requirements = (AssertCurrentSchemaId(current_schema_id=self._schema.schema_id),)
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ else:
+ self._table._do_commit(updates=updates, requirements=requirements) # pylint: disable=W0212
+
+ def _apply(self) -> Schema:
+ """Apply the pending changes to the original schema and returns the result.
+
+ Returns:
+ the result Schema when all pending updates are applied
+ """
+ struct = visit(self._schema, _ApplyChanges(self._adds, self._updates, self._deletes, self._moves))
+ if struct is None:
+ # Should never happen
+ raise ValueError("Could not apply changes")
+
+ new_schema = Schema(*struct.fields)
+
+ field_ids = set()
+ for name in self._identifier_field_names:
+ try:
+ field = new_schema.find_field(name, self._case_sensitive)
+ except ValueError as e:
+ raise ValueError(
+ f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first."
+ ) from e
+
+ field_ids.add(field.field_id)
+
+ return Schema(*struct.fields, identifier_field_ids=field_ids)
def assign_new_column_id(self) -> int:
return next(self._last_column_id)
-def _apply_changes(schema_: Schema, adds: Dict[int, List[NestedField]], identifier_field_names: List[str]) -> Schema:
- struct = visit(schema_, _ApplyChanges(adds))
- name_to_id: Dict[str, int] = index_by_name(struct)
- for name in identifier_field_names:
- if name not in name_to_id:
- raise ValueError(f"Cannot add field {name} as an identifier field: not found in current schema or added columns")
+class _ApplyChanges(SchemaVisitor[Optional[IcebergType]]):
+ _adds: Dict[int, List[NestedField]]
+ _updates: Dict[int, NestedField]
+ _deletes: Set[int]
+ _moves: Dict[int, List[Move]]
- return Schema(*struct.fields)
+ def __init__(
+ self, adds: Dict[int, List[NestedField]], updates: Dict[int, NestedField], deletes: Set[int], moves: Dict[int, List[Move]]
+ ) -> None:
+ self._adds = adds
+ self._updates = updates
+ self._deletes = deletes
+ self._moves = moves
+ def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ added = self._adds.get(TABLE_ROOT_ID)
+ moves = self._moves.get(TABLE_ROOT_ID)
-class _ApplyChanges(SchemaVisitor[IcebergType]):
- def __init__(self, adds: Dict[int, List[NestedField]]):
- self.adds = adds
+ if added is not None or moves is not None:
+ if not isinstance(struct_result, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {struct_result}")
- def schema(self, schema: Schema, struct_result: IcebergType) -> IcebergType:
- fields = _ApplyChanges.add_fields(schema.as_struct().fields, self.adds.get(TABLE_ROOT_ID))
- if len(fields) > 0:
- return StructType(*fields)
+ if new_fields := _add_and_move_fields(struct_result.fields, added or [], moves or []):
+ return StructType(*new_fields)
return struct_result
- def struct(self, struct: StructType, field_results: List[IcebergType]) -> IcebergType:
- has_change = False
- new_fields: List[NestedField] = []
- for i in range(len(field_results)):
- type_: Optional[IcebergType] = field_results[i]
- if type_ is None:
- has_change = True
+ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]:
+ has_changes = False
+ new_fields = []
+
+ for idx, result_type in enumerate(field_results):
+ result_type = field_results[idx]
+
+ # Has been deleted
+ if result_type is None:
+ has_changes = True
continue
- field: NestedField = struct.fields[i]
- new_fields.append(field)
+ field = struct.fields[idx]
+
+ name = field.name
+ doc = field.doc
+ required = field.required
+
+ # There is an update
+ if update := self._updates.get(field.field_id):
+ name = update.name
+ doc = update.doc
+ required = update.required
+
+ if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+ new_fields.append(field)
+ else:
+ has_changes = True
+ new_fields.append(
+ NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+ )
- if has_change:
+ if has_changes:
return StructType(*new_fields)
return struct
- def field(self, field: NestedField, field_result: IcebergType) -> IcebergType:
- field_id: int = field.field_id
- if field_id in self.adds:
- new_fields = self.adds[field_id]
- if len(new_fields) > 0:
- fields = _ApplyChanges.add_fields(field_result.fields, new_fields)
- if len(fields) > 0:
- return StructType(*fields)
+ def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ # the API validates deletes, updates, and additions don't conflict handle deletes
+ if field.field_id in self._deletes:
+ return None
+
+ # handle updates
+ if (update := self._updates.get(field.field_id)) and field.field_type != update.field_type:
+ return update.field_type
+
+ if isinstance(field_result, StructType):
+ # handle add & moves
+ added = self._adds.get(field.field_id)
+ moves = self._moves.get(field.field_id)
+ if added is not None or moves is not None:
+ if not isinstance(field.field_type, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {field}")
+
+ if new_fields := _add_and_move_fields(field_result.fields, added or [], moves or []):
+ return StructType(*new_fields)
return field_result
- def list(self, list_type: ListType, element_result: IcebergType) -> IcebergType:
- element_field: NestedField = list_type.element_field
- element_type = self.field(element_field, element_result)
+ def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ element_type = self.field(list_type.element_field, element_result)
if element_type is None:
- raise ValueError(f"Cannot delete element type from list: {element_field}")
+ raise ValueError(f"Cannot delete element type from list: {element_result}")
+
+ return ListType(element_id=list_type.element_id, element=element_type, element_required=list_type.element_required)
- is_element_optional = not list_type.element_required
+ def map(
+ self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType]
+ ) -> Optional[IcebergType]:
+ key_id: int = map_type.key_field.field_id
- if is_element_optional == element_field.required and list_type.element_type == element_type:
- return list_type
+ if key_id in self._deletes:
+ raise ValueError(f"Cannot delete map keys: {map_type}")
- return ListType(list_type.element_id, element_type, is_element_optional)
+ if key_id in self._updates:
+ raise ValueError(f"Cannot update map keys: {map_type}")
- def map(self, map_type: MapType, key_result: IcebergType, value_result: IcebergType) -> IcebergType:
- key_id: int = map_type.key_field.field_id
- if key_id in self.adds:
+ if key_id in self._adds:
raise ValueError(f"Cannot add fields to map keys: {map_type}")
+ if map_type.key_type != key_result:
+ raise ValueError(f"Cannot alter map keys: {map_type}")
+
value_field: NestedField = map_type.value_field
value_type = self.field(value_field, value_result)
if value_type is None:
raise ValueError(f"Cannot delete value type from map: {value_field}")
- is_value_optional = not map_type.value_required
+ return MapType(
+ key_id=map_type.key_id,
+ key_type=map_type.key_type,
+ value_id=map_type.value_id,
+ value_type=value_type,
+ value_required=map_type.value_required,
+ )
- if is_value_optional != value_field.required and map_type.value_type == value_type:
- return map_type
+ def primitive(self, primitive: PrimitiveType) -> Optional[IcebergType]:
+ return primitive
- return MapType(map_type.key_id, map_type.key_field, map_type.value_id, value_type, not is_value_optional)
- def primitive(self, primitive: PrimitiveType) -> IcebergType:
- return primitive
+def _add_fields(fields: Tuple[NestedField, ...], adds: Optional[List[NestedField]]) -> Tuple[NestedField, ...]:
+ adds = adds or []
+ return fields + tuple(adds)
+
- @staticmethod
- def add_fields(fields: Tuple[NestedField, ...], adds: Optional[List[NestedField]]) -> List[NestedField]:
- new_fields: List[NestedField] = []
- new_fields.extend(fields)
- if adds:
- new_fields.extend(adds)
- return new_fields
+def _move_fields(fields: Tuple[NestedField, ...], moves: List[Move]) -> Tuple[NestedField, ...]:
+ reordered = list(copy(fields))
+ for move in moves:
+ # Find the field that we're about to move
+ field = next(field for field in reordered if field.field_id == move.field_id)
+ # Remove the field that we're about to move from the list
+ reordered = [field for field in reordered if field.field_id != move.field_id]
+
+ if move.op == MoveOperation.First:
+ reordered = [field] + reordered
+ elif move.op == MoveOperation.Before or move.op == MoveOperation.After:
+ other_field_id = move.other_field_id
+ other_field_pos = next(i for i, field in enumerate(reordered) if field.field_id == other_field_id)
+ if move.op == MoveOperation.Before:
+ reordered.insert(other_field_pos, field)
+ else:
+ reordered.insert(other_field_pos + 1, field)
+ else:
+ raise ValueError(f"Unknown operation: {move.op}")
+
+ return tuple(reordered)
+
+
+def _add_and_move_fields(
+ fields: Tuple[NestedField, ...], adds: List[NestedField], moves: List[Move]
+) -> Optional[Tuple[NestedField, ...]]:
+ if adds:
Review Comment:
Do the calls to this method need `adds or []` and `moves or []` since this check is done?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313615392
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
+ tbl.metadata_location,
+ properties={
+ "s3.endpoint": "http://127.0.0.1:9000",
+ "py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+)
+```
+
+The static-table is considered read-only.
-Add new columns through the `Transaction` or `UpdateSchema` API:
+## Schema evolution
-Use the Transaction API:
+PyIceberg supports full schema evolution through the Python API. It takes care of setting the field-IDs and makes sure that only non-breaking changes are done (can be overriden).
+
+In the examples below, the `.update_schema()` is called from the table itself.
+
+```python
+with table.update_schema() as update:
+ update.add_column("some_field", IntegerType(), "doc")
+```
+
+You can also initiate a transaction if you want to make more changes than just evolving the schema:
```python
with table.transaction() as transaction:
- transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
+ with transaction.update_schema() as update_schema:
+ update.add_column("some_other_field", IntegerType(), "doc")
+ # ... Update properties etc
```
-Or, without a context manager:
+### Add column
+
+Using `add_column` you can add a column, without having to worry about the field-id:
```python
-transaction = table.transaction()
-transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
-transaction.commit_transaction()
+with table.update_schema() as update:
+ update.add_column("retries", IntegerType(), "Number of retries to place the bid")
+ # In a struct
+ update.add_column("details.confirmed_by", StringType(), "Name of the exchange")
```
-Or, use the UpdateSchema API directly:
+### Rename column
+
+Renaming a field in an Iceberg table is simple:
```python
with table.update_schema() as update:
- update.add_column("x", IntegerType(), "doc")
+ update.rename("retries", "num_retries")
+ # In a struct, only the new name field
+ update.rename("properties.confirmed_by", "exchange")
```
-Or, without a context manager:
+### Rename column
+
+Move a field inside of struct:
```python
-table.update_schema().add_column("x", IntegerType(), "doc").commit()
+with table.update_schema() as update:
+ update.move_first("symbol")
+ update.move_after("bid", "ask")
+ # In a struct, only the new name field
Review Comment:
What do you mean here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313614782
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
+ tbl.metadata_location,
Review Comment:
What's `tbl` here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313644902
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
+ # DecimalType(9, 2),
+ # DecimalType(9, 3),
+ # DecimalType(18, 2)
+]
+
+
+@pytest.mark.parametrize("from_type", disallowed_promotions_types, ids=str)
+@pytest.mark.parametrize("to_type", disallowed_promotions_types, ids=str)
+@pytest.mark.integration
+def test_disallowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ if from_type != to_type and (from_type, to_type) not in allowed_promotions:
+ with pytest.raises(ValidationError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert str(exc_info.value).startswith("Cannot change column type: bar:")
+ else:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=to_type, required=True),
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.rename_column("foo", "vo")
+
+ assert simple_table.schema() == Schema(
+ NestedField(field_id=1, name="vo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple_nested(catalog: Catalog) -> None:
Review Comment:
Would be nice to have a case with `.` names, like `rename_column(("a.b", "c.d"), "e.f")` produces column `("a.b", "e.f")`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Follow up on update schema
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1302507167
##########
python/mkdocs/docs/api.md:
##########
@@ -150,32 +150,25 @@ catalog.create_table(
Add new columns through the `Transaction` or `UpdateSchema` API:
-Use the Transaction API:
-
```python
-with table.transaction() as transaction:
- transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
+with table.update_schema() as update:
+ update.add_column("x", IntegerType(), "doc")
```
-Or, without a context manager:
+Or, without a context manager by calling the `.commit()` explicitly:
```python
-transaction = table.transaction()
-transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
-transaction.commit_transaction()
+table.update_schema().add_column("x", IntegerType(), "doc").commit()
```
-Or, use the UpdateSchema API directly:
+Use the Transaction API:
Review Comment:
I tried to indicate that by also setting a property in the same transaction, but I agree that it is better to explicitly state it. Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305004461
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
Review Comment:
Could dedup by moving this into `_find_for_move`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306716314
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
Review Comment:
I left the old methods in place, WDYT?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305008179
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
+ """Allow incompatible changes to the schema.
+
+ Returns:
+ This for method chaining
+ """
+ self._allow_incompatible_changes = True
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._schema.highest_field_id, new_schema.highest_field_id)
+ updates = [
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ ]
+ requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ else:
+ self._table._do_commit( # pylint: disable=W0212
+ CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ )
+
+ def _apply(self) -> Schema:
+ """Apply the pending changes to the original schema and returns the result.
+
+ Returns:
+ the result Schema when all pending updates are applied
+ """
+ struct = visit(self._schema, _ApplyChanges(self._adds, self._updates, self._deletes, self._moves))
+ if struct is None:
+ # Should never happen
+ raise ValueError("Could not apply changes")
+
+ # @TODO: This differs still a bit from the Java side,
+ # The validate identifier field is missing
+ field_ids = set()
+ for name in self._identifier_field_names:
+ field = self._schema.find_field(name, self._case_sensitive)
+ field_ids.add(field.field_id)
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot delete identifier field {name}. To force deletion, update the identifier fields first.")
+
+ # If it nested, also check if the parents aren't deleted
+ column_name = self._id_to_parent.get(field.field_id)
+ while column_name is not None:
+ parent = self._schema.find_field(column_name)
+ if parent.field_id in self._deletes:
+ raise ValueError(
+ f"Cannot delete field {parent.field_id} as it will delete nested identifier field {name}",
+ )
+ column_name = self._id_to_parent.get(parent.field_id)
+
+ return Schema(*struct.fields, identifier_field_ids=field_ids)
def assign_new_column_id(self) -> int:
return next(self._last_column_id)
-def _apply_changes(schema_: Schema, adds: Dict[int, List[NestedField]], identifier_field_names: List[str]) -> Schema:
- struct = visit(schema_, _ApplyChanges(adds))
- name_to_id: Dict[str, int] = index_by_name(struct)
- for name in identifier_field_names:
- if name not in name_to_id:
- raise ValueError(f"Cannot add field {name} as an identifier field: not found in current schema or added columns")
-
- return Schema(*struct.fields)
+class _ApplyChanges(SchemaVisitor[Optional[IcebergType]]):
+ _adds: Dict[int, List[NestedField]]
+ _updates: Dict[int, NestedField]
+ _deletes: Set[int]
+ _moves: Dict[int, List[Move]]
+ def __init__(
+ self, adds: Dict[int, List[NestedField]], updates: Dict[int, NestedField], deletes: Set[int], moves: Dict[int, List[Move]]
+ ) -> None:
+ self._adds = adds
+ self._updates = updates
+ self._deletes = deletes
+ self._moves = moves
-class _ApplyChanges(SchemaVisitor[IcebergType]):
- def __init__(self, adds: Dict[int, List[NestedField]]):
- self.adds = adds
+ def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ if new_fields := _add_fields(struct_result.fields if struct_result else [], self._adds.get(TABLE_ROOT_ID)):
+ return StructType(*new_fields)
+ else:
+ return struct_result
- def schema(self, schema: Schema, struct_result: IcebergType) -> IcebergType:
- fields = _ApplyChanges.add_fields(schema.as_struct().fields, self.adds.get(TABLE_ROOT_ID))
- if len(fields) > 0:
- return StructType(*fields)
+ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]:
+ has_changes = False
+ new_fields = []
- return struct_result
+ for idx, result_type in enumerate(field_results):
+ result_type = field_results[idx]
- def struct(self, struct: StructType, field_results: List[IcebergType]) -> IcebergType:
- has_change = False
- new_fields: List[NestedField] = []
- for i in range(len(field_results)):
- type_: Optional[IcebergType] = field_results[i]
- if type_ is None:
- has_change = True
+ # Has been deleted
+ if result_type is None:
+ has_changes = True
continue
- field: NestedField = struct.fields[i]
- new_fields.append(field)
+ field = struct.fields[idx]
+
+ name = field.name
+ doc = field.doc
+ required = field.required
- if has_change:
+ # There is an update
+ if update := self._updates.get(field.field_id):
+ name = update.name
+ doc = update.doc
+ required = update.required
+
+ if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+ new_fields.append(field)
+ else:
+ has_changes = True
+ new_fields.append(
+ NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+ )
+
+ if has_changes:
return StructType(*new_fields)
return struct
- def field(self, field: NestedField, field_result: IcebergType) -> IcebergType:
- field_id: int = field.field_id
- if field_id in self.adds:
- new_fields = self.adds[field_id]
- if len(new_fields) > 0:
- fields = _ApplyChanges.add_fields(field_result.fields, new_fields)
- if len(fields) > 0:
- return StructType(*fields)
+ def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ # the API validates deletes, updates, and additions don't conflict handle deletes
+ if field.field_id in self._deletes:
+ return None
- return field_result
+ # handle updates
+ if (update := self._updates.get(field.field_id)) and field.field_type != update.field_type:
+ return update.field_type
- def list(self, list_type: ListType, element_result: IcebergType) -> IcebergType:
- element_field: NestedField = list_type.element_field
- element_type = self.field(element_field, element_result)
- if element_type is None:
- raise ValueError(f"Cannot delete element type from list: {element_field}")
+ # handle add & moves
+ added = self._adds.get(field.field_id)
+ moves = self._moves.get(field.field_id)
+ if added is not None or moves is not None:
+ if not isinstance(field.field_type, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {field}")
- is_element_optional = not list_type.element_required
+ if new_fields := _add_and_move_fields(field.field_type.fields, added or [], moves or []):
+ return StructType(*new_fields)
- if is_element_optional == element_field.required and list_type.element_type == element_type:
- return list_type
+ return field_result
+
+ def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ element_type = self.field(list_type.element_field, element_result)
+ if element_type is None:
+ raise ValueError(f"Cannot delete element type from list: {element_result}")
- return ListType(list_type.element_id, element_type, is_element_optional)
+ return ListType(element_id=list_type.element_id, element=element_type, element_required=list_type.element_required)
- def map(self, map_type: MapType, key_result: IcebergType, value_result: IcebergType) -> IcebergType:
+ def map(
+ self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType]
+ ) -> Optional[IcebergType]:
key_id: int = map_type.key_field.field_id
- if key_id in self.adds:
+ if key_id in self._adds:
Review Comment:
I think there are a lot more checks here in the Java version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306719615
##########
python/pyiceberg/table/__init__.py:
##########
@@ -527,8 +519,13 @@ def history(self) -> List[SnapshotLogEntry]:
"""Get the snapshot history of this table."""
return self.metadata.snapshot_log
- def update_schema(self) -> UpdateSchema:
- return UpdateSchema(self.schema(), self)
+ def update_schema(self, allow_incompatible_changes: bool = False, case_sensitive: bool = True) -> UpdateSchema:
+ return UpdateSchema(self, allow_incompatible_changes=allow_incompatible_changes, case_sensitive=case_sensitive)
+
+ def _do_commit(self, request: CommitTableRequest) -> None:
Review Comment:
Yes, I think that's cleaner as well, updated!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314244822
##########
python/mkdocs/docs/api.md:
##########
@@ -133,52 +118,140 @@ partition_spec = PartitionSpec(
from pyiceberg.table.sorting import SortOrder, SortField
from pyiceberg.transforms import IdentityTransform
-sort_order = SortOrder(SortField(source_id=4, transform=IdentityTransform()))
-
-catalog = load_catalog("prod")
+# Sort on the symbol
+sort_order = SortOrder(SortField(source_id=2, transform=IdentityTransform()))
catalog.create_table(
- identifier="default.bids",
- location="/Users/fokkodriesprong/Desktop/docker-spark-iceberg/wh/bids/",
+ identifier="docs_example.bids",
schema=schema,
partition_spec=partition_spec,
sort_order=sort_order,
)
```
-### Update table schema
+## Load a table
+
+### Catalog table
+
+Loading the `bids` table:
+
+```python
+table = catalog.load_table("docs_example.bids")
+# Equivalent to:
+table = catalog.load_table(("docs_example", "bids"))
+# The tuple syntax can be used if the namespace or table contains a dot.
+```
+
+This returns a `Table` that represents an Iceberg table that can be queried and altered.
+
+### Static table
+
+To load a table directly from a metadata file (i.e., **without** using a catalog), you can use a `StaticTable` as follows:
+
+```python
+from pyiceberg.table import StaticTable
+
+static_table = StaticTable.from_metadata(
+ # For example:
+ # "s3a://warehouse/wh/nyc.db/taxis/metadata/00002-6ea51ce3-62aa-4197-9cf8-43d07c3440ca.metadata.json",
+ tbl.metadata_location,
+ properties={
+ "s3.endpoint": "http://127.0.0.1:9000",
+ "py-io-impl": "pyiceberg.io.pyarrow.PyArrowFileIO",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+)
+```
+
+The static-table is considered read-only.
-Add new columns through the `Transaction` or `UpdateSchema` API:
+## Schema evolution
-Use the Transaction API:
+PyIceberg supports full schema evolution through the Python API. It takes care of setting the field-IDs and makes sure that only non-breaking changes are done (can be overriden).
+
+In the examples below, the `.update_schema()` is called from the table itself.
+
+```python
+with table.update_schema() as update:
+ update.add_column("some_field", IntegerType(), "doc")
+```
+
+You can also initiate a transaction if you want to make more changes than just evolving the schema:
```python
with table.transaction() as transaction:
- transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
+ with transaction.update_schema() as update_schema:
+ update.add_column("some_other_field", IntegerType(), "doc")
+ # ... Update properties etc
```
-Or, without a context manager:
+### Add column
+
+Using `add_column` you can add a column, without having to worry about the field-id:
```python
-transaction = table.transaction()
-transaction.update_schema().add_column("x", IntegerType(), "doc").commit()
-transaction.commit_transaction()
+with table.update_schema() as update:
+ update.add_column("retries", IntegerType(), "Number of retries to place the bid")
+ # In a struct
+ update.add_column("details.confirmed_by", StringType(), "Name of the exchange")
```
-Or, use the UpdateSchema API directly:
+### Rename column
+
+Renaming a field in an Iceberg table is simple:
```python
with table.update_schema() as update:
- update.add_column("x", IntegerType(), "doc")
+ update.rename("retries", "num_retries")
+ # In a struct, only the new name field
+ update.rename("properties.confirmed_by", "exchange")
```
-Or, without a context manager:
+### Rename column
+
+Move a field inside of struct:
```python
-table.update_schema().add_column("x", IntegerType(), "doc").commit()
+with table.update_schema() as update:
+ update.move_first("symbol")
+ update.move_after("bid", "ask")
+ # In a struct, only the new name field
Review Comment:
That you don't have to provide the full struct path in the `to` argument, I've rephrased it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314248605
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move {full_name} after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._table.metadata.last_column_id, new_schema.highest_field_id)
+ updates = (
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ )
+ requirements = (AssertCurrentSchemaId(current_schema_id=self._schema.schema_id),)
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
Review Comment:
Great observation. I checked and we only allow a single `AddSchemaUpdate` in a transaction. So I think we're okay for now. I've added a test to capture the behavior.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314254092
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
Review Comment:
Yes it passes right now :)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314250295
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
Review Comment:
The schema-IDs are updated correctly, but we don't validate them. The Python `__eq__` is the same as Java's schema: https://github.com/apache/iceberg/blob/master/api/src/main/java/org/apache/iceberg/Schema.java#L464-L473
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1315076119
##########
python/mkdocs/docs/api.md:
##########
@@ -204,20 +196,20 @@ Renaming a field in an Iceberg table is simple:
```python
with table.update_schema() as update:
update.rename("retries", "num_retries")
- # In a struct, only the new name field
+ # This will rename `confirmed_by` to `exchange`
update.rename("properties.confirmed_by", "exchange")
```
-### Rename column
+### Move column
Move a field inside of struct:
```python
with table.update_schema() as update:
update.move_first("symbol")
update.move_after("bid", "ask")
- # In a struct, only the new name field
- update.move_before("details.exchange", "properties.created_by")
+ # This will move `confirmed_by` before `exchange`
Review Comment:
Nit: `created_by` is moved, not `confirmed_by`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304796697
##########
python/pyiceberg/table/__init__.py:
##########
@@ -225,17 +221,13 @@ def commit_transaction(self) -> Table:
"""
# Strip the catalog name
if len(self._updates) > 0:
- response = self._table.catalog._commit_table( # pylint: disable=W0212
+ self._table._do_commit( # pylint: disable=W0212
Review Comment:
I'm fine with this being private for now, but we may want to actually expose it later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305003229
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
Review Comment:
Don't we also need to update `_id_to_parent`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304998601
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
Review Comment:
In the Java API, this is the leaf field name. There's no need to split.
`rename("a.b", "c.d")` should produce field `a.c.d`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306716271
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
Review Comment:
I like it, updated!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306719711
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
+ """Allow incompatible changes to the schema.
+
+ Returns:
+ This for method chaining
+ """
+ self._allow_incompatible_changes = True
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._schema.highest_field_id, new_schema.highest_field_id)
+ updates = [
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ ]
+ requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ else:
+ self._table._do_commit( # pylint: disable=W0212
+ CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ )
+
+ def _apply(self) -> Schema:
+ """Apply the pending changes to the original schema and returns the result.
+
+ Returns:
+ the result Schema when all pending updates are applied
+ """
+ struct = visit(self._schema, _ApplyChanges(self._adds, self._updates, self._deletes, self._moves))
+ if struct is None:
+ # Should never happen
+ raise ValueError("Could not apply changes")
+
+ # @TODO: This differs still a bit from the Java side,
+ # The validate identifier field is missing
+ field_ids = set()
+ for name in self._identifier_field_names:
+ field = self._schema.find_field(name, self._case_sensitive)
+ field_ids.add(field.field_id)
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot delete identifier field {name}. To force deletion, update the identifier fields first.")
+
+ # If it nested, also check if the parents aren't deleted
+ column_name = self._id_to_parent.get(field.field_id)
+ while column_name is not None:
+ parent = self._schema.find_field(column_name)
+ if parent.field_id in self._deletes:
+ raise ValueError(
+ f"Cannot delete field {parent.field_id} as it will delete nested identifier field {name}",
+ )
+ column_name = self._id_to_parent.get(parent.field_id)
+
+ return Schema(*struct.fields, identifier_field_ids=field_ids)
def assign_new_column_id(self) -> int:
return next(self._last_column_id)
-def _apply_changes(schema_: Schema, adds: Dict[int, List[NestedField]], identifier_field_names: List[str]) -> Schema:
- struct = visit(schema_, _ApplyChanges(adds))
- name_to_id: Dict[str, int] = index_by_name(struct)
- for name in identifier_field_names:
- if name not in name_to_id:
- raise ValueError(f"Cannot add field {name} as an identifier field: not found in current schema or added columns")
-
- return Schema(*struct.fields)
+class _ApplyChanges(SchemaVisitor[Optional[IcebergType]]):
+ _adds: Dict[int, List[NestedField]]
+ _updates: Dict[int, NestedField]
+ _deletes: Set[int]
+ _moves: Dict[int, List[Move]]
+ def __init__(
+ self, adds: Dict[int, List[NestedField]], updates: Dict[int, NestedField], deletes: Set[int], moves: Dict[int, List[Move]]
+ ) -> None:
+ self._adds = adds
+ self._updates = updates
+ self._deletes = deletes
+ self._moves = moves
-class _ApplyChanges(SchemaVisitor[IcebergType]):
- def __init__(self, adds: Dict[int, List[NestedField]]):
- self.adds = adds
+ def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ if new_fields := _add_fields(struct_result.fields if struct_result else [], self._adds.get(TABLE_ROOT_ID)):
+ return StructType(*new_fields)
+ else:
+ return struct_result
- def schema(self, schema: Schema, struct_result: IcebergType) -> IcebergType:
- fields = _ApplyChanges.add_fields(schema.as_struct().fields, self.adds.get(TABLE_ROOT_ID))
- if len(fields) > 0:
- return StructType(*fields)
+ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]:
+ has_changes = False
+ new_fields = []
- return struct_result
+ for idx, result_type in enumerate(field_results):
+ result_type = field_results[idx]
- def struct(self, struct: StructType, field_results: List[IcebergType]) -> IcebergType:
- has_change = False
- new_fields: List[NestedField] = []
- for i in range(len(field_results)):
- type_: Optional[IcebergType] = field_results[i]
- if type_ is None:
- has_change = True
+ # Has been deleted
+ if result_type is None:
+ has_changes = True
continue
- field: NestedField = struct.fields[i]
- new_fields.append(field)
+ field = struct.fields[idx]
+
+ name = field.name
+ doc = field.doc
+ required = field.required
- if has_change:
+ # There is an update
+ if update := self._updates.get(field.field_id):
+ name = update.name
+ doc = update.doc
+ required = update.required
+
+ if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+ new_fields.append(field)
+ else:
+ has_changes = True
+ new_fields.append(
+ NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+ )
+
+ if has_changes:
return StructType(*new_fields)
return struct
- def field(self, field: NestedField, field_result: IcebergType) -> IcebergType:
- field_id: int = field.field_id
- if field_id in self.adds:
- new_fields = self.adds[field_id]
- if len(new_fields) > 0:
- fields = _ApplyChanges.add_fields(field_result.fields, new_fields)
- if len(fields) > 0:
- return StructType(*fields)
+ def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ # the API validates deletes, updates, and additions don't conflict handle deletes
+ if field.field_id in self._deletes:
+ return None
- return field_result
+ # handle updates
+ if (update := self._updates.get(field.field_id)) and field.field_type != update.field_type:
+ return update.field_type
- def list(self, list_type: ListType, element_result: IcebergType) -> IcebergType:
- element_field: NestedField = list_type.element_field
- element_type = self.field(element_field, element_result)
- if element_type is None:
- raise ValueError(f"Cannot delete element type from list: {element_field}")
+ # handle add & moves
+ added = self._adds.get(field.field_id)
+ moves = self._moves.get(field.field_id)
+ if added is not None or moves is not None:
+ if not isinstance(field.field_type, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {field}")
- is_element_optional = not list_type.element_required
+ if new_fields := _add_and_move_fields(field.field_type.fields, added or [], moves or []):
+ return StructType(*new_fields)
- if is_element_optional == element_field.required and list_type.element_type == element_type:
- return list_type
+ return field_result
+
+ def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ element_type = self.field(list_type.element_field, element_result)
+ if element_type is None:
+ raise ValueError(f"Cannot delete element type from list: {element_result}")
- return ListType(list_type.element_id, element_type, is_element_optional)
+ return ListType(element_id=list_type.element_id, element=element_type, element_required=list_type.element_required)
- def map(self, map_type: MapType, key_result: IcebergType, value_result: IcebergType) -> IcebergType:
+ def map(
+ self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType]
+ ) -> Optional[IcebergType]:
key_id: int = map_type.key_field.field_id
- if key_id in self.adds:
+ if key_id in self._adds:
Review Comment:
Yes, added these as well
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313620950
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
Review Comment:
Nit: `required=False`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313620448
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
Review Comment:
I thought this was suspicious at first, but it's actually correct. Nice fix! We'll have to add the same to Java.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313629522
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move {full_name} after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._table.metadata.last_column_id, new_schema.highest_field_id)
+ updates = (
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ )
+ requirements = (AssertCurrentSchemaId(current_schema_id=self._schema.schema_id),)
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
Review Comment:
Not a problem, but note that if you run multiple schema updates in the same transaction, the transaction needs to ensure it sends only one `AssertCurrentSchemaId`. Right now that will fail but in the future we'll want to suppress the additional ones so that only the first schema ID is validated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313643901
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
Review Comment:
Changing the scale of a decimal is not an allowed promotion.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313626097
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
Review Comment:
top-level field right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314249586
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
- self._adds.setdefault(parent_id, []).append(field)
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, case_sensitive=self._case_sensitive)
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field.field_type}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if other_struct := self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {move.full_name} to a different struct: {other_struct}")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(Move(field_id=field_id, full_name=full_name, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_full_name = (
+ ".".join(
+ before_path,
+ )
+ if isinstance(before_path, tuple)
+ else before_path
+ )
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move {full_name} before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Move the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ full_name = ".".join(path) if isinstance(path, tuple) else path
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_full_name = ".".join(after_name) if isinstance(after_name, tuple) else after_name
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move {full_name} after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(Move(field_id=field_id, full_name=full_name, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def commit(self) -> None:
+ """Apply the pending changes and commit."""
+ new_schema = self._apply()
+
+ if new_schema != self._schema:
+ last_column_id = max(self._table.metadata.last_column_id, new_schema.highest_field_id)
+ updates = (
+ AddSchemaUpdate(schema=new_schema, last_column_id=last_column_id),
+ SetCurrentSchemaUpdate(schema_id=-1),
+ )
+ requirements = (AssertCurrentSchemaId(current_schema_id=self._schema.schema_id),)
+
+ if self._transaction is not None:
+ self._transaction._append_updates(*updates) # pylint: disable=W0212
+ self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ else:
+ self._table._do_commit(updates=updates, requirements=requirements) # pylint: disable=W0212
+
+ def _apply(self) -> Schema:
+ """Apply the pending changes to the original schema and returns the result.
+
+ Returns:
+ the result Schema when all pending updates are applied
+ """
+ struct = visit(self._schema, _ApplyChanges(self._adds, self._updates, self._deletes, self._moves))
+ if struct is None:
+ # Should never happen
+ raise ValueError("Could not apply changes")
+
+ new_schema = Schema(*struct.fields)
+
+ field_ids = set()
+ for name in self._identifier_field_names:
+ try:
+ field = new_schema.find_field(name, self._case_sensitive)
+ except ValueError as e:
+ raise ValueError(
+ f"Cannot find identifier field {name}. In case of deletion, update the identifier fields first."
+ ) from e
+
+ field_ids.add(field.field_id)
+
+ return Schema(*struct.fields, identifier_field_ids=field_ids)
def assign_new_column_id(self) -> int:
return next(self._last_column_id)
-def _apply_changes(schema_: Schema, adds: Dict[int, List[NestedField]], identifier_field_names: List[str]) -> Schema:
- struct = visit(schema_, _ApplyChanges(adds))
- name_to_id: Dict[str, int] = index_by_name(struct)
- for name in identifier_field_names:
- if name not in name_to_id:
- raise ValueError(f"Cannot add field {name} as an identifier field: not found in current schema or added columns")
+class _ApplyChanges(SchemaVisitor[Optional[IcebergType]]):
+ _adds: Dict[int, List[NestedField]]
+ _updates: Dict[int, NestedField]
+ _deletes: Set[int]
+ _moves: Dict[int, List[Move]]
- return Schema(*struct.fields)
+ def __init__(
+ self, adds: Dict[int, List[NestedField]], updates: Dict[int, NestedField], deletes: Set[int], moves: Dict[int, List[Move]]
+ ) -> None:
+ self._adds = adds
+ self._updates = updates
+ self._deletes = deletes
+ self._moves = moves
+ def schema(self, schema: Schema, struct_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ added = self._adds.get(TABLE_ROOT_ID)
+ moves = self._moves.get(TABLE_ROOT_ID)
-class _ApplyChanges(SchemaVisitor[IcebergType]):
- def __init__(self, adds: Dict[int, List[NestedField]]):
- self.adds = adds
+ if added is not None or moves is not None:
+ if not isinstance(struct_result, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {struct_result}")
- def schema(self, schema: Schema, struct_result: IcebergType) -> IcebergType:
- fields = _ApplyChanges.add_fields(schema.as_struct().fields, self.adds.get(TABLE_ROOT_ID))
- if len(fields) > 0:
- return StructType(*fields)
+ if new_fields := _add_and_move_fields(struct_result.fields, added or [], moves or []):
+ return StructType(*new_fields)
return struct_result
- def struct(self, struct: StructType, field_results: List[IcebergType]) -> IcebergType:
- has_change = False
- new_fields: List[NestedField] = []
- for i in range(len(field_results)):
- type_: Optional[IcebergType] = field_results[i]
- if type_ is None:
- has_change = True
+ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]]) -> Optional[IcebergType]:
+ has_changes = False
+ new_fields = []
+
+ for idx, result_type in enumerate(field_results):
+ result_type = field_results[idx]
+
+ # Has been deleted
+ if result_type is None:
+ has_changes = True
continue
- field: NestedField = struct.fields[i]
- new_fields.append(field)
+ field = struct.fields[idx]
+
+ name = field.name
+ doc = field.doc
+ required = field.required
+
+ # There is an update
+ if update := self._updates.get(field.field_id):
+ name = update.name
+ doc = update.doc
+ required = update.required
+
+ if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+ new_fields.append(field)
+ else:
+ has_changes = True
+ new_fields.append(
+ NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+ )
- if has_change:
+ if has_changes:
return StructType(*new_fields)
return struct
- def field(self, field: NestedField, field_result: IcebergType) -> IcebergType:
- field_id: int = field.field_id
- if field_id in self.adds:
- new_fields = self.adds[field_id]
- if len(new_fields) > 0:
- fields = _ApplyChanges.add_fields(field_result.fields, new_fields)
- if len(fields) > 0:
- return StructType(*fields)
+ def field(self, field: NestedField, field_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ # the API validates deletes, updates, and additions don't conflict handle deletes
+ if field.field_id in self._deletes:
+ return None
+
+ # handle updates
+ if (update := self._updates.get(field.field_id)) and field.field_type != update.field_type:
+ return update.field_type
+
+ if isinstance(field_result, StructType):
+ # handle add & moves
+ added = self._adds.get(field.field_id)
+ moves = self._moves.get(field.field_id)
+ if added is not None or moves is not None:
+ if not isinstance(field.field_type, StructType):
+ raise ValueError(f"Cannot add fields to non-struct: {field}")
+
+ if new_fields := _add_and_move_fields(field_result.fields, added or [], moves or []):
+ return StructType(*new_fields)
return field_result
- def list(self, list_type: ListType, element_result: IcebergType) -> IcebergType:
- element_field: NestedField = list_type.element_field
- element_type = self.field(element_field, element_result)
+ def list(self, list_type: ListType, element_result: Optional[IcebergType]) -> Optional[IcebergType]:
+ element_type = self.field(list_type.element_field, element_result)
if element_type is None:
- raise ValueError(f"Cannot delete element type from list: {element_field}")
+ raise ValueError(f"Cannot delete element type from list: {element_result}")
+
+ return ListType(element_id=list_type.element_id, element=element_type, element_required=list_type.element_required)
- is_element_optional = not list_type.element_required
+ def map(
+ self, map_type: MapType, key_result: Optional[IcebergType], value_result: Optional[IcebergType]
+ ) -> Optional[IcebergType]:
+ key_id: int = map_type.key_field.field_id
- if is_element_optional == element_field.required and list_type.element_type == element_type:
- return list_type
+ if key_id in self._deletes:
+ raise ValueError(f"Cannot delete map keys: {map_type}")
- return ListType(list_type.element_id, element_type, is_element_optional)
+ if key_id in self._updates:
+ raise ValueError(f"Cannot update map keys: {map_type}")
- def map(self, map_type: MapType, key_result: IcebergType, value_result: IcebergType) -> IcebergType:
- key_id: int = map_type.key_field.field_id
- if key_id in self.adds:
+ if key_id in self._adds:
raise ValueError(f"Cannot add fields to map keys: {map_type}")
+ if map_type.key_type != key_result:
+ raise ValueError(f"Cannot alter map keys: {map_type}")
+
value_field: NestedField = map_type.value_field
value_type = self.field(value_field, value_result)
if value_type is None:
raise ValueError(f"Cannot delete value type from map: {value_field}")
- is_value_optional = not map_type.value_required
+ return MapType(
+ key_id=map_type.key_id,
+ key_type=map_type.key_type,
+ value_id=map_type.value_id,
+ value_type=value_type,
+ value_required=map_type.value_required,
+ )
- if is_value_optional != value_field.required and map_type.value_type == value_type:
- return map_type
+ def primitive(self, primitive: PrimitiveType) -> Optional[IcebergType]:
+ return primitive
- return MapType(map_type.key_id, map_type.key_field, map_type.value_id, value_type, not is_value_optional)
- def primitive(self, primitive: PrimitiveType) -> IcebergType:
- return primitive
+def _add_fields(fields: Tuple[NestedField, ...], adds: Optional[List[NestedField]]) -> Tuple[NestedField, ...]:
+ adds = adds or []
+ return fields + tuple(adds)
+
- @staticmethod
- def add_fields(fields: Tuple[NestedField, ...], adds: Optional[List[NestedField]]) -> List[NestedField]:
- new_fields: List[NestedField] = []
- new_fields.extend(fields)
- if adds:
- new_fields.extend(adds)
- return new_fields
+def _move_fields(fields: Tuple[NestedField, ...], moves: List[Move]) -> Tuple[NestedField, ...]:
+ reordered = list(copy(fields))
+ for move in moves:
+ # Find the field that we're about to move
+ field = next(field for field in reordered if field.field_id == move.field_id)
+ # Remove the field that we're about to move from the list
+ reordered = [field for field in reordered if field.field_id != move.field_id]
+
+ if move.op == MoveOperation.First:
+ reordered = [field] + reordered
+ elif move.op == MoveOperation.Before or move.op == MoveOperation.After:
+ other_field_id = move.other_field_id
+ other_field_pos = next(i for i, field in enumerate(reordered) if field.field_id == other_field_id)
+ if move.op == MoveOperation.Before:
+ reordered.insert(other_field_pos, field)
+ else:
+ reordered.insert(other_field_pos + 1, field)
+ else:
+ raise ValueError(f"Unknown operation: {move.op}")
+
+ return tuple(reordered)
+
+
+def _add_and_move_fields(
+ fields: Tuple[NestedField, ...], adds: List[NestedField], moves: List[Move]
+) -> Optional[Tuple[NestedField, ...]]:
+ if adds:
Review Comment:
This will check the length. I prefer to pass in empty lists instead of None's, but I think we need to update `adds` to `len(adds) > 0` to make it more explicit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314249840
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
Review Comment:
Added those using search-replace 🙈
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
Review Comment:
Added those using search-replace 🙈
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314246550
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
Review Comment:
I agree, I removed the other fields
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314245324
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type or updated.field_type,
+ doc=doc or updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type or field.field_type,
+ doc=doc or field.doc,
+ required=field.required,
+ )
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ if required is not None:
+ self._set_column_requirement(path, required=required)
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
+
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
Review Comment:
I'll clean them up, less is more
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314310373
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
+ # DecimalType(9, 2),
+ # DecimalType(9, 3),
+ # DecimalType(18, 2)
+]
+
+
+@pytest.mark.parametrize("from_type", disallowed_promotions_types, ids=str)
+@pytest.mark.parametrize("to_type", disallowed_promotions_types, ids=str)
+@pytest.mark.integration
+def test_disallowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ if from_type != to_type and (from_type, to_type) not in allowed_promotions:
+ with pytest.raises(ValidationError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert str(exc_info.value).startswith("Cannot change column type: bar:")
+ else:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=to_type, required=True),
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.rename_column("foo", "vo")
+
+ assert simple_table.schema() == Schema(
+ NestedField(field_id=1, name="vo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(
+ field_id=1,
+ name="foo",
+ field_type=StructType(NestedField(field_id=2, name="bar", field_type=StringType())),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.rename_column("foo.bar", "vo")
+
+ assert tbl.schema() == Schema(
+ NestedField(
+ field_id=1,
+ name="foo",
+ field_type=StructType(NestedField(field_id=2, name="vo", field_type=StringType())),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_rename(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=5,
+ key_type=StringType(),
+ value_id=6,
+ value_type=StructType(
+ NestedField(field_id=7, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=8, name="y", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=9,
+ element_type=StructType(
+ NestedField(field_id=10, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=11, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=12, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=13, name="leeftijd", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ NestedField(field_id=4, name="foo", field_type=StringType(), required=True),
+ schema_id=0,
+ identifier_field_ids=[],
Review Comment:
This was a great test case. With case-insensitive the identifier-field-ids were not properly updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305005552
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
+ if parent_name := self._id_to_parent.get(move.field_id):
+ parent_field = self._schema.find_field(parent_name, self._case_sensitive)
+ if not parent_field.is_struct:
+ raise ValueError(f"Cannot move fields in non-struct type: {parent_field}")
+
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.field_id) != self._id_to_parent.get(move.other_field_id):
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[parent_field.field_id] = self._moves.get(parent_field.field_id, []) + [move]
+ else:
+ if move.op == MoveOperation.After or move.op == MoveOperation.Before:
+ if move.other_field_id is None:
+ raise ValueError("Expected other field when performing before/after move")
+
+ if self._id_to_parent.get(move.other_field_id) is not None:
+ raise ValueError(f"Cannot move field {full_name} to a different struct")
+
+ self._moves[TABLE_ROOT_ID] = self._moves.get(TABLE_ROOT_ID, []) + [move]
+
+ def move_first(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to the first position of the parent struct.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ self._move(full_name, Move(field_id=field_id, op=MoveOperation.First))
+
+ return self
+
+ def move_before(self, path: Union[str, Tuple[str, ...]], before_path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to before another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ before_path = (before_path,) if isinstance(before_path, str) else before_path
+ before_full_name = ".".join(before_path)
+ before_field_id = self._find_for_move(before_full_name)
+
+ if before_field_id is None:
+ raise ValueError(f"Cannot move before missing column: {before_full_name}")
+
+ if field_id == before_field_id:
+ raise ValueError(f"Cannot move {full_name} before itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=before_field_id, op=MoveOperation.Before))
+
+ return self
+
+ def move_after(self, path: Union[str, Tuple[str, ...]], after_name: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Moves the field to after another field.
+
+ Args:
+ path: The path to the field.
+
+ Returns:
+ The UpdateSchema with the move operation staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field_id = self._find_for_move(full_name)
+
+ if field_id is None:
+ raise ValueError(f"Cannot move missing column: {full_name}")
+
+ after_path = (after_name,) if isinstance(after_name, str) else after_name
+ after_full_name = ".".join(after_path)
+ after_field_id = self._find_for_move(after_full_name)
+
+ if after_field_id is None:
+ raise ValueError(f"Cannot move after missing column: {after_full_name}")
+
+ if field_id == after_field_id:
+ raise ValueError(f"Cannot move {full_name} after itself")
+
+ self._move(full_name, Move(field_id=field_id, other_field_id=after_field_id, op=MoveOperation.After))
+
+ return self
+
+ def allow_incompatible_changes(self) -> UpdateSchema:
Review Comment:
I think it is probably cleaner not to set this using an API method. If it is only set in the constructor then it must be constant for the entire update operation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306714104
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
Review Comment:
This is in Python as well, but we need to full path for the identifier columns 👍🏻
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
Review Comment:
This is in Python as well, but we need to full path for the identifier columns 👍🏻
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306714196
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
Review Comment:
We had the same train of thought, I've updated this to:
```python
if path_from in self._identifier_field_names:
self._identifier_field_names.remove(path_from)
self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305003741
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
+ pass
+
+ return self._added_name_to_id.get(name)
+
+ def _move(self, full_name: str, move: Move) -> None:
Review Comment:
Why not keep `full_name` in `Move`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305002926
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
Review Comment:
Flagging this again. Seems weird for a `find` method to raise `ValueError`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305002566
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
Review Comment:
Rather than having separate methods like the Java implementation, what about using a combined `update_column` with named parameters?
```python
def update_column(self, path: Union[str, Tuple[str, ...]], field_type: Optional[IcebergType] = None, required: Optional[boolean] = None, doc: Optional[str] = None) -> UpdateSchema:
# find the column ...
field_type = field_type if field_type is None else field.field_type
doc = doc if doc is None else field.doc
required = required if required is None else field.required
...
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1304998154
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
Review Comment:
I don't think this should split. The `from` field is going to be looked up by full name:
```python
findField(("a", "b").join(".")) == findField(("a.b",).join("."))
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313625687
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
Review Comment:
Since there are other methods to add a new change or update the pending change for `required` and `doc`, I think this should go inside the `field_type is not None` branch above. That way, each change is independently checked to see if it is necessary and no change is created if it is not.
The way it currently works, there will always be an update created, even if `field_type` does not change because this is outside of the checks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313633583
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, required=True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Make a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def set_identifier_fields(self, *fields: str) -> None:
+ self._identifier_field_names = set(fields)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ field = self._schema.find_field(name, self._case_sensitive)
- full_name = parent_field.name + "." + name
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
+ if not self._allow_incompatible_changes and required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
+
+ def update_column(
+ self,
+ path: Union[str, Tuple[str, ...]],
+ field_type: Optional[IcebergType] = None,
+ required: Optional[bool] = None,
+ doc: Optional[str] = None,
+ ) -> UpdateSchema:
+ """Update the type of column.
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ Args:
+ path: The path to the field.
+ field_type: The new type
+ required: If the field should be required
+ doc: Documentation describing the column
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field_type is not None:
+ if not self._allow_incompatible_changes and field.field_type != field_type:
Review Comment:
The Java equivalent ensures that only primitive fields are updated because the only type that can be passed in is a `PrimitiveType`. I think we need to have an additional check here that this is not replacing a nested type.
It may be that `promote` already does this for us, but I want to highlight that we cannot allow calling this to update a `struct`, `map`, or `list` type.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313642172
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
Review Comment:
Each schema change should produce a new schema ID in table metadata. If you were testing just `_apply` then this would be correct, but since this is actually committed, the schema ID needs to be assigned for every change.
@Fokko, this is a blocker.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313639350
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
Review Comment:
Looks like this may not need to be done in an integration test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1314250558
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
Review Comment:
![Uploading image.png…]()
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
Review Comment:
![Uploading image.png…]()
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
Review Comment:
<img width="1512" alt="image" src="https://github.com/apache/iceberg/assets/1134248/77a0f7d7-df77-48fc-8310-cbee6b95d9b0">
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#issuecomment-1704412018
I accidentally tagged this PR in another PR that was closed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313620802
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +976,630 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
+ Because "." may be interpreted as a column path separator or may be used in field names, it
+ is not allowed to add nested column by passing in a string. To add to nested structures or
+ to add fields with names that contain "." use a tuple instead to indicate the path.
+
+ If type is a nested type, its field IDs are reassigned when added to the existing schema.
+
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ if isinstance(path, str):
+ if "." in path:
+ raise ValueError(f"Cannot add column with ambiguous name: {path}, provide a tuple instead")
+ path = (path,)
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_full_path = ".".join(parent)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(parent_full_path, self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ existing_field = None
+ try:
+ existing_field = self._schema.find_field(full_name, self._case_sensitive)
+ except ValueError:
+ pass
+
+ if existing_field is not None and existing_field.field_id not in self._deletes:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+ self._id_to_parent[new_id] = parent_full_path
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ if parent_id in self._adds:
+ self._adds[parent_id].append(field)
+ else:
+ self._adds[parent_id] = [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Delete a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, case_sensitive=self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Update the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ path_from = ".".join(path_from) if isinstance(path_from, tuple) else path_from
+ field_from = self._schema.find_field(path_from, self._case_sensitive)
+
+ if field_from.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {path_from}")
+
+ if updated := self._updates.get(field_from.field_id):
+ self._updates[field_from.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[field_from.field_id] = NestedField(
+ field_id=field_from.field_id,
+ name=new_name,
+ field_type=field_from.field_type,
+ doc=field_from.doc,
+ required=field_from.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(path_from)
+ self._identifier_field_names.add(f"{path_from[:-len(field_from.name)]}{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
Review Comment:
Isn't this `update_column('a', required=True)`? Is this intended to be a nicer way to set that?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1313645255
##########
python/tests/test_integration_schema.py:
##########
@@ -0,0 +1,2617 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint:disable=redefined-outer-name
+from typing import Dict
+
+import pytest
+
+from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
+from pyiceberg.schema import Schema, prune_columns
+from pyiceberg.table import Table, UpdateSchema
+from pyiceberg.types import (
+ BinaryType,
+ BooleanType,
+ DateType,
+ DecimalType,
+ DoubleType,
+ FixedType,
+ FloatType,
+ IntegerType,
+ ListType,
+ LongType,
+ MapType,
+ NestedField,
+ PrimitiveType,
+ StringType,
+ StructType,
+ TimestampType,
+ TimestamptzType,
+ TimeType,
+ UUIDType,
+)
+
+
+@pytest.fixture()
+def catalog() -> Catalog:
+ return load_catalog(
+ "local",
+ **{
+ "type": "rest",
+ "uri": "http://localhost:8181",
+ "s3.endpoint": "http://localhost:9000",
+ "s3.access-key-id": "admin",
+ "s3.secret-access-key": "password",
+ },
+ )
+
+
+@pytest.fixture()
+def simple_table(catalog: Catalog, table_schema_simple: Schema) -> Table:
+ return _create_table_with_schema(catalog, table_schema_simple)
+
+
+@pytest.mark.integration
+def test_add_column(simple_table: Table) -> None:
+ update = UpdateSchema(simple_table)
+ update.add_column(path="b", field_type=IntegerType())
+ apply_schema: Schema = update._apply() # pylint: disable=W0212
+ assert len(apply_schema.fields) == 4
+
+ assert apply_schema == Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(field_id=4, name="b", field_type=IntegerType(), required=False),
+ identifier_field_ids=[2],
+ )
+ assert apply_schema.schema_id == 0
+ assert apply_schema.highest_field_id == 4
+
+
+@pytest.mark.integration
+def test_add_primitive_type_column(simple_table: Table) -> None:
+ primitive_type: Dict[str, PrimitiveType] = {
+ "boolean": BooleanType(),
+ "int": IntegerType(),
+ "long": LongType(),
+ "float": FloatType(),
+ "double": DoubleType(),
+ "date": DateType(),
+ "time": TimeType(),
+ "timestamp": TimestampType(),
+ "timestamptz": TimestamptzType(),
+ "string": StringType(),
+ "uuid": UUIDType(),
+ "binary": BinaryType(),
+ }
+
+ for name, type_ in primitive_type.items():
+ field_name = f"new_column_{name}"
+ update = UpdateSchema(simple_table)
+ update.add_column(path=field_name, field_type=type_, doc=f"new_column_{name}")
+ new_schema = update._apply() # pylint: disable=W0212
+
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == type_
+ assert field.doc == f"new_column_{name}"
+
+
+@pytest.mark.integration
+def test_add_nested_type_column(simple_table: Table) -> None:
+ # add struct type column
+ field_name = "new_column_struct"
+ update = UpdateSchema(simple_table)
+ struct_ = StructType(
+ NestedField(1, "lat", DoubleType()),
+ NestedField(2, "long", DoubleType()),
+ )
+ update.add_column(path=field_name, field_type=struct_)
+ schema_ = update._apply() # pylint: disable=W0212
+ field: NestedField = schema_.find_field(field_name)
+ assert field.field_type == StructType(
+ NestedField(5, "lat", DoubleType()),
+ NestedField(6, "long", DoubleType()),
+ )
+ assert schema_.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_map_type_column(simple_table: Table) -> None:
+ # add map type column
+ field_name = "new_column_map"
+ update = UpdateSchema(simple_table)
+ map_ = MapType(1, StringType(), 2, IntegerType(), False)
+ update.add_column(path=field_name, field_type=map_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == MapType(5, StringType(), 6, IntegerType(), False)
+ assert new_schema.highest_field_id == 6
+
+
+@pytest.mark.integration
+def test_add_nested_list_type_column(simple_table: Table) -> None:
+ # add list type column
+ field_name = "new_column_list"
+ update = UpdateSchema(simple_table)
+ list_ = ListType(
+ element_id=101,
+ element_type=StructType(
+ NestedField(102, "lat", DoubleType()),
+ NestedField(103, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ update.add_column(path=field_name, field_type=list_)
+ new_schema = update._apply() # pylint: disable=W0212
+ field: NestedField = new_schema.find_field(field_name)
+ assert field.field_type == ListType(
+ element_id=5,
+ element_type=StructType(
+ NestedField(6, "lat", DoubleType()),
+ NestedField(7, "long", DoubleType()),
+ ),
+ element_required=False,
+ )
+ assert new_schema.highest_field_id == 7
+
+
+def _create_table_with_schema(catalog: Catalog, schema: Schema) -> Table:
+ tbl_name = "default.test_schema_evolution"
+ try:
+ catalog.drop_table(tbl_name)
+ except NoSuchTableError:
+ pass
+ return catalog.create_table(identifier=tbl_name, schema=schema)
+
+
+@pytest.mark.integration
+def test_add_already_exists(catalog: Catalog, table_schema_nested: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_nested)
+ update = UpdateSchema(table)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column("foo", IntegerType())
+ assert "already exists: foo" in str(exc_info.value)
+
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("location", "latitude"), field_type=IntegerType())
+ assert "already exists: location.latitude" in str(exc_info.value)
+
+
+@pytest.mark.integration
+def test_add_to_non_struct_type(catalog: Catalog, table_schema_simple: Schema) -> None:
+ table = _create_table_with_schema(catalog, table_schema_simple)
+ update = UpdateSchema(table)
+ with pytest.raises(ValueError) as exc_info:
+ update.add_column(path=("foo", "lat"), field_type=IntegerType())
+ assert "Cannot add column 'lat' to non-struct type" in str(exc_info.value)
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
+ schema = Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ )
+ tbl = _create_table_with_schema(catalog, schema)
+
+ assert tbl.schema() == schema
+
+ with tbl.transaction() as tx:
+ tx.update_schema().add_column("col_string", StringType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ schema_id=1,
+ )
+
+ tbl.update_schema().add_column("col_integer", IntegerType()).commit()
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ schema_id=1,
+ )
+
+ with pytest.raises(CommitFailedException) as exc_info:
+ with tbl.transaction() as tx:
+ # Start a new update
+ schema_update = tx.update_schema()
+
+ # Do a concurrent update
+ tbl.update_schema().add_column("col_long", LongType()).commit()
+
+ # stage another update in the transaction
+ schema_update.add_column("col_double", DoubleType()).commit()
+
+ assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
+ NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),
+ NestedField(field_id=3, name="col_string", field_type=StringType(), required=False),
+ NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
+ NestedField(field_id=5, name="col_long", field_type=LongType(), required=False),
+ schema_id=1,
+ )
+
+
+@pytest.mark.integration
+@pytest.mark.integration
+def test_schema_evolution_nested(catalog: Catalog) -> None:
+ nested_schema = Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ key_id=10,
+ key_type=StringType(),
+ value_id=11,
+ value_type=StructType(
+ NestedField(field_id=110, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=111, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ element_id=20,
+ element_type=StructType(
+ NestedField(field_id=200, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=201, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=30, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=31, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ )
+
+ tbl = _create_table_with_schema(catalog, nested_schema)
+
+ assert tbl.schema().highest_field_id == 12
+
+ with tbl.update_schema() as schema_update:
+ schema_update.add_column(("location_lookup", "z"), FloatType())
+ schema_update.add_column(("locations", "z"), FloatType())
+ schema_update.add_column(("person", "address"), StringType())
+
+ assert str(tbl.schema()) == str(
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=4,
+ key_type=StringType(),
+ value_id=5,
+ value_type=StructType(
+ NestedField(field_id=6, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=7, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=13, name="z", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=8,
+ element_type=StructType(
+ NestedField(field_id=9, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=10, name="y", field_type=FloatType(), required=False),
+ NestedField(field_id=14, name="z", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=11, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=12, name="age", field_type=IntegerType(), required=True),
+ NestedField(field_id=15, name="address", field_type=StringType(), required=False),
+ ),
+ required=False,
+ ),
+ schema_id=1,
+ identifier_field_ids=[],
+ )
+ )
+
+
+schema_nested = Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ NestedField(
+ field_id=4,
+ name="qux",
+ field_type=ListType(type="list", element_id=8, element_type=StringType(), element_required=True),
+ required=True,
+ ),
+ NestedField(
+ field_id=5,
+ name="quux",
+ field_type=MapType(
+ type="map",
+ key_id=9,
+ key_type=StringType(),
+ value_id=10,
+ value_type=MapType(
+ type="map", key_id=11, key_type=StringType(), value_id=12, value_type=IntegerType(), value_required=True
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=6,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=13,
+ element_type=StructType(
+ NestedField(field_id=14, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=15, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=7,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=16, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=17, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ schema_id=0,
+ identifier_field_ids=[2],
+)
+
+
+@pytest.fixture()
+def nested_table(catalog: Catalog) -> Table:
+ return _create_table_with_schema(catalog, schema_nested)
+
+
+@pytest.mark.integration
+def test_no_changes(simple_table: Table, table_schema_simple: Schema) -> None:
+ with simple_table.update_schema() as _:
+ pass
+
+ assert simple_table.schema() == table_schema_simple
+
+
+@pytest.mark.integration
+def test_delete_field(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("foo")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_field_case_insensitive(simple_table: Table) -> None:
+ with simple_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column("FOO")
+
+ assert simple_table.schema() == Schema(
+ # foo is missing 👍
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields(simple_table: Table) -> None:
+ with pytest.raises(ValueError) as exc_info:
+ with simple_table.update_schema() as schema_update:
+ schema_update.delete_column("bar")
+
+ assert "Cannot find identifier field bar. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.integration
+def test_delete_identifier_fields_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="foo", field_type=StringType(), required=False),
+ NestedField(
+ field_id=2,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=3, name="name", field_type=StringType(), required=True),
+ NestedField(field_id=4, name="age", field_type=IntegerType(), required=True),
+ ),
+ required=True,
+ ),
+ schema_id=1,
+ identifier_field_ids=[3],
+ ),
+ )
+
+ with pytest.raises(ValueError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.delete_column("person")
+
+ assert "Cannot find identifier field person.name. In case of deletion, update the identifier fields first." in str(exc_info)
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "foo",
+ "baz",
+ "qux",
+ "quux",
+ "location",
+ "location.element.latitude",
+ "location.element.longitude",
+ "person",
+ "person.name",
+ "person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema() as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.parametrize(
+ "field",
+ [
+ "Foo",
+ "Baz",
+ "Qux",
+ "Quux",
+ "Location",
+ "Location.element.latitude",
+ "Location.element.longitude",
+ "Person",
+ "Person.name",
+ "Person.age",
+ ],
+)
+@pytest.mark.integration
+def test_deletes_case_insensitive(field: str, nested_table: Table) -> None:
+ with nested_table.update_schema(case_sensitive=False) as schema_update:
+ schema_update.delete_column(field)
+
+ selected_ids = {
+ field_id
+ for field_id in schema_nested.field_ids
+ if not isinstance(schema_nested.find_field(field_id).field_type, (MapType, ListType))
+ and not schema_nested.find_column_name(field_id).startswith(field.lower()) # type: ignore
+ }
+ expected_schema = prune_columns(schema_nested, selected_ids, select_full_types=False)
+
+ assert expected_schema == nested_table.schema()
+
+
+@pytest.mark.integration
+def test_update_types(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", LongType())
+ schema_update.update_column("location.latitude", DoubleType())
+ schema_update.update_column("location.longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_update_types_case_insensitive(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=IntegerType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=FloatType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema(case_sensitive=False) as schema_update:
+ schema_update.update_column("baR", LongType())
+ schema_update.update_column("Location.Latitude", DoubleType())
+ schema_update.update_column("Location.Longitude", DoubleType())
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=LongType(), required=True),
+ NestedField(
+ field_id=2,
+ name="location",
+ field_type=ListType(
+ type="list",
+ element_id=3,
+ element_type=StructType(
+ NestedField(field_id=4, name="latitude", field_type=DoubleType(), required=False),
+ NestedField(field_id=5, name="longitude", field_type=DoubleType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ )
+
+
+allowed_promotions = [
+ (StringType(), BinaryType()),
+ (BinaryType(), StringType()),
+ (IntegerType(), LongType()),
+ (FloatType(), DoubleType()),
+ (DecimalType(9, 2), DecimalType(18, 2)),
+]
+
+
+@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
+@pytest.mark.integration
+def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))
+
+
+disallowed_promotions_types = [
+ BooleanType(),
+ IntegerType(),
+ LongType(),
+ FloatType(),
+ DoubleType(),
+ DateType(),
+ TimeType(),
+ TimestampType(),
+ TimestamptzType(),
+ StringType(),
+ UUIDType(),
+ BinaryType(),
+ FixedType(3),
+ FixedType(4),
+ # We'll just allow Decimal promotions right now
+ # https://github.com/apache/iceberg/issues/8389
+ # DecimalType(9, 2),
+ # DecimalType(9, 3),
+ # DecimalType(18, 2)
+]
+
+
+@pytest.mark.parametrize("from_type", disallowed_promotions_types, ids=str)
+@pytest.mark.parametrize("to_type", disallowed_promotions_types, ids=str)
+@pytest.mark.integration
+def test_disallowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(field_id=1, name="bar", field_type=from_type, required=True),
+ ),
+ )
+
+ if from_type != to_type and (from_type, to_type) not in allowed_promotions:
+ with pytest.raises(ValidationError) as exc_info:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert str(exc_info.value).startswith("Cannot change column type: bar:")
+ else:
+ with tbl.update_schema() as schema_update:
+ schema_update.update_column("bar", to_type)
+
+ assert tbl.schema() == Schema(
+ NestedField(field_id=1, name="bar", field_type=to_type, required=True),
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple(simple_table: Table) -> None:
+ with simple_table.update_schema() as schema_update:
+ schema_update.rename_column("foo", "vo")
+
+ assert simple_table.schema() == Schema(
+ NestedField(field_id=1, name="vo", field_type=StringType(), required=False),
+ NestedField(field_id=2, name="bar", field_type=IntegerType(), required=True),
+ NestedField(field_id=3, name="baz", field_type=BooleanType(), required=False),
+ schema_id=1,
+ identifier_field_ids=[2],
+ )
+
+
+@pytest.mark.integration
+def test_rename_simple_nested(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(
+ field_id=1,
+ name="foo",
+ field_type=StructType(NestedField(field_id=2, name="bar", field_type=StringType())),
+ required=True,
+ ),
+ ),
+ )
+
+ with tbl.update_schema() as schema_update:
+ schema_update.rename_column("foo.bar", "vo")
+
+ assert tbl.schema() == Schema(
+ NestedField(
+ field_id=1,
+ name="foo",
+ field_type=StructType(NestedField(field_id=2, name="vo", field_type=StringType())),
+ required=True,
+ ),
+ )
+
+
+@pytest.mark.integration
+def test_rename(catalog: Catalog) -> None:
+ tbl = _create_table_with_schema(
+ catalog,
+ Schema(
+ NestedField(
+ field_id=1,
+ name="location_lookup",
+ field_type=MapType(
+ type="map",
+ key_id=5,
+ key_type=StringType(),
+ value_id=6,
+ value_type=StructType(
+ NestedField(field_id=7, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=8, name="y", field_type=FloatType(), required=False),
+ ),
+ value_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=2,
+ name="locations",
+ field_type=ListType(
+ type="list",
+ element_id=9,
+ element_type=StructType(
+ NestedField(field_id=10, name="x", field_type=FloatType(), required=False),
+ NestedField(field_id=11, name="y", field_type=FloatType(), required=False),
+ ),
+ element_required=True,
+ ),
+ required=True,
+ ),
+ NestedField(
+ field_id=3,
+ name="person",
+ field_type=StructType(
+ NestedField(field_id=12, name="name", field_type=StringType(), required=False),
+ NestedField(field_id=13, name="leeftijd", field_type=IntegerType(), required=True),
+ ),
+ required=False,
+ ),
+ NestedField(field_id=4, name="foo", field_type=StringType(), required=True),
+ schema_id=0,
+ identifier_field_ids=[],
Review Comment:
Could add identifier fields to test nested identifier rename here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] rdblue commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "rdblue (via GitHub)" <gi...@apache.org>.
rdblue commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1305000992
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
Review Comment:
I don't think this invocation is clear. When passing a boolean arg, can you use a named argument?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306716402
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
+
+ new_type = assign_fresh_schema_ids(field_type, self.assign_new_column_id)
+ field = NestedField(field_id=new_id, name=name, field_type=new_type, required=required, doc=doc)
+
+ self._adds[parent_id] = self._adds.get(parent_id, []) + [field]
- self._internal_add_column(parent, name, not required, type_var, doc)
return self
- def allow_incompatible_changes(self) -> UpdateSchema:
- """Allow incompatible changes to the schema.
+ def delete_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Deletes a column from a table.
+
+ Args:
+ path: The path to the column.
Returns:
- This for method chaining
+ The UpdateSchema with the delete operation staged.
"""
- self._allow_incompatible_changes = True
+ name = (path,) if isinstance(path, str) else path
+ full_name = ".".join(name)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._adds:
+ raise ValueError(f"Cannot delete a column that has additions: {full_name}")
+ if field.field_id in self._updates:
+ raise ValueError(f"Cannot delete a column that has updates: {full_name}")
+
+ self._deletes.add(field.field_id)
+
return self
- def commit(self) -> None:
- """Apply the pending changes and commit."""
- new_schema = self._apply()
- updates = [
- AddSchemaUpdate(schema=new_schema, last_column_id=new_schema.highest_field_id),
- SetCurrentSchemaUpdate(schema_id=-1),
- ]
- requirements = [AssertCurrentSchemaId(current_schema_id=self._schema.schema_id)]
+ def rename_column(self, path_from: Union[str, Tuple[str, ...]], new_name: str) -> UpdateSchema:
+ """Updates the name of a column.
+
+ Args:
+ path_from: The path to the column to be renamed.
+ new_name: The new path of the column.
+
+ Returns:
+ The UpdateSchema with the rename operation staged.
+ """
+ name_from = tuple(path_from.split(".")) if isinstance(path_from, str) else path_from
+ new_name = new_name.split(".")[-1]
- if self._transaction is not None:
- self._transaction._append_updates(*updates) # pylint: disable=W0212
- self._transaction._append_requirements(*requirements) # pylint: disable=W0212
+ parent_path = name_from[:-1]
+ full_name_from = ".".join(name_from)
+
+ from_field = self._schema.find_field(full_name_from, self._case_sensitive)
+
+ if from_field.field_id in self._deletes:
+ raise ValueError(f"Cannot rename a column that will be deleted: {full_name_from}")
+
+ if updated := self._updates.get(from_field.field_id):
+ self._updates[from_field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=new_name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
else:
- table_update_response = self._table.catalog._commit_table( # pylint: disable=W0212
- CommitTableRequest(identifier=self._table.identifier[1:], updates=updates, requirements=requirements)
+ self._updates[from_field.field_id] = NestedField(
+ field_id=from_field.field_id,
+ name=new_name,
+ field_type=from_field.field_type,
+ doc=from_field.doc,
+ required=from_field.required,
)
- self._table.metadata = table_update_response.metadata
- self._table.metadata_location = table_update_response.metadata_location
- def _apply(self) -> Schema:
- """Apply the pending changes to the original schema and returns the result.
+ if path_from in self._identifier_field_names:
+ self._identifier_field_names.remove(full_name_from)
+ self._identifier_field_names.add(f"{'.'.join(parent_path)}.{new_name}")
+
+ return self
+
+ def require_column(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column required.
+
+ This is a breaking change since writers have to make sure that
+ this value is not-null.
+
+ Args:
+ path: The path to the field
Returns:
- the result Schema when all pending updates are applied
+ The UpdateSchema with the requirement change staged.
"""
- return _apply_changes(self._schema, self._adds, self._identifier_field_names)
+ self._set_column_requirement(path, True)
+ return self
- def _internal_add_column(
- self, parent: Optional[str], name: str, is_optional: bool, type_var: IcebergType, doc: Optional[str]
- ) -> None:
- full_name: str = name
- parent_id: int = TABLE_ROOT_ID
+ def make_column_optional(self, path: Union[str, Tuple[str, ...]]) -> UpdateSchema:
+ """Makes a column optional.
- exist_field: Optional[NestedField] = None
- if parent:
- parent_field = self._schema.find_field(parent, self._case_sensitive)
- parent_type = parent_field.field_type
- if isinstance(parent_type, MapType):
- parent_field = parent_type.value_field
- elif isinstance(parent_type, ListType):
- parent_field = parent_type.element_field
+ Args:
+ path: The path to the field.
- if not parent_field.field_type.is_struct:
- raise ValueError(f"Cannot add column to non-struct type: {parent}")
+ Returns:
+ The UpdateSchema with the requirement change staged.
+ """
+ self._set_column_requirement(path, False)
+ return self
- parent_id = parent_field.field_id
+ def _set_column_requirement(self, path: Union[str, Tuple[str, ...]], required: bool) -> None:
+ path = (path,) if isinstance(path, str) else path
+ name = ".".join(path)
+
+ field = self._schema.find_field(name, self._case_sensitive)
- try:
- exist_field = self._schema.find_field(parent + "." + name, self._case_sensitive)
- except ValueError:
- pass
+ if (field.required and required) or (field.optional and not required):
+ # if the change is a noop, allow it even if allowIncompatibleChanges is false
+ return
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {parent}.{name}")
+ if self._allow_incompatible_changes and not required:
+ raise ValueError(f"Cannot change column nullability: {name}: optional -> required")
- full_name = parent_field.name + "." + name
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {name}")
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=updated.doc,
+ required=required,
+ )
else:
- try:
- exist_field = self._schema.find_field(name, self._case_sensitive)
- except ValueError:
- pass
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=field.doc,
+ required=required,
+ )
- if exist_field:
- raise ValueError(f"Cannot add column, name already exists: {name}")
+ def update_column(self, path: Union[str, Tuple[str, ...]], field_type: IcebergType) -> UpdateSchema:
+ """Update the type of column.
- # assign new IDs in order
- new_id = self.assign_new_column_id()
+ Args:
+ path: The path to the field.
+ field_type: The new type
- # update tracking for moves
- self._added_name_to_id[full_name] = new_id
+ Returns:
+ The UpdateSchema with the type update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.field_type == field_type:
+ # Nothing changed
+ return self
+
+ try:
+ promote(field.field_type, field_type)
+ except ResolveError as e:
+ raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=field_type,
+ doc=updated.doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field_type,
+ doc=field.doc,
+ required=field.required,
+ )
- new_type = assign_fresh_schema_ids(type_var, self.assign_new_column_id)
- field = NestedField(new_id, name, new_type, not is_optional, doc)
+ return self
- self._adds.setdefault(parent_id, []).append(field)
+ def update_column_doc(self, path: Union[str, Tuple[str, ...]], doc: str) -> UpdateSchema:
+ """Update the documentation of column.
+
+ Args:
+ path: The path to the field.
+ doc: The new documentation of the column
+
+ Returns:
+ The UpdateSchema with the doc update staged.
+ """
+ path = (path,) if isinstance(path, str) else path
+ full_name = ".".join(path)
+
+ field = self._schema.find_field(full_name, self._case_sensitive)
+
+ if field.field_id in self._deletes:
+ raise ValueError(f"Cannot update a column that will be deleted: {full_name}")
+
+ if field.doc == doc:
+ # Noop
+ return self
+
+ if updated := self._updates.get(field.field_id):
+ self._updates[field.field_id] = NestedField(
+ field_id=updated.field_id,
+ name=updated.name,
+ field_type=updated.field_type,
+ doc=doc,
+ required=updated.required,
+ )
+ else:
+ self._updates[field.field_id] = NestedField(
+ field_id=field.field_id,
+ name=field.name,
+ field_type=field.field_type,
+ doc=doc,
+ required=field.required,
+ )
+
+ return self
+
+ def _find_for_move(self, name: str) -> Optional[int]:
+ try:
+ return self._schema.find_field(name, self._case_sensitive).field_id
+ except ValueError:
Review Comment:
See other comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org
[GitHub] [iceberg] Fokko commented on a diff in pull request #8374: Python: Add updates, moves and deletes
Posted by "Fokko (via GitHub)" <gi...@apache.org>.
Fokko commented on code in PR #8374:
URL: https://github.com/apache/iceberg/pull/8374#discussion_r1306717140
##########
python/pyiceberg/table/__init__.py:
##########
@@ -934,206 +963,611 @@ def case_sensitive(self, case_sensitive: bool) -> UpdateSchema:
return self
def add_column(
- self, name: str, type_var: IcebergType, doc: Optional[str] = None, parent: Optional[str] = None, required: bool = False
+ self, path: Union[str, Tuple[str, ...]], field_type: IcebergType, doc: Optional[str] = None, required: bool = False
) -> UpdateSchema:
"""Add a new column to a nested struct or Add a new top-level column.
Args:
- name: Name for the new column.
- type_var: Type for the new column.
+ path: Name for the new column.
+ field_type: Type for the new column.
doc: Documentation string for the new column.
- parent: Name of the parent struct to the column will be added to.
required: Whether the new column is required.
Returns:
- This for method chaining
+ This for method chaining.
"""
- if "." in name:
- raise ValueError(f"Cannot add column with ambiguous name: {name}")
+ path = (path,) if isinstance(path, str) else path
+
+ if "." in path[-1]:
+ raise ValueError(f"Cannot add column with ambiguous name: {path[-1]}, provide a tuple instead")
if required and not self._allow_incompatible_changes:
# Table format version 1 and 2 cannot add required column because there is no initial value
- raise ValueError(f"Incompatible change: cannot add required column: {name}")
+ raise ValueError(f'Incompatible change: cannot add required column: {".".join(path)}')
+
+ name = path[-1]
+ parent = path[:-1]
+
+ full_name = ".".join(path)
+ parent_id: int = TABLE_ROOT_ID
+
+ if len(parent) > 0:
+ parent_field = self._schema.find_field(".".join(parent), self._case_sensitive)
+ parent_type = parent_field.field_type
+ if isinstance(parent_type, MapType):
+ parent_field = parent_type.value_field
+ elif isinstance(parent_type, ListType):
+ parent_field = parent_type.element_field
+
+ if not parent_field.field_type.is_struct:
+ raise ValueError(f"Cannot add column '{name}' to non-struct type: {'.'.join(parent)}")
+
+ parent_id = parent_field.field_id
+
+ exists = False
+ try:
+ exists = self._schema.find_field(full_name, self._case_sensitive) is not None
+ except ValueError:
+ pass
+
+ if exists:
+ raise ValueError(f"Cannot add column, name already exists: {full_name}")
+
+ # assign new IDs in order
+ new_id = self.assign_new_column_id()
+
+ # update tracking for moves
+ self._added_name_to_id[full_name] = new_id
Review Comment:
Good one, added! The `_id_to_parent` visitor is part of https://github.com/apache/iceberg/pull/8393
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org