You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by GitBox <gi...@apache.org> on 2022/01/20 21:41:08 UTC
[GitHub] [iceberg] emkornfield commented on a change in pull request #3677: Python: Adding TableMetadata object from dict, bytestream, and s3 sources
emkornfield commented on a change in pull request #3677:
URL: https://github.com/apache/iceberg/pull/3677#discussion_r789171429
##########
File path: python/src/iceberg/table/metadata.py
##########
@@ -0,0 +1,354 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import codecs
+import json
+
+import attr
+from jsonschema import validate as validate_json
+from jsonschema.exceptions import ValidationError
+
+from iceberg.io.file import FileIO
+from iceberg.utils.io import get_file_io
+
+TABLE_METADATA_V1_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema",
+ "type": "object",
+ "required": [
+ "format-version",
+ "location",
+ "last-updated-ms",
+ "last-column-id",
+ "schema",
+ "partition-spec",
+ ],
+ "properties": {
+ "format-version": {"type": "number"},
+ "table-uuid": {"type": "string"},
+ "location": {"type": "string"},
+ "last-updated-ms": {"type": "number"},
+ "last-column-id": {"type": "number"},
+ "schema": {
+ "type": "object",
+ },
+ "schemas": {
+ "type": "array",
+ },
+ "current-schema-id": {"type": "number"},
+ "partition-spec": {
+ "type": "object",
+ },
+ "partition-specs": {
+ "type": "array",
+ },
+ "default-spec-id": {"type": "number"},
+ "last-partition-id": {"type": "number"},
+ "properties": {
+ "type": "object",
+ },
+ "current-snapshot-id": {"type": "number"},
+ "snapshots": {
+ "type": "array",
+ },
+ "snapshot-log": {
+ "type": "array",
+ },
+ "metadata-log": {
+ "type": "array",
+ },
+ "sort-orders": {
+ "type": "array",
+ },
+ "default-sort-order-id": {"type": "number"},
+ },
+ "additionalProperties": False,
+}
+
+TABLE_METADATA_V2_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema",
+ "type": "object",
+ "required": [
+ "format-version",
+ "table-uuid",
+ "location",
+ "last-sequence-number",
+ "last-updated-ms",
+ "last-column-id",
+ "schemas",
+ "current-schema-id",
+ "partition-specs",
+ "default-spec-id",
+ "last-partition-id",
+ "default-sort-order-id",
+ "sort-orders",
+ ],
+ "properties": {
+ "format-version": {"type": "number"},
+ "table-uuid": {"type": "string"},
+ "location": {"type": "string"},
+ "last-sequence-number": {"type": "number"},
+ "last-updated-ms": {"type": "number"},
+ "last-column-id": {"type": "number"},
+ "schemas": {
+ "type": "array",
+ },
+ "current-schema-id": {"type": "number"},
+ "partition-specs": {
+ "type": "array",
+ },
+ "default-spec-id": {"type": "number"},
+ "last-partition-id": {"type": "number"},
+ "properties": {
+ "type": "object",
+ },
+ "current-snapshot-id": {"type": "number"},
+ "snapshots": {
+ "type": "array",
+ },
+ "snapshot-log": {
+ "type": "array",
+ },
+ "metadata-log": {
+ "type": "array",
+ },
+ "sort-orders": {
+ "type": "array",
+ },
+ "default-sort-order-id": {"type": "number"},
+ },
+ "additionalProperties": False,
+}
+
+
+@attr.s(frozen=True, auto_attribs=True)
+class TableMetadata:
+ """Metadata for an Iceberg table as specified in the Apache Iceberg
+ spec (https://iceberg.apache.org/spec/#iceberg-table-spec)"""
+
+ format_version: int
+ """An integer version number for the format. Currently, this can be 1 or 2
+ based on the spec. Implementations must throw an exception if a table’s
+ version is higher than the supported version."""
+
+ table_uuid: str
+ """A UUID that identifies the table, generated when the table is created.
+ Implementations must throw an exception if a table’s UUID does not match
+ the expected UUID after refreshing metadata."""
+
+ location: str
+ """The table’s base location. This is used by writers to determine where
+ to store data files, manifest files, and table metadata files."""
+
+ last_sequence_number: int
+ """The table’s highest assigned sequence number, a monotonically
+ increasing long that tracks the order of snapshots in a table."""
+
+ last_updated_ms: int
+ """Timestamp in milliseconds from the unix epoch when the table
+ was last updated. Each table metadata file should update this
+ field just before writing."""
+
+ last_column_id: int
+ """An integer; the highest assigned column ID for the table.
+ This is used to ensure columns are always assigned an unused ID
+ when evolving schemas."""
+
+ schema: dict
+ """The table’s current schema. (Deprecated: use schemas and
+ current-schema-id instead)"""
+
+ schemas: list
+ """A list of schemas, stored as objects with schema-id."""
+
+ current_schema_id: int
+ """ID of the table’s current schema."""
+
+ partition_spec: dict
+ """The table’s current partition spec, stored as only fields.
+ Note that this is used by writers to partition data, but is
+ not used when reading because reads use the specs stored in
+ manifest files. (Deprecated: use partition-specs and default-spec-id
+ instead)"""
+
+ partition_specs: list
+ """A list of partition specs, stored as full partition spec objects."""
+
+ default_spec_id: int
+ """ID of the “current” spec that writers should use by default."""
+
+ last_partition_id: int
+ """An integer; the highest assigned partition field ID across all
+ partition specs for the table. This is used to ensure partition fields
+ are always assigned an unused ID when evolving specs."""
+
+ properties: dict
+ """ A string to string map of table properties. This is used to
+ control settings that affect reading and writing and is not intended
+ to be used for arbitrary metadata. For example, commit.retry.num-retries
+ is used to control the number of commit retries."""
+
+ current_snapshot_id: int
+ """ID of the current table snapshot."""
+
+ snapshots: list
+ """A list of valid snapshots. Valid snapshots are snapshots for which
+ all data files exist in the file system. A data file must not be
+ deleted from the file system until the last snapshot in which it was
+ listed is garbage collected."""
+
+ snapshot_log: list
+ """A list (optional) of timestamp and snapshot ID pairs that encodes
+ changes to the current snapshot for the table. Each time the
+ current-snapshot-id is changed, a new entry should be added with the
+ last-updated-ms and the new current-snapshot-id. When snapshots are
+ expired from the list of valid snapshots, all entries before a snapshot
+ that has expired should be removed."""
+
+ metadata_log: list
+ """A list (optional) of timestamp and metadata file location pairs that
+ encodes changes to the previous metadata files for the table. Each time
+ a new metadata file is created, a new entry of the previous metadata
+ file location should be added to the list. Tables can be configured to
+ remove oldest metadata log entries and keep a fixed-size log of the most
+ recent entries after a commit."""
+
+ sort_orders: list
+ """A list of sort orders, stored as full sort order objects."""
+
+ default_sort_order_id: int
+ """Default sort order id of the table. Note that this could be used by
+ writers, but is not used when reading because reads use the specs stored
+ in manifest files."""
+
+ def validate(self):
+ """Checks that the table metadata object is valid. The validation schema
+ used depends on the Iceberg table metadata version."""
+ if self.format_version == 1:
+ self.validate_v1(self.to_dict())
+ elif self.format_version == 2:
+ self.validate_v2(self.to_dict())
+ else:
+ raise ValueError(f"Unknown table metadata version {self.format_version}")
+
+ @staticmethod
+ def validate_v1(metadata: dict):
+ """Perform a JSONSchema validation using the v1 Iceberg table metadata schema"""
+ try:
+ validate_json(instance=metadata, schema=TABLE_METADATA_V1_SCHEMA)
+ except ValidationError as e:
+ # TODO Log something here
+ raise (e)
+
+ @staticmethod
+ def validate_v2(metadata: dict):
+ """Perform a JSONSchema validation using the v2 Iceberg table metadata schema"""
+ try:
+ validate_json(instance=metadata, schema=TABLE_METADATA_V2_SCHEMA)
+ except ValidationError as e:
+ # TODO Log something here
+ raise (e)
+
+ @classmethod
+ def from_byte_stream(cls, byte_stream, encoding="utf-8"):
+ """Instantiate a TableMetadata object from a byte stream
+
+ Args:
+ byte_stream: A file-like byte stream object
+ encoding (default "utf-8"): The byte encoder to use for the reader
+ """
+ reader = codecs.getreader(encoding)
+ metadata = json.load(reader(byte_stream))
+ return cls.from_dict(metadata)
+
+ @classmethod
+ def from_file(cls, path: str, custom_file_io: FileIO = None, **kwargs):
Review comment:
api nit (apologies if the style has been discussed elsewhere) but it it might be more user-friendly to be less specific on individual parameters. e.g. have one parameter "file" which can be ["str", "FileIO", ...] and make the decision internally on what to use. Passing the base class as a factory seems a little strange since presumably clients know they have a custom_file_io and combine it with the path outside of this function.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org