You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by GitBox <gi...@apache.org> on 2022/02/06 04:38:05 UTC
[GitHub] [iceberg] emkornfield commented on a change in pull request #3677: Python: Adding TableMetadata object from dict, bytestream, and InputFile implementation
emkornfield commented on a change in pull request #3677:
URL: https://github.com/apache/iceberg/pull/3677#discussion_r800125575
##########
File path: python/src/iceberg/table/metadata.py
##########
@@ -0,0 +1,365 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import codecs
+import json
+
+import attr
+from jsonschema import validate as validate_json
+from jsonschema.exceptions import ValidationError
+
+from iceberg.io.base import InputFile, OutputFile
+
+TABLE_METADATA_V1_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema",
+ "type": "object",
+ "required": [
+ "format-version",
+ "location",
+ "last-updated-ms",
+ "last-column-id",
+ "schema",
+ "partition-spec",
+ ],
+ "properties": {
+ "format-version": {"type": "number"},
+ "table-uuid": {"type": "string"},
+ "location": {"type": "string"},
+ "last-updated-ms": {"type": "number"},
+ "last-column-id": {"type": "number"},
+ "schema": {
+ "type": "object",
+ },
+ "schemas": {
+ "type": "array",
+ },
+ "current-schema-id": {"type": "number"},
+ "partition-spec": {
+ "type": "object",
+ },
+ "partition-specs": {
+ "type": "array",
+ },
+ "default-spec-id": {"type": "number"},
+ "last-partition-id": {"type": "number"},
+ "properties": {
+ "type": "object",
+ },
+ "current-snapshot-id": {"type": "number"},
+ "snapshots": {
+ "type": "array",
+ },
+ "snapshot-log": {
+ "type": "array",
+ },
+ "metadata-log": {
+ "type": "array",
+ },
+ "sort-orders": {
+ "type": "array",
+ },
+ "default-sort-order-id": {"type": "number"},
+ },
+ "additionalProperties": False,
+}
+
+TABLE_METADATA_V2_SCHEMA = {
+ "$schema": "http://json-schema.org/draft-07/schema",
+ "type": "object",
+ "required": [
+ "format-version",
+ "table-uuid",
+ "location",
+ "last-sequence-number",
+ "last-updated-ms",
+ "last-column-id",
+ "schemas",
+ "current-schema-id",
+ "partition-specs",
+ "default-spec-id",
+ "last-partition-id",
+ "default-sort-order-id",
+ "sort-orders",
+ ],
+ "properties": {
+ "format-version": {"type": "number"},
+ "table-uuid": {"type": "string"},
+ "location": {"type": "string"},
+ "last-sequence-number": {"type": "number"},
+ "last-updated-ms": {"type": "number"},
+ "last-column-id": {"type": "number"},
+ "schemas": {
+ "type": "array",
+ },
+ "current-schema-id": {"type": "number"},
+ "partition-specs": {
+ "type": "array",
+ },
+ "default-spec-id": {"type": "number"},
+ "last-partition-id": {"type": "number"},
+ "properties": {
+ "type": "object",
+ },
+ "current-snapshot-id": {"type": "number"},
+ "snapshots": {
+ "type": "array",
+ },
+ "snapshot-log": {
+ "type": "array",
+ },
+ "metadata-log": {
+ "type": "array",
+ },
+ "sort-orders": {
+ "type": "array",
+ },
+ "default-sort-order-id": {"type": "number"},
+ },
+ "additionalProperties": False,
+}
+
+
+@attr.s(frozen=True, auto_attribs=True)
+class TableMetadata:
+ """Metadata for an Iceberg table as specified in the Apache Iceberg
+ spec (https://iceberg.apache.org/spec/#iceberg-table-spec)"""
+
+ format_version: int
+ """An integer version number for the format. Currently, this can be 1 or 2
+ based on the spec. Implementations must throw an exception if a table’s
+ version is higher than the supported version."""
+
+ table_uuid: str
+ """A UUID that identifies the table, generated when the table is created.
+ Implementations must throw an exception if a table’s UUID does not match
+ the expected UUID after refreshing metadata."""
+
+ location: str
+ """The table’s base location. This is used by writers to determine where
+ to store data files, manifest files, and table metadata files."""
+
+ last_sequence_number: int
+ """The table’s highest assigned sequence number, a monotonically
+ increasing long that tracks the order of snapshots in a table."""
+
+ last_updated_ms: int
+ """Timestamp in milliseconds from the unix epoch when the table
+ was last updated. Each table metadata file should update this
+ field just before writing."""
+
+ last_column_id: int
+ """An integer; the highest assigned column ID for the table.
+ This is used to ensure columns are always assigned an unused ID
+ when evolving schemas."""
+
+ schema: dict
+ """The table’s current schema. (Deprecated: use schemas and
+ current-schema-id instead)"""
+
+ schemas: list
+ """A list of schemas, stored as objects with schema-id."""
+
+ current_schema_id: int
+ """ID of the table’s current schema."""
+
+ partition_spec: dict
+ """The table’s current partition spec, stored as only fields.
+ Note that this is used by writers to partition data, but is
+ not used when reading because reads use the specs stored in
+ manifest files. (Deprecated: use partition-specs and default-spec-id
+ instead)"""
+
+ partition_specs: list
Review comment:
Is the intention to refine collection types later with actual object types or keep these fairly generic?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org