You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ke...@apache.org on 2025/07/08 22:07:14 UTC
(iceberg-python) branch main updated: Partition statistics metadata reading (#2146)
This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 3fb92aa0 Partition statistics metadata reading (#2146)
3fb92aa0 is described below
commit 3fb92aa08f47ef402dd743f01bc2c0d5c8cee18f
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Wed Jul 9 00:07:08 2025 +0200
Partition statistics metadata reading (#2146)
# Rationale for this change
Took @smaheshwar-pltr's draft PR and added a test:
https://github.com/apache/iceberg-python/pull/2033
Cherry-picked his original work to ensure it gets attributed to the
original author.
# Are these changes tested?
# Are there any user-facing changes?
<!-- In the case of user-facing changes, please add the changelog label.
-->
---------
Co-authored-by: Sreesh Maheshwar <sm...@palantir.com>
---
pyiceberg/table/metadata.py | 10 +++++++-
pyiceberg/table/statistics.py | 11 ++++++++-
tests/table/test_metadata.py | 4 ++--
tests/table/test_statistics.py | 54 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py
index f248700c..9c2ae29c 100644
--- a/pyiceberg/table/metadata.py
+++ b/pyiceberg/table/metadata.py
@@ -36,7 +36,7 @@ from pyiceberg.table.sorting import (
SortOrder,
assign_fresh_sort_order_ids,
)
-from pyiceberg.table.statistics import StatisticsFile
+from pyiceberg.table.statistics import PartitionStatisticsFile, StatisticsFile
from pyiceberg.typedef import (
EMPTY_DICT,
IcebergBaseModel,
@@ -222,6 +222,14 @@ class TableMetadataCommonFields(IcebergBaseModel):
table correctly. A table can contain many statistics files
associated with different table snapshots."""
+ partition_statistics: List[PartitionStatisticsFile] = Field(alias="partition-statistics", default_factory=list)
+ """A optional list of partition statistics files.
+ Partition statistics are not required for reading or planning
+ and readers may ignore them. Each table snapshot may be associated
+ with at most one partition statistics file. A writer can optionally
+ write the partition statistics file during each write operation,
+ or it can also be computed on demand."""
+
# validators
@field_validator("properties", mode="before")
def transform_properties_dict_value_to_str(cls, properties: Properties) -> Dict[str, str]:
diff --git a/pyiceberg/table/statistics.py b/pyiceberg/table/statistics.py
index 151f5e96..a2e1b149 100644
--- a/pyiceberg/table/statistics.py
+++ b/pyiceberg/table/statistics.py
@@ -29,15 +29,24 @@ class BlobMetadata(IcebergBaseModel):
properties: Optional[Dict[str, str]] = None
-class StatisticsFile(IcebergBaseModel):
+class StatisticsCommonFields(IcebergBaseModel):
+ """Common fields between table and partition statistics structs found on metadata."""
+
snapshot_id: int = Field(alias="snapshot-id")
statistics_path: str = Field(alias="statistics-path")
file_size_in_bytes: int = Field(alias="file-size-in-bytes")
+
+
+class StatisticsFile(StatisticsCommonFields):
file_footer_size_in_bytes: int = Field(alias="file-footer-size-in-bytes")
key_metadata: Optional[str] = Field(alias="key-metadata", default=None)
blob_metadata: List[BlobMetadata] = Field(alias="blob-metadata")
+class PartitionStatisticsFile(StatisticsCommonFields):
+ pass
+
+
def filter_statistics_by_snapshot_id(
statistics: List[StatisticsFile],
reject_snapshot_id: int,
diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py
index a8410cff..9141189e 100644
--- a/tests/table/test_metadata.py
+++ b/tests/table/test_metadata.py
@@ -173,13 +173,13 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]) -> None:
def test_serialize_v1(example_table_metadata_v1: Dict[str, Any]) -> None:
table_metadata = TableMetadataV1(**example_table_metadata_v1)
table_metadata_json = table_metadata.model_dump_json()
- expected = """{"location":"s3://bucket/test/location","table-uuid":"d20125c8-7284-442c-9aea-15fee620737c","last-updated-ms":1602638573874,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]}],"current-schema-id":0,"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"fiel [...]
+ expected = """{"location":"s3://bucket/test/location","table-uuid":"d20125c8-7284-442c-9aea-15fee620737c","last-updated-ms":1602638573874,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]}],"current-schema-id":0,"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"fiel [...]
assert table_metadata_json == expected
def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]) -> None:
table_metadata = TableMetadataV2(**example_table_metadata_v2).model_dump_json()
- expected = """{"location":"s3://bucket/test/location","table-uuid":"9c12d441-03fe-4693-9a96-a0705ddf69c1","last-updated-ms":1602638573590,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]},{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema [...]
+ expected = """{"location":"s3://bucket/test/location","table-uuid":"9c12d441-03fe-4693-9a96-a0705ddf69c1","last-updated-ms":1602638573590,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]},{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema [...]
assert table_metadata == expected
diff --git a/tests/table/test_statistics.py b/tests/table/test_statistics.py
new file mode 100644
index 00000000..6c91990e
--- /dev/null
+++ b/tests/table/test_statistics.py
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pyiceberg.table.statistics import BlobMetadata, PartitionStatisticsFile, StatisticsFile
+
+
+def test_partition_statistics_file() -> None:
+ partition_statistics_file_json = (
+ """{"snapshot-id":123,"statistics-path":"s3://bucket/statistics.parquet","file-size-in-bytes":345}"""
+ )
+ partition_statistics_file = PartitionStatisticsFile.model_validate_json(partition_statistics_file_json)
+
+ assert partition_statistics_file == PartitionStatisticsFile(
+ snapshot_id=123, statistics_path="s3://bucket/statistics.parquet", file_size_in_bytes=345
+ )
+
+ assert partition_statistics_file.model_dump_json() == partition_statistics_file_json
+
+
+def test_statistics_file() -> None:
+ statistics_file_json = """{"snapshot-id":123,"statistics-path":"s3://bucket/statistics.parquet","file-size-in-bytes":345,"file-footer-size-in-bytes":456,"blob-metadata":[{"type":"apache-datasketches-theta-v1","snapshot-id":567,"sequence-number":22,"fields":[1,2,3],"properties":{"foo":"bar"}}]}"""
+ statistics_file = StatisticsFile.model_validate_json(statistics_file_json)
+
+ assert statistics_file == StatisticsFile(
+ snapshot_id=123,
+ statistics_path="s3://bucket/statistics.parquet",
+ file_size_in_bytes=345,
+ file_footer_size_in_bytes=456,
+ key_metadata=None,
+ blob_metadata=[
+ BlobMetadata(
+ type="apache-datasketches-theta-v1",
+ snapshot_id=567,
+ sequence_number=22,
+ fields=[1, 2, 3],
+ properties={"foo": "bar"},
+ )
+ ],
+ )
+
+ assert statistics_file.model_dump_json() == statistics_file_json