You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2025/09/26 19:34:42 UTC
(iceberg-go) branch main updated: feat(metadata): Add support for Statistics and PartitionStatistics in table metadata (#577)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 38794865 feat(metadata): Add support for Statistics and PartitionStatistics in table metadata (#577)
38794865 is described below
commit 38794865aea6c4d09cec6f9d9a96ca0991469af0
Author: Alessandro Nori <al...@datadoghq.com>
AuthorDate: Fri Sep 26 21:34:37 2025 +0200
feat(metadata): Add support for Statistics and PartitionStatistics in table metadata (#577)
This PR adds support for reading Iceberg table metadata containing
statistics and partition statistics fields, as specified in
https://iceberg.apache.org/spec/#table-statistics.
---
table/metadata.go | 59 +++++++++++++++-------
table/metadata_internal_test.go | 107 ++++++++++++++++++++++++++++++++++++++--
table/statistics.go | 87 ++++++++++++++++++++++++++++++++
3 files changed, 230 insertions(+), 23 deletions(-)
diff --git a/table/metadata.go b/table/metadata.go
index 6b95b8f9..2240549b 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -136,6 +136,19 @@ type Metadata interface {
NameMapping() iceberg.NameMapping
LastSequenceNumber() int64
+ // Statistics returns an optional list of table statistics.
+ // Table statistics files are valid Puffin files.
+ // StatisticsFile are informational. A reader can choose to ignore statistics information.
+ // StatisticsFile support is not required to read the table correctly.
+ // A table can contain many statistics files associated with different table snapshots.
+ Statistics() iter.Seq[StatisticsFile]
+ // PartitionStatistics returns an optional list of partition statistics files.
+ // Partition statistics are not required for reading or planning
+ // and readers may ignore them. Each table snapshot may be associated
+ // with at most one partition statistics file. A writer can optionally
+ // write the partition statistics file during each write operation,
+ // or it can also be computed on demand.
+ PartitionStatistics() iter.Seq[PartitionStatisticsFile]
}
type MetadataBuilder struct {
@@ -1086,24 +1099,26 @@ func sliceEqualHelper[T interface{ Equals(T) bool }](s1, s2 []T) bool {
// https://iceberg.apache.org/spec/#iceberg-table-spec
type commonMetadata struct {
- FormatVersion int `json:"format-version"`
- UUID uuid.UUID `json:"table-uuid"`
- Loc string `json:"location"`
- LastUpdatedMS int64 `json:"last-updated-ms"`
- LastColumnId int `json:"last-column-id"`
- SchemaList []*iceberg.Schema `json:"schemas"`
- CurrentSchemaID int `json:"current-schema-id"`
- Specs []iceberg.PartitionSpec `json:"partition-specs"`
- DefaultSpecID int `json:"default-spec-id"`
- LastPartitionID *int `json:"last-partition-id,omitempty"`
- Props iceberg.Properties `json:"properties,omitempty"`
- SnapshotList []Snapshot `json:"snapshots,omitempty"`
- CurrentSnapshotID *int64 `json:"current-snapshot-id,omitempty"`
- SnapshotLog []SnapshotLogEntry `json:"snapshot-log,omitempty"`
- MetadataLog []MetadataLogEntry `json:"metadata-log,omitempty"`
- SortOrderList []SortOrder `json:"sort-orders"`
- DefaultSortOrderID int `json:"default-sort-order-id"`
- SnapshotRefs map[string]SnapshotRef `json:"refs,omitempty"`
+ FormatVersion int `json:"format-version"`
+ UUID uuid.UUID `json:"table-uuid"`
+ Loc string `json:"location"`
+ LastUpdatedMS int64 `json:"last-updated-ms"`
+ LastColumnId int `json:"last-column-id"`
+ SchemaList []*iceberg.Schema `json:"schemas"`
+ CurrentSchemaID int `json:"current-schema-id"`
+ Specs []iceberg.PartitionSpec `json:"partition-specs"`
+ DefaultSpecID int `json:"default-spec-id"`
+ LastPartitionID *int `json:"last-partition-id,omitempty"`
+ Props iceberg.Properties `json:"properties,omitempty"`
+ SnapshotList []Snapshot `json:"snapshots,omitempty"`
+ CurrentSnapshotID *int64 `json:"current-snapshot-id,omitempty"`
+ SnapshotLog []SnapshotLogEntry `json:"snapshot-log,omitempty"`
+ MetadataLog []MetadataLogEntry `json:"metadata-log,omitempty"`
+ SortOrderList []SortOrder `json:"sort-orders"`
+ DefaultSortOrderID int `json:"default-sort-order-id"`
+ SnapshotRefs map[string]SnapshotRef `json:"refs,omitempty"`
+ StatisticsList []StatisticsFile `json:"statistics,omitempty"`
+ PartitionStatsList []PartitionStatisticsFile `json:"partition-statistics,omitempty"`
}
func initCommonMetadataForDeserialization() commonMetadata {
@@ -1250,6 +1265,14 @@ func (c *commonMetadata) Properties() iceberg.Properties {
return c.Props
}
+func (c *commonMetadata) Statistics() iter.Seq[StatisticsFile] {
+ return slices.Values(c.StatisticsList)
+}
+
+func (c *commonMetadata) PartitionStatistics() iter.Seq[PartitionStatisticsFile] {
+ return slices.Values(c.PartitionStatsList)
+}
+
// preValidate updates values in the metadata struct with defaults based on
// combinations of struct members. Such as initializing slices as empty slices
// if they were null in the metadata, or normalizing inconsistencies between
diff --git a/table/metadata_internal_test.go b/table/metadata_internal_test.go
index 8b573fb5..7b58eb01 100644
--- a/table/metadata_internal_test.go
+++ b/table/metadata_internal_test.go
@@ -676,12 +676,37 @@ func TestMetadataV1Serialize(t *testing.T) {
DefaultSpecID: 0,
SortOrderList: []SortOrder{UnsortedSortOrder},
DefaultSortOrderID: 0,
+ StatisticsList: []StatisticsFile{
+ {
+ SnapshotID: 1234567890,
+ StatisticsPath: "s3://bucket/statistics/stats1.puffin",
+ FileSizeInBytes: 1024000,
+ FileFooterSizeInBytes: 512,
+ KeyMetadata: nil,
+ BlobMetadata: []BlobMetadata{
+ {
+ Type: BlobTypeApacheDatasketchesThetaV1,
+ SnapshotID: 1234567890,
+ SequenceNumber: 1,
+ Fields: []int32{1, 2},
+ Properties: map[string]string{"ndv": "1000"},
+ },
+ },
+ },
+ },
+ PartitionStatsList: []PartitionStatisticsFile{
+ {
+ SnapshotID: 1234567890,
+ StatisticsPath: "s3://bucket/partition-stats/part1.parquet",
+ FileSizeInBytes: 512000,
+ },
+ },
},
}
data, err := json.Marshal(toserialize)
require.NoError(t, err)
- assert.JSONEq(t, `{
+ assert.JSONEq(t, `{
"format-version":1,
"table-uuid":"dd93fa46-a1a7-43bb-8748-6cc7eff107a3",
"location":"s3a://warehouse/iceberg/iceberg-test-2.db/test-table-2",
@@ -699,7 +724,31 @@ func TestMetadataV1Serialize(t *testing.T) {
"partition-specs":[{"spec-id":0,"fields":[]}],
"default-spec-id":0,
"sort-orders":[{"order-id":0,"fields":[]}],
- "default-sort-order-id":0
+ "default-sort-order-id":0,
+ "statistics": [
+ {
+ "snapshot-id": 1234567890,
+ "statistics-path": "s3://bucket/statistics/stats1.puffin",
+ "file-size-in-bytes": 1024000,
+ "file-footer-size-in-bytes": 512,
+ "blob-metadata": [
+ {
+ "type": "apache-datasketches-theta-v1",
+ "snapshot-id": 1234567890,
+ "sequence-number": 1,
+ "fields": [1, 2],
+ "properties": {"ndv": "1000"}
+ }
+ ]
+ }
+ ],
+ "partition-statistics": [
+ {
+ "snapshot-id": 1234567890,
+ "statistics-path": "s3://bucket/partition-stats/part1.parquet",
+ "file-size-in-bytes": 512000
+ }
+ ]
}`, string(data))
}
@@ -709,7 +758,7 @@ func TestMetadataV2Serialize(t *testing.T) {
toserialize := &metadataV2{
LastSeqNum: 1,
commonMetadata: commonMetadata{
- FormatVersion: 1,
+ FormatVersion: 2,
UUID: uuid.MustParse("dd93fa46-a1a7-43bb-8748-6cc7eff107a3"),
Loc: "s3a://warehouse/iceberg/iceberg-test-2.db/test-table-2",
LastUpdatedMS: 1742412491193,
@@ -720,6 +769,30 @@ func TestMetadataV2Serialize(t *testing.T) {
DefaultSpecID: 0,
SortOrderList: []SortOrder{UnsortedSortOrder},
DefaultSortOrderID: 0,
+ StatisticsList: []StatisticsFile{
+ {
+ SnapshotID: 9876543210,
+ StatisticsPath: "s3://bucket/v2/statistics/stats2.puffin",
+ FileSizeInBytes: 2048000,
+ FileFooterSizeInBytes: 1024,
+ BlobMetadata: []BlobMetadata{
+ {
+ Type: BlobTypeDeletionVectorV1,
+ SnapshotID: 9876543210,
+ SequenceNumber: 5,
+ Fields: []int32{1},
+ Properties: map[string]string{"deletion-vector-size": "500"},
+ },
+ },
+ },
+ },
+ PartitionStatsList: []PartitionStatisticsFile{
+ {
+ SnapshotID: 9876543210,
+ StatisticsPath: "s3://bucket/v2/partition-stats/part2.parquet",
+ FileSizeInBytes: 768000,
+ },
+ },
},
}
@@ -727,7 +800,7 @@ func TestMetadataV2Serialize(t *testing.T) {
require.NoError(t, err)
assert.JSONEq(t, `{
"last-sequence-number": 1,
- "format-version":1,
+ "format-version": 2,
"table-uuid":"dd93fa46-a1a7-43bb-8748-6cc7eff107a3",
"location":"s3a://warehouse/iceberg/iceberg-test-2.db/test-table-2",
"last-updated-ms":1742412491193,
@@ -744,7 +817,31 @@ func TestMetadataV2Serialize(t *testing.T) {
"partition-specs":[{"spec-id":0,"fields":[]}],
"default-spec-id":0,
"sort-orders":[{"order-id":0,"fields":[]}],
- "default-sort-order-id":0
+ "default-sort-order-id":0,
+ "statistics": [
+ {
+ "snapshot-id": 9876543210,
+ "statistics-path": "s3://bucket/v2/statistics/stats2.puffin",
+ "file-size-in-bytes": 2048000,
+ "file-footer-size-in-bytes": 1024,
+ "blob-metadata": [
+ {
+ "type": "deletion-vector-v1",
+ "snapshot-id": 9876543210,
+ "sequence-number": 5,
+ "fields": [1],
+ "properties": {"deletion-vector-size": "500"}
+ }
+ ]
+ }
+ ],
+ "partition-statistics": [
+ {
+ "snapshot-id": 9876543210,
+ "statistics-path": "s3://bucket/v2/partition-stats/part2.parquet",
+ "file-size-in-bytes": 768000
+ }
+ ]
}`, string(data))
}
diff --git a/table/statistics.go b/table/statistics.go
new file mode 100644
index 00000000..0f7e40a6
--- /dev/null
+++ b/table/statistics.go
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package table
+
+import (
+ "encoding/json"
+ "fmt"
+)
+
+// BlobType is the type of blob in a Puffin file
+type BlobType string
+
+const (
+ BlobTypeApacheDatasketchesThetaV1 BlobType = "apache-datasketches-theta-v1"
+ BlobTypeDeletionVectorV1 BlobType = "deletion-vector-v1"
+)
+
+func (bt *BlobType) IsValid() bool {
+ switch *bt {
+ case BlobTypeApacheDatasketchesThetaV1, BlobTypeDeletionVectorV1:
+ return true
+ default:
+ return false
+ }
+}
+
+func (bt *BlobType) UnmarshalJSON(data []byte) error {
+ var s string
+ if err := json.Unmarshal(data, &s); err != nil {
+ return err
+ }
+
+ *bt = BlobType(s)
+ if !bt.IsValid() {
+ return fmt.Errorf("invalid blob type: %s", s)
+ }
+
+ return nil
+}
+
+// StatisticsFile represents a statistics file in the Puffin format, that can be used to read table data more
+// efficiently.
+//
+// Statistics are informational. A reader can choose to ignore statistics information.
+// Statistics support is not required to read the table correctly.
+type StatisticsFile struct {
+ SnapshotID int64 `json:"snapshot-id"`
+ StatisticsPath string `json:"statistics-path"`
+ FileSizeInBytes int64 `json:"file-size-in-bytes"`
+ FileFooterSizeInBytes int64 `json:"file-footer-size-in-bytes"`
+ KeyMetadata *string `json:"key-metadata,omitempty"`
+ BlobMetadata []BlobMetadata `json:"blob-metadata"`
+}
+
+// BlobMetadata is the metadata of a statistics or indices blob.
+type BlobMetadata struct {
+ Type BlobType `json:"type"`
+ SnapshotID int64 `json:"snapshot-id"`
+ SequenceNumber int64 `json:"sequence-number"`
+ Fields []int32 `json:"fields"`
+ Properties map[string]string `json:"properties"`
+}
+
+// PartitionStatisticsFile represents a partition statistics file that can be used to read table data more efficiently.
+//
+// Statistics are informational. A reader can choose to ignore statistics information. Statistics
+// support is not required to read the table correctly.
+type PartitionStatisticsFile struct {
+ SnapshotID int64 `json:"snapshot-id"`
+ StatisticsPath string `json:"statistics-path"`
+ FileSizeInBytes int64 `json:"file-size-in-bytes"`
+}