You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/13 18:54:17 UTC
(iceberg-go) branch main updated: fix(table): prune StatisticsList and PartitionStatsList in RemoveSnapshots (#873)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new c82d2546 fix(table): prune StatisticsList and PartitionStatsList in RemoveSnapshots (#873)
c82d2546 is described below
commit c82d2546be93b0a87f6f5805f62ffe15f91e871f
Author: Ali Asghar <98...@users.noreply.github.com>
AuthorDate: Mon Apr 13 23:54:11 2026 +0500
fix(table): prune StatisticsList and PartitionStatsList in RemoveSnapshots (#873)
## Summary
This PR fixes two related bugs:
**1. `buildCommonMetadata` silently dropped all statistics (primary
bug)**
`buildCommonMetadata` in `metadata.go` was not copying `StatisticsList`
or `PartitionStatsList` into the rebuilt metadata. This meant *every*
`Build()` call — not just `RemoveSnapshots` — silently dropped all
statistics files. Any builder operation (add snapshot, update schema,
set properties, etc.) would lose statistics on the resulting metadata.
**2. `RemoveSnapshots` left orphaned statistics entries**
Even after fixing the builder, `RemoveSnapshots` would leave stale
`StatisticsList`/`PartitionStatsList` entries pointing at snapshots that
had just been removed. Added pruning logic consistent with the existing
`snapshotList`/`snapshotLog` pattern.
Fixes #836
## Test plan
- [x] `go test ./...`
- [x] `gofmt` clean
- [x] `TestRemoveSnapshotsPrunesStatistics` — seeds statistics for two
snapshots, removes one, asserts the right entry is pruned and the result
survives `Build()`.
- [x] `TestBuildPreservesStatistics` — no-op `MetadataBuilderFromBase →
Build()` round-trip, asserts statistics are preserved. Pins the
`buildCommonMetadata` fix independently of `RemoveSnapshots`.
Signed-off-by: Ali <al...@gmail.com>
---------
Signed-off-by: Ali <al...@gmail.com>
---
table/metadata.go | 15 ++++
table/metadata_builder_internal_test.go | 124 ++++++++++++++++++++++++++++++++
2 files changed, 139 insertions(+)
diff --git a/table/metadata.go b/table/metadata.go
index d3b18f21..2b611934 100644
--- a/table/metadata.go
+++ b/table/metadata.go
@@ -186,6 +186,8 @@ type MetadataBuilder struct {
sortOrderList []SortOrder
defaultSortOrderID int
refs map[string]SnapshotRef
+ statisticsList []StatisticsFile
+ partitionStatsList []PartitionStatisticsFile
previousFileEntry *MetadataLogEntry
// >v1 specific
@@ -261,6 +263,8 @@ func MetadataBuilderFromBase(metadata Metadata, currentFileLocation string) (*Me
b.refs = maps.Collect(metadata.Refs())
b.snapshotLog = slices.Collect(metadata.SnapshotLogs())
b.metadataLog = slices.Collect(metadata.PreviousFiles())
+ b.statisticsList = slices.Collect(metadata.Statistics())
+ b.partitionStatsList = slices.Collect(metadata.PartitionStatistics())
if currentFileLocation != "" {
b.previousFileEntry = &MetadataLogEntry{
@@ -505,6 +509,15 @@ func (b *MetadataBuilder) RemoveSnapshots(snapshotIds []int64, postCommit bool)
}
b.refs = newRefs
+ // Prune statistics entries whose snapshot was removed so that table
+ // metadata does not retain stale statistics references.
+ b.statisticsList = slices.DeleteFunc(b.statisticsList, func(e StatisticsFile) bool {
+ return slices.Contains(snapshotIds, e.SnapshotID)
+ })
+ b.partitionStatsList = slices.DeleteFunc(b.partitionStatsList, func(e PartitionStatisticsFile) bool {
+ return slices.Contains(snapshotIds, e.SnapshotID)
+ })
+
b.updates = append(b.updates, NewRemoveSnapshotsUpdate(snapshotIds, postCommit))
return nil
@@ -871,6 +884,8 @@ func (b *MetadataBuilder) buildCommonMetadata() (*commonMetadata, error) {
SortOrderList: b.sortOrderList,
DefaultSortOrderID: b.defaultSortOrderID,
SnapshotRefs: b.refs,
+ StatisticsList: b.statisticsList,
+ PartitionStatsList: b.partitionStatsList,
}, nil
}
diff --git a/table/metadata_builder_internal_test.go b/table/metadata_builder_internal_test.go
index 44990209..30a7ef8b 100644
--- a/table/metadata_builder_internal_test.go
+++ b/table/metadata_builder_internal_test.go
@@ -19,11 +19,13 @@ package table
import (
"fmt"
+ "slices"
"testing"
"time"
"github.com/apache/iceberg-go"
"github.com/davecgh/go-spew/spew"
+ "github.com/google/uuid"
"github.com/stretchr/testify/require"
)
@@ -622,6 +624,128 @@ func TestRemoveSnapshotRemovesBranch(t *testing.T) {
}
}
+// TestRemoveSnapshotsPrunesStatistics verifies that RemoveSnapshots also
+// prunes StatisticsList and PartitionStatsList entries whose SnapshotID
+// matches a removed snapshot. See iceberg-go#836.
+func TestRemoveSnapshotsPrunesStatistics(t *testing.T) {
+ const (
+ removedID = int64(100)
+ keptID = int64(200)
+ )
+
+ // Build minimal metadata directly so we can seed statistics and
+ // snapshots without going through AddSnapshot's timestamp checks.
+ currentID := int64(300)
+ lastPartitionID := 999
+ commonMeta := commonMetadata{
+ FormatVersion: 2,
+ UUID: uuid.New(),
+ Loc: "s3://test/table",
+ LastUpdatedMS: 1000,
+ LastColumnId: 1,
+ SchemaList: []*iceberg.Schema{iceberg.NewSchema(0)},
+ CurrentSchemaID: 0,
+ Specs: []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+ DefaultSpecID: 0,
+ LastPartitionID: &lastPartitionID,
+ Props: iceberg.Properties{},
+ SnapshotList: []Snapshot{
+ {SnapshotID: removedID, TimestampMs: 1001, ManifestList: "/snap-100.avro"},
+ {SnapshotID: keptID, TimestampMs: 1002, ManifestList: "/snap-200.avro"},
+ {SnapshotID: currentID, TimestampMs: 1003, ManifestList: "/snap-300.avro"},
+ },
+ CurrentSnapshotID: ¤tID,
+ SnapshotLog: []SnapshotLogEntry{{SnapshotID: currentID, TimestampMs: 1003}},
+ SortOrderList: []SortOrder{UnsortedSortOrder},
+ DefaultSortOrderID: 0,
+ SnapshotRefs: map[string]SnapshotRef{MainBranch: {SnapshotID: currentID, SnapshotRefType: BranchRef}},
+ StatisticsList: []StatisticsFile{
+ {SnapshotID: removedID, StatisticsPath: "s3://stats/removed.puffin"},
+ {SnapshotID: keptID, StatisticsPath: "s3://stats/kept.puffin"},
+ },
+ PartitionStatsList: []PartitionStatisticsFile{
+ {SnapshotID: removedID, StatisticsPath: "s3://partstats/removed.parquet"},
+ {SnapshotID: keptID, StatisticsPath: "s3://partstats/kept.parquet"},
+ },
+ }
+ meta := &metadataV2{LastSeqNum: 0, commonMetadata: commonMeta}
+
+ builder, err := MetadataBuilderFromBase(meta, "")
+ require.NoError(t, err)
+
+ // Sanity: MetadataBuilderFromBase must load statistics from the base.
+ require.Len(t, builder.statisticsList, 2,
+ "MetadataBuilderFromBase should load StatisticsList from the base metadata")
+ require.Len(t, builder.partitionStatsList, 2,
+ "MetadataBuilderFromBase should load PartitionStatsList from the base metadata")
+
+ require.NoError(t, builder.RemoveSnapshots([]int64{removedID}, false))
+
+ require.Len(t, builder.statisticsList, 1, "statistics for removed snapshot should be pruned")
+ require.Equal(t, keptID, builder.statisticsList[0].SnapshotID)
+
+ require.Len(t, builder.partitionStatsList, 1, "partition statistics for removed snapshot should be pruned")
+ require.Equal(t, keptID, builder.partitionStatsList[0].SnapshotID)
+
+ // And the kept entries survive a Build round trip.
+ rebuilt, err := builder.Build()
+ require.NoError(t, err)
+ require.Len(t, slices.Collect(rebuilt.Statistics()), 1)
+ require.Len(t, slices.Collect(rebuilt.PartitionStatistics()), 1)
+}
+
+// TestBuildPreservesStatistics pins the buildCommonMetadata fix: a plain
+// MetadataBuilderFromBase → Build() round-trip must preserve StatisticsList
+// and PartitionStatsList without any RemoveSnapshots call. Before the fix,
+// Build() silently dropped all statistics on every builder operation.
+func TestBuildPreservesStatistics(t *testing.T) {
+ const snapID = int64(42)
+ currentID := snapID
+ lastPartitionID := 0
+ commonMeta := commonMetadata{
+ FormatVersion: 2,
+ UUID: uuid.New(),
+ Loc: "s3://test/table",
+ LastUpdatedMS: 1000,
+ LastColumnId: 1,
+ SchemaList: []*iceberg.Schema{iceberg.NewSchema(0)},
+ CurrentSchemaID: 0,
+ Specs: []iceberg.PartitionSpec{*iceberg.UnpartitionedSpec},
+ DefaultSpecID: 0,
+ LastPartitionID: &lastPartitionID,
+ Props: iceberg.Properties{},
+ SnapshotList: []Snapshot{
+ {SnapshotID: snapID, TimestampMs: 1001, ManifestList: "/snap.avro"},
+ },
+ CurrentSnapshotID: ¤tID,
+ SnapshotLog: []SnapshotLogEntry{{SnapshotID: snapID, TimestampMs: 1001}},
+ SortOrderList: []SortOrder{UnsortedSortOrder},
+ DefaultSortOrderID: 0,
+ SnapshotRefs: map[string]SnapshotRef{MainBranch: {SnapshotID: snapID, SnapshotRefType: BranchRef}},
+ StatisticsList: []StatisticsFile{
+ {SnapshotID: snapID, StatisticsPath: "s3://stats/snap.puffin"},
+ },
+ PartitionStatsList: []PartitionStatisticsFile{
+ {SnapshotID: snapID, StatisticsPath: "s3://partstats/snap.parquet"},
+ },
+ }
+ meta := &metadataV2{commonMetadata: commonMeta}
+
+ builder, err := MetadataBuilderFromBase(meta, "")
+ require.NoError(t, err)
+
+ // No mutations — just rebuild.
+ rebuilt, err := builder.Build()
+ require.NoError(t, err)
+
+ require.Len(t, slices.Collect(rebuilt.Statistics()), 1,
+ "Build() must not silently drop StatisticsList")
+ require.Len(t, slices.Collect(rebuilt.PartitionStatistics()), 1,
+ "Build() must not silently drop PartitionStatsList")
+ require.Equal(t, snapID, slices.Collect(rebuilt.Statistics())[0].SnapshotID)
+ require.Equal(t, snapID, slices.Collect(rebuilt.PartitionStatistics())[0].SnapshotID)
+}
+
func TestExpireMetadataLog(t *testing.T) {
builder1 := builderWithoutChanges(2)
meta, err := builder1.Build()