You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/17 23:10:22 UTC
(iceberg-go) branch main updated: perf(compaction): compute survivor minimum once (#1820)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 4390d2af2 perf(compaction): compute survivor minimum once (#1820)
4390d2af2 is described below
commit 4390d2af2bb011c0df9797e06e7db6c4a7ef551b
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 18 01:10:15 2026 +0200
perf(compaction): compute survivor minimum once (#1820)
* perf(compaction): compute survivor minimum once
* test(compaction): cover conservative delete decisions
* fix(compaction): satisfy integer range lint
---
table/compaction/eq_delete_decision.go | 3 +-
table/compaction/eq_delete_decision_bench_test.go | 67 +++++++++++++++++++
table/compaction/eq_delete_decision_test.go | 81 ++++++++++++++++++++++-
3 files changed, 149 insertions(+), 2 deletions(-)
diff --git a/table/compaction/eq_delete_decision.go b/table/compaction/eq_delete_decision.go
index 5fedd7e8f..92759ad2e 100644
--- a/table/compaction/eq_delete_decision.go
+++ b/table/compaction/eq_delete_decision.go
@@ -138,6 +138,7 @@ func DecideDeadEqualityDeletes(survey *SurvivorSurvey, candidates []iceberg.Mani
dead := make([]iceberg.DataFile, 0, len(candidates))
seen := make(map[string]struct{}, len(candidates))
+ minSeq := survey.conservativeMinSeq()
for _, e := range candidates {
df := e.DataFile()
path := df.FilePath()
@@ -147,7 +148,7 @@ func DecideDeadEqualityDeletes(survey *SurvivorSurvey, candidates []iceberg.Mani
if e.SequenceNum() < 0 {
continue
}
- if survey.conservativeMinSeq() >= e.SequenceNum() {
+ if minSeq >= e.SequenceNum() {
seen[path] = struct{}{}
dead = append(dead, df)
}
diff --git a/table/compaction/eq_delete_decision_bench_test.go b/table/compaction/eq_delete_decision_bench_test.go
new file mode 100644
index 000000000..d2dcfa17b
--- /dev/null
+++ b/table/compaction/eq_delete_decision_bench_test.go
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compaction_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/iceberg-go"
+ "github.com/apache/iceberg-go/table/compaction"
+)
+
+var benchmarkDeadEqualityDeletes []iceberg.DataFile
+
+func BenchmarkDecideDeadEqualityDeletes(b *testing.B) {
+ for _, tc := range []struct {
+ name string
+ partitions int
+ deletes int
+ }{
+ {name: "P=1/D=1", partitions: 1, deletes: 1},
+ {name: "P=100/D=100", partitions: 100, deletes: 100},
+ {name: "P=1000/D=1000", partitions: 1000, deletes: 1000},
+ {name: "P=10000/D=1000", partitions: 10000, deletes: 1000},
+ {name: "P=1000/D=10000", partitions: 1000, deletes: 10000},
+ } {
+ b.Run(tc.name, func(b *testing.B) {
+ survey := compaction.NewSurvivorSurvey()
+ for i := range tc.partitions {
+ survey.AddSurvivor(map[int]any{1000: i}, 10)
+ }
+
+ specs := compactionTestSpecs()
+ candidates := make([]iceberg.ManifestEntry, tc.deletes)
+ for i := range candidates {
+ candidates[i] = makeEqDeleteEntry(
+ b, specs[0], nil, 1, fmt.Sprintf("/eq-%d.parquet", i))
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for range b.N {
+ benchmarkDeadEqualityDeletes = compaction.DecideDeadEqualityDeletes(
+ survey, candidates)
+ }
+ b.StopTimer()
+ if len(benchmarkDeadEqualityDeletes) != tc.deletes {
+ b.Fatalf("expected %d dead equality deletes, got %d", tc.deletes, len(benchmarkDeadEqualityDeletes))
+ }
+ })
+ }
+}
diff --git a/table/compaction/eq_delete_decision_test.go b/table/compaction/eq_delete_decision_test.go
index 9b117075a..9b82dc18a 100644
--- a/table/compaction/eq_delete_decision_test.go
+++ b/table/compaction/eq_delete_decision_test.go
@@ -235,6 +235,85 @@ func TestDecideDeadEqualityDeletesConservativelyMatchesAcrossSpecs(t *testing.T)
survey, []iceberg.ManifestEntry{voidCandidate}))
}
+func TestDecideDeadEqualityDeletesUsesConservativeMinimum(t *testing.T) {
+ type candidate struct {
+ path string
+ seq int64
+ }
+
+ tests := []struct {
+ name string
+ survivors []survivor
+ candidates []candidate
+ wantPaths []string
+ }{
+ {
+ name: "minimum across partition buckets",
+ survivors: []survivor{
+ {partition: map[int]any{1000: "us"}, seq: 3},
+ {partition: map[int]any{1000: "eu"}, seq: 9},
+ },
+ candidates: []candidate{
+ {path: "/dead-before-min.parquet", seq: 2},
+ {path: "/dead-at-min.parquet", seq: 3},
+ {path: "/alive-after-min.parquet", seq: 4},
+ {path: "/unknown-seq.parquet", seq: -1},
+ },
+ wantPaths: []string{
+ "/dead-before-min.parquet",
+ "/dead-at-min.parquet",
+ },
+ },
+ {
+ name: "includes empty-partition bucket",
+ survivors: []survivor{
+ {partition: nil, seq: 7},
+ {partition: map[int]any{1000: "us"}, seq: 10},
+ },
+ candidates: []candidate{
+ {path: "/dead-at-empty-min.parquet", seq: 7},
+ {path: "/alive-after-empty-min.parquet", seq: 8},
+ },
+ wantPaths: []string{"/dead-at-empty-min.parquet"},
+ },
+ {
+ name: "deduplicates paths",
+ survivors: []survivor{
+ {partition: map[int]any{1000: "us"}, seq: 5},
+ },
+ candidates: []candidate{
+ {path: "/duplicate.parquet", seq: 5},
+ {path: "/duplicate.parquet", seq: 5},
+ {path: "/unknown-seq.parquet", seq: -1},
+ },
+ wantPaths: []string{"/duplicate.parquet"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ specs := compactionTestSpecs()
+ survey := compaction.NewSurvivorSurvey()
+ for _, s := range tt.survivors {
+ survey.AddSurvivor(s.partition, s.seq)
+ }
+
+ candidates := make([]iceberg.ManifestEntry, 0, len(tt.candidates))
+ for _, c := range tt.candidates {
+ candidates = append(candidates,
+ makeEqDeleteEntry(t, specs[0], nil, c.seq, c.path))
+ }
+
+ got := compaction.DecideDeadEqualityDeletes(survey, candidates)
+ gotPaths := make([]string, len(got))
+ for i, file := range got {
+ gotPaths[i] = file.FilePath()
+ }
+ assert.Equal(t, tt.wantPaths, gotPaths)
+ })
+ }
+}
+
// TestSurvivorSurvey_AddSurvivor_DefensiveSeq asserts that a survivor
// with sequence number < 0 is recorded as if seq=0 — guaranteeing it
// stays "alive" against every eq-delete.
@@ -293,7 +372,7 @@ func compactionTestSpecs() compactionTestSpecLookup {
// reads only path, partition, content type, and seq, so a minimal
// builder configuration is enough.
func makeEqDeleteEntry(
- t *testing.T,
+ t testing.TB,
spec iceberg.PartitionSpec,
part map[int]any,
seq int64,