You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/25 20:26:35 UTC
(iceberg-go) branch main updated: fix(table): respect gc.enabled during snapshot expiration (#1737)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 43738fb54 fix(table): respect gc.enabled during snapshot expiration (#1737)
43738fb54 is described below
commit 43738fb54b7a71b234f9b75fb863600a40772312
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 25 22:26:29 2026 +0200
fix(table): respect gc.enabled during snapshot expiration (#1737)
* fix(table): respect gc.enabled during snapshot expiration
Signed-off-by: Minh Vu <vu...@gmail.com>
* fix(table): satisfy snapshot expiration lint
Signed-off-by: Minh Vu <vu...@gmail.com>
* fix(table): guard snapshot cleanup when GC is disabled
Signed-off-by: Minh Vu <vu...@gmail.com>
* fix(table): align GC validation with expiration
Signed-off-by: Minh Vu <vu...@gmail.com>
* fix(table): match Java gc.enabled parsing
Signed-off-by: Minh Vu <vu...@gmail.com>
---------
Signed-off-by: Minh Vu <vu...@gmail.com>
---
table/orphan_cleanup.go | 2 +-
table/orphan_cleanup_test.go | 32 ++++++++++++++++++++++++
table/properties.go | 14 +++++++++++
table/transaction.go | 11 ++++++++
table/transaction_internal_test.go | 51 ++++++++++++++++++++++++++++++++++++++
table/updates.go | 2 +-
table/updates_test.go | 45 +++++++++++++++++++++++++++++++++
website/src/configuration.md | 2 +-
8 files changed, 156 insertions(+), 3 deletions(-)
diff --git a/table/orphan_cleanup.go b/table/orphan_cleanup.go
index 22ae4694a..b20b8b033 100644
--- a/table/orphan_cleanup.go
+++ b/table/orphan_cleanup.go
@@ -1238,7 +1238,7 @@ func pathPrefix(path string) (scheme, authority string, ok bool) {
// catalog drop operation should typically proceed so the catalog does not
// get out of sync with storage.
func (t Table) PurgeFiles(ctx context.Context) error {
- gcEnabled := t.Metadata().Properties().GetBool("gc.enabled", true)
+ gcEnabled := isGCEnabled(t.Metadata().Properties())
fs, err := t.FS(ctx)
if err != nil {
diff --git a/table/orphan_cleanup_test.go b/table/orphan_cleanup_test.go
index 961d38220..7592261db 100644
--- a/table/orphan_cleanup_test.go
+++ b/table/orphan_cleanup_test.go
@@ -1616,6 +1616,38 @@ func (m mockFileInfo) ModTime() time.Time { return m.modTime }
func (m mockFileInfo) IsDir() bool { return m.mode.IsDir() }
func (m mockFileInfo) Sys() any { return nil }
+func TestPurgeFilesSkipsDataFilesForMalformedGCEnabled(t *testing.T) {
+ const orphanDataPath = "s3://bucket/table/data/orphan.parquet"
+
+ for _, gcValue := range []string{"1", "garbage", "false ", "true "} {
+ t.Run(gcValue, func(t *testing.T) {
+ meta, err := NewMetadata(
+ iceberg.NewSchema(0),
+ iceberg.UnpartitionedSpec,
+ UnsortedSortOrder,
+ "s3://bucket/table",
+ iceberg.Properties{GCEnabledKey: gcValue},
+ )
+ require.NoError(t, err)
+
+ fsys := &mockListableIO{entries: []mockWalkEntry{{
+ path: orphanDataPath,
+ info: mockFileInfo{name: "orphan.parquet"},
+ }}}
+ tbl := New(
+ Identifier{"db", "tbl"},
+ meta,
+ "s3://bucket/table/metadata/v1.metadata.json",
+ func(context.Context) (io.IO, error) { return fsys, nil },
+ nil,
+ )
+
+ require.NoError(t, tbl.PurgeFiles(context.Background()))
+ assert.NotContains(t, fsys.removed, orphanDataPath)
+ })
+ }
+}
+
func TestDeleteOrphanFilesPrefixMismatchModes(t *testing.T) {
tests := []struct {
name string
diff --git a/table/properties.go b/table/properties.go
index a7f227c03..b42c656ea 100644
--- a/table/properties.go
+++ b/table/properties.go
@@ -19,7 +19,9 @@ package table
import (
"math"
+ "strings"
+ "github.com/apache/iceberg-go"
"github.com/apache/iceberg-go/table/internal"
)
@@ -99,6 +101,9 @@ const (
ParquetVariantBufferSizeKey = internal.ParquetVariantBufferSizeKey
ParquetVariantBufferSizeDefault = internal.ParquetVariantBufferSizeDefault
+ GCEnabledKey = "gc.enabled"
+ GCEnabledDefault = true
+
MinSnapshotsToKeepKey = "history.expire.min-snapshots-to-keep"
MinSnapshotsToKeepDefault = 1
@@ -151,6 +156,15 @@ const (
CommitTotalRetryTimeoutMsDefault = 30 * 60 * 1000
)
+func isGCEnabled(props iceberg.Properties) bool {
+ value, ok := props[GCEnabledKey]
+ if !ok {
+ return GCEnabledDefault
+ }
+
+ return strings.EqualFold(value, "true")
+}
+
// Reserved properties
const (
PropertyFormatVersion = "format-version"
diff --git a/table/transaction.go b/table/transaction.go
index 23f4f4214..baf32c921 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -704,6 +704,9 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error {
if err != nil {
return err
}
+ if err := validateGCEnabledForSnapshotExpiration(meta.props); err != nil {
+ return err
+ }
// Read table-level retention properties as the last-resort defaults,
// mirroring the Java implementation. The unprefixed property names are
@@ -3270,6 +3273,14 @@ func (t *Transaction) Commit(ctx context.Context) (*Table, error) {
return t.tbl, nil
}
+func validateGCEnabledForSnapshotExpiration(props iceberg.Properties) error {
+ if !isGCEnabled(props) {
+ return errors.New("cannot expire snapshots: GC is disabled (deleting files may corrupt other tables)")
+ }
+
+ return nil
+}
+
type StagedTable struct {
*Table
}
diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go
index f78ff4a61..848ee90a5 100644
--- a/table/transaction_internal_test.go
+++ b/table/transaction_internal_test.go
@@ -852,6 +852,57 @@ func expireSnapshotsSnapshot(t *testing.T, txn *Transaction, snapshotID int64) *
return nil
}
+func TestExpireSnapshotsRejectsWhenGCDisabled(t *testing.T) {
+ tests := []struct {
+ name string
+ gcValue string
+ opts []ExpireSnapshotsOpt
+ }{
+ {name: "false with post-commit cleanup", gcValue: "false"},
+ {name: "false without post-commit cleanup", gcValue: "false", opts: []ExpireSnapshotsOpt{WithPostCommit(false)}},
+ {name: "one", gcValue: "1"},
+ {name: "garbage", gcValue: "garbage"},
+ {name: "false with trailing space", gcValue: "false "},
+ {name: "true with trailing space", gcValue: "true "},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ txn := newTransactionWithSnapshotRefs(t)
+ txn.meta.props = iceberg.Properties{GCEnabledKey: tt.gcValue}
+
+ err := txn.ExpireSnapshots(tt.opts...)
+ require.ErrorContains(t, err, "GC is disabled")
+ require.Empty(t, txn.reqs)
+ _, err = txn.meta.SnapshotByID(10)
+ require.NoError(t, err)
+ })
+ }
+}
+
+func TestIsGCEnabled(t *testing.T) {
+ tests := []struct {
+ name string
+ props iceberg.Properties
+ want bool
+ }{
+ {name: "missing", want: true},
+ {name: "true", props: iceberg.Properties{GCEnabledKey: "true"}, want: true},
+ {name: "mixed case true", props: iceberg.Properties{GCEnabledKey: "TrUe"}, want: true},
+ {name: "false", props: iceberg.Properties{GCEnabledKey: "false"}},
+ {name: "one", props: iceberg.Properties{GCEnabledKey: "1"}},
+ {name: "garbage", props: iceberg.Properties{GCEnabledKey: "garbage"}},
+ {name: "false with trailing space", props: iceberg.Properties{GCEnabledKey: "false "}},
+ {name: "true with trailing space", props: iceberg.Properties{GCEnabledKey: "true "}},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ require.Equal(t, tt.want, isGCEnabled(tt.props))
+ })
+ }
+}
+
func TestTransactionApplyDedupesEquivalentRequirementsWithinAndAcrossCalls(t *testing.T) {
txn := newTransactionWithSnapshotRefs(t)
diff --git a/table/updates.go b/table/updates.go
index 5c002623e..a9c840313 100644
--- a/table/updates.go
+++ b/table/updates.go
@@ -632,7 +632,7 @@ func (u *removeSnapshotsUpdate) Apply(builder *MetadataBuilder) error {
}
func (u *removeSnapshotsUpdate) PostCommit(ctx context.Context, preTable *Table, postTable *Table) error {
- if !u.postCommit {
+ if !u.postCommit || postTable == nil || !isGCEnabled(postTable.Properties()) {
return nil
}
diff --git a/table/updates_test.go b/table/updates_test.go
index fd79aea90..4ccce89d1 100644
--- a/table/updates_test.go
+++ b/table/updates_test.go
@@ -145,6 +145,51 @@ func TestRemoveSnapshotsPostCommitSkipped(t *testing.T) {
assert.NoError(t, err)
}
+func TestRemoveSnapshotsPostCommitSkippedWhenGCDisabled(t *testing.T) {
+ tests := []struct {
+ name string
+ gcValue string
+ }{
+ {name: "false", gcValue: "false"},
+ {name: "one", gcValue: "1"},
+ {name: "garbage", gcValue: "garbage"},
+ {name: "false with trailing space", gcValue: "false "},
+ {name: "true with trailing space", gcValue: "true "},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ preMeta := buildMetaJSON(metaJSONOpts{
+ snapshots: `{"snapshot-id":1,"timestamp-ms":1000}`,
+ statistics: `{"snapshot-id":1,"statistics-path":"s3://bucket/stats/snap1.puffin","file-size-in-bytes":100,"file-footer-size-in-bytes":10,"blob-metadata":[]}`,
+ })
+ postMeta := buildMetaJSON(metaJSONOpts{})
+
+ pre, err := ParseMetadataString(preMeta)
+ require.NoError(t, err)
+ post, err := ParseMetadataString(postMeta)
+ require.NoError(t, err)
+
+ postBuilder, err := MetadataBuilderFromBase(post, "")
+ require.NoError(t, err)
+ require.NoError(t, postBuilder.SetProperties(iceberg.Properties{GCEnabledKey: tt.gcValue}))
+ post, err = postBuilder.Build()
+ require.NoError(t, err)
+
+ tio := newTrackingCallsIO()
+ tio.files["s3://bucket/stats/snap1.puffin"] = []byte("puffin")
+ fsF := testFSF(tio)
+ preTable := New(Identifier{"ns", "tbl"}, pre, "metadata.json", fsF, nil)
+ postTable := New(Identifier{"ns", "tbl"}, post, "metadata.json", fsF, nil)
+
+ update := NewRemoveSnapshotsUpdate([]int64{1}, true)
+ require.NoError(t, update.PostCommit(context.Background(), preTable, postTable))
+ require.Contains(t, tio.files, "s3://bucket/stats/snap1.puffin")
+ require.Zero(t, tio.removeCount["s3://bucket/stats/snap1.puffin"])
+ })
+ }
+}
+
func TestRemoveSnapshotsPostCommitDeletesStatisticsFiles(t *testing.T) {
// preTable has snapshot 1 with associated statistics and partition statistics files.
// postTable has no snapshots and no statistics (snapshot 1 has been expired).
diff --git a/website/src/configuration.md b/website/src/configuration.md
index a8dcf0bfc..1b128a02c 100644
--- a/website/src/configuration.md
+++ b/website/src/configuration.md
@@ -335,7 +335,7 @@ Property key constants are in [`table/properties.go`](https://github.com/apache/
| `history.expire.min-snapshots-to-keep` | Minimum snapshots to retain when expiring. Default `1`. |
| `history.expire.max-snapshot-age-ms` | Maximum age of retained snapshots. Default `432000000` (5 days). |
| `history.expire.max-ref-age-ms` | Maximum age of branch/tag refs that are not the main branch. Default `Long.MAX_VALUE` (forever). |
-| `gc.enabled` | Gate for orphan-file cleanup. |
+| `gc.enabled` | Controls physical garbage collection. Snapshot expiration is rejected when `false`, and data-file cleanup skips deletion. |
The unprefixed retention property names are accepted as compatibility fallbacks. The `history.expire.*` properties take precedence when both forms are set.