You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/25 20:26:35 UTC

(iceberg-go) branch main updated: fix(table): respect gc.enabled during snapshot expiration (#1737)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 43738fb54 fix(table): respect gc.enabled during snapshot expiration (#1737)
43738fb54 is described below

commit 43738fb54b7a71b234f9b75fb863600a40772312
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Tue Aug 25 22:26:29 2026 +0200

    fix(table): respect gc.enabled during snapshot expiration (#1737)
    
    * fix(table): respect gc.enabled during snapshot expiration
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    * fix(table): satisfy snapshot expiration lint
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    * fix(table): guard snapshot cleanup when GC is disabled
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    * fix(table): align GC validation with expiration
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    * fix(table): match Java gc.enabled parsing
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
    
    ---------
    
    Signed-off-by: Minh Vu <vu...@gmail.com>
---
 table/orphan_cleanup.go            |  2 +-
 table/orphan_cleanup_test.go       | 32 ++++++++++++++++++++++++
 table/properties.go                | 14 +++++++++++
 table/transaction.go               | 11 ++++++++
 table/transaction_internal_test.go | 51 ++++++++++++++++++++++++++++++++++++++
 table/updates.go                   |  2 +-
 table/updates_test.go              | 45 +++++++++++++++++++++++++++++++++
 website/src/configuration.md       |  2 +-
 8 files changed, 156 insertions(+), 3 deletions(-)

diff --git a/table/orphan_cleanup.go b/table/orphan_cleanup.go
index 22ae4694a..b20b8b033 100644
--- a/table/orphan_cleanup.go
+++ b/table/orphan_cleanup.go
@@ -1238,7 +1238,7 @@ func pathPrefix(path string) (scheme, authority string, ok bool) {
 // catalog drop operation should typically proceed so the catalog does not
 // get out of sync with storage.
 func (t Table) PurgeFiles(ctx context.Context) error {
-	gcEnabled := t.Metadata().Properties().GetBool("gc.enabled", true)
+	gcEnabled := isGCEnabled(t.Metadata().Properties())
 
 	fs, err := t.FS(ctx)
 	if err != nil {
diff --git a/table/orphan_cleanup_test.go b/table/orphan_cleanup_test.go
index 961d38220..7592261db 100644
--- a/table/orphan_cleanup_test.go
+++ b/table/orphan_cleanup_test.go
@@ -1616,6 +1616,38 @@ func (m mockFileInfo) ModTime() time.Time   { return m.modTime }
 func (m mockFileInfo) IsDir() bool          { return m.mode.IsDir() }
 func (m mockFileInfo) Sys() any             { return nil }
 
+func TestPurgeFilesSkipsDataFilesForMalformedGCEnabled(t *testing.T) {
+	const orphanDataPath = "s3://bucket/table/data/orphan.parquet"
+
+	for _, gcValue := range []string{"1", "garbage", "false ", "true "} {
+		t.Run(gcValue, func(t *testing.T) {
+			meta, err := NewMetadata(
+				iceberg.NewSchema(0),
+				iceberg.UnpartitionedSpec,
+				UnsortedSortOrder,
+				"s3://bucket/table",
+				iceberg.Properties{GCEnabledKey: gcValue},
+			)
+			require.NoError(t, err)
+
+			fsys := &mockListableIO{entries: []mockWalkEntry{{
+				path: orphanDataPath,
+				info: mockFileInfo{name: "orphan.parquet"},
+			}}}
+			tbl := New(
+				Identifier{"db", "tbl"},
+				meta,
+				"s3://bucket/table/metadata/v1.metadata.json",
+				func(context.Context) (io.IO, error) { return fsys, nil },
+				nil,
+			)
+
+			require.NoError(t, tbl.PurgeFiles(context.Background()))
+			assert.NotContains(t, fsys.removed, orphanDataPath)
+		})
+	}
+}
+
 func TestDeleteOrphanFilesPrefixMismatchModes(t *testing.T) {
 	tests := []struct {
 		name           string
diff --git a/table/properties.go b/table/properties.go
index a7f227c03..b42c656ea 100644
--- a/table/properties.go
+++ b/table/properties.go
@@ -19,7 +19,9 @@ package table
 
 import (
 	"math"
+	"strings"
 
+	"github.com/apache/iceberg-go"
 	"github.com/apache/iceberg-go/table/internal"
 )
 
@@ -99,6 +101,9 @@ const (
 	ParquetVariantBufferSizeKey     = internal.ParquetVariantBufferSizeKey
 	ParquetVariantBufferSizeDefault = internal.ParquetVariantBufferSizeDefault
 
+	GCEnabledKey     = "gc.enabled"
+	GCEnabledDefault = true
+
 	MinSnapshotsToKeepKey     = "history.expire.min-snapshots-to-keep"
 	MinSnapshotsToKeepDefault = 1
 
@@ -151,6 +156,15 @@ const (
 	CommitTotalRetryTimeoutMsDefault = 30 * 60 * 1000
 )
 
+func isGCEnabled(props iceberg.Properties) bool {
+	value, ok := props[GCEnabledKey]
+	if !ok {
+		return GCEnabledDefault
+	}
+
+	return strings.EqualFold(value, "true")
+}
+
 // Reserved properties
 const (
 	PropertyFormatVersion            = "format-version"
diff --git a/table/transaction.go b/table/transaction.go
index 23f4f4214..baf32c921 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -704,6 +704,9 @@ func (t *Transaction) ExpireSnapshots(opts ...ExpireSnapshotsOpt) error {
 	if err != nil {
 		return err
 	}
+	if err := validateGCEnabledForSnapshotExpiration(meta.props); err != nil {
+		return err
+	}
 
 	// Read table-level retention properties as the last-resort defaults,
 	// mirroring the Java implementation. The unprefixed property names are
@@ -3270,6 +3273,14 @@ func (t *Transaction) Commit(ctx context.Context) (*Table, error) {
 	return t.tbl, nil
 }
 
+func validateGCEnabledForSnapshotExpiration(props iceberg.Properties) error {
+	if !isGCEnabled(props) {
+		return errors.New("cannot expire snapshots: GC is disabled (deleting files may corrupt other tables)")
+	}
+
+	return nil
+}
+
 type StagedTable struct {
 	*Table
 }
diff --git a/table/transaction_internal_test.go b/table/transaction_internal_test.go
index f78ff4a61..848ee90a5 100644
--- a/table/transaction_internal_test.go
+++ b/table/transaction_internal_test.go
@@ -852,6 +852,57 @@ func expireSnapshotsSnapshot(t *testing.T, txn *Transaction, snapshotID int64) *
 	return nil
 }
 
+func TestExpireSnapshotsRejectsWhenGCDisabled(t *testing.T) {
+	tests := []struct {
+		name    string
+		gcValue string
+		opts    []ExpireSnapshotsOpt
+	}{
+		{name: "false with post-commit cleanup", gcValue: "false"},
+		{name: "false without post-commit cleanup", gcValue: "false", opts: []ExpireSnapshotsOpt{WithPostCommit(false)}},
+		{name: "one", gcValue: "1"},
+		{name: "garbage", gcValue: "garbage"},
+		{name: "false with trailing space", gcValue: "false "},
+		{name: "true with trailing space", gcValue: "true "},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			txn := newTransactionWithSnapshotRefs(t)
+			txn.meta.props = iceberg.Properties{GCEnabledKey: tt.gcValue}
+
+			err := txn.ExpireSnapshots(tt.opts...)
+			require.ErrorContains(t, err, "GC is disabled")
+			require.Empty(t, txn.reqs)
+			_, err = txn.meta.SnapshotByID(10)
+			require.NoError(t, err)
+		})
+	}
+}
+
+func TestIsGCEnabled(t *testing.T) {
+	tests := []struct {
+		name  string
+		props iceberg.Properties
+		want  bool
+	}{
+		{name: "missing", want: true},
+		{name: "true", props: iceberg.Properties{GCEnabledKey: "true"}, want: true},
+		{name: "mixed case true", props: iceberg.Properties{GCEnabledKey: "TrUe"}, want: true},
+		{name: "false", props: iceberg.Properties{GCEnabledKey: "false"}},
+		{name: "one", props: iceberg.Properties{GCEnabledKey: "1"}},
+		{name: "garbage", props: iceberg.Properties{GCEnabledKey: "garbage"}},
+		{name: "false with trailing space", props: iceberg.Properties{GCEnabledKey: "false "}},
+		{name: "true with trailing space", props: iceberg.Properties{GCEnabledKey: "true "}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.want, isGCEnabled(tt.props))
+		})
+	}
+}
+
 func TestTransactionApplyDedupesEquivalentRequirementsWithinAndAcrossCalls(t *testing.T) {
 	txn := newTransactionWithSnapshotRefs(t)
 
diff --git a/table/updates.go b/table/updates.go
index 5c002623e..a9c840313 100644
--- a/table/updates.go
+++ b/table/updates.go
@@ -632,7 +632,7 @@ func (u *removeSnapshotsUpdate) Apply(builder *MetadataBuilder) error {
 }
 
 func (u *removeSnapshotsUpdate) PostCommit(ctx context.Context, preTable *Table, postTable *Table) error {
-	if !u.postCommit {
+	if !u.postCommit || postTable == nil || !isGCEnabled(postTable.Properties()) {
 		return nil
 	}
 
diff --git a/table/updates_test.go b/table/updates_test.go
index fd79aea90..4ccce89d1 100644
--- a/table/updates_test.go
+++ b/table/updates_test.go
@@ -145,6 +145,51 @@ func TestRemoveSnapshotsPostCommitSkipped(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestRemoveSnapshotsPostCommitSkippedWhenGCDisabled(t *testing.T) {
+	tests := []struct {
+		name    string
+		gcValue string
+	}{
+		{name: "false", gcValue: "false"},
+		{name: "one", gcValue: "1"},
+		{name: "garbage", gcValue: "garbage"},
+		{name: "false with trailing space", gcValue: "false "},
+		{name: "true with trailing space", gcValue: "true "},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			preMeta := buildMetaJSON(metaJSONOpts{
+				snapshots:  `{"snapshot-id":1,"timestamp-ms":1000}`,
+				statistics: `{"snapshot-id":1,"statistics-path":"s3://bucket/stats/snap1.puffin","file-size-in-bytes":100,"file-footer-size-in-bytes":10,"blob-metadata":[]}`,
+			})
+			postMeta := buildMetaJSON(metaJSONOpts{})
+
+			pre, err := ParseMetadataString(preMeta)
+			require.NoError(t, err)
+			post, err := ParseMetadataString(postMeta)
+			require.NoError(t, err)
+
+			postBuilder, err := MetadataBuilderFromBase(post, "")
+			require.NoError(t, err)
+			require.NoError(t, postBuilder.SetProperties(iceberg.Properties{GCEnabledKey: tt.gcValue}))
+			post, err = postBuilder.Build()
+			require.NoError(t, err)
+
+			tio := newTrackingCallsIO()
+			tio.files["s3://bucket/stats/snap1.puffin"] = []byte("puffin")
+			fsF := testFSF(tio)
+			preTable := New(Identifier{"ns", "tbl"}, pre, "metadata.json", fsF, nil)
+			postTable := New(Identifier{"ns", "tbl"}, post, "metadata.json", fsF, nil)
+
+			update := NewRemoveSnapshotsUpdate([]int64{1}, true)
+			require.NoError(t, update.PostCommit(context.Background(), preTable, postTable))
+			require.Contains(t, tio.files, "s3://bucket/stats/snap1.puffin")
+			require.Zero(t, tio.removeCount["s3://bucket/stats/snap1.puffin"])
+		})
+	}
+}
+
 func TestRemoveSnapshotsPostCommitDeletesStatisticsFiles(t *testing.T) {
 	// preTable has snapshot 1 with associated statistics and partition statistics files.
 	// postTable has no snapshots and no statistics (snapshot 1 has been expired).
diff --git a/website/src/configuration.md b/website/src/configuration.md
index a8dcf0bfc..1b128a02c 100644
--- a/website/src/configuration.md
+++ b/website/src/configuration.md
@@ -335,7 +335,7 @@ Property key constants are in [`table/properties.go`](https://github.com/apache/
 | `history.expire.min-snapshots-to-keep` | Minimum snapshots to retain when expiring. Default `1`. |
 | `history.expire.max-snapshot-age-ms` | Maximum age of retained snapshots. Default `432000000` (5 days). |
 | `history.expire.max-ref-age-ms` | Maximum age of branch/tag refs that are not the main branch. Default `Long.MAX_VALUE` (forever). |
-| `gc.enabled` | Gate for orphan-file cleanup. |
+| `gc.enabled` | Controls physical garbage collection. Snapshot expiration is rejected when `false`, and data-file cleanup skips deletion. |
 
 The unprefixed retention property names are accepted as compatibility fallbacks. The `history.expire.*` properties take precedence when both forms are set.