You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2025/06/10 22:32:51 UTC

(iceberg-go) branch main updated: refactor: pass context into actual IO (#452)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new 237da8f  refactor: pass context into actual IO (#452)
237da8f is described below

commit 237da8fd2034d772cd5969a372d2e411e3dfc66d
Author: Andrei Tserakhau <la...@gmail.com>
AuthorDate: Wed Jun 11 00:32:46 2025 +0200

    refactor: pass context into actual IO (#452)
    
    related https://github.com/apache/iceberg-go/issues/437
    
    All I/O operations must respect the context passed to PlanFiles(ctx) and
    simmilar methods.
    
    To support that, we stop passing pre-constructed I/O objects and instead
    construct them ad hoc within PlanFiles, using the provided context
---
 catalog/glue/glue.go                  |  31 +++++----
 catalog/internal/utils.go             |  27 ++++----
 catalog/rest/rest.go                  |  13 ++--
 catalog/rest/rest_integration_test.go |  15 +++--
 catalog/sql/sql.go                    |  19 ++++--
 cmd/iceberg/output.go                 |  10 ++-
 io/io.go                              |  12 ++++
 table/scanner.go                      |  25 +++++--
 table/table.go                        |  71 +++++++++++++-------
 table/table_test.go                   | 120 ++++++++++++++++++++++++----------
 table/transaction.go                  |  55 +++++++++++-----
 table/transaction_test.go             |   2 +-
 12 files changed, 275 insertions(+), 125 deletions(-)

diff --git a/catalog/glue/glue.go b/catalog/glue/glue.go
index f3d14c5..65b3899 100644
--- a/catalog/glue/glue.go
+++ b/catalog/glue/glue.go
@@ -223,13 +223,14 @@ func (c *Catalog) LoadTable(ctx context.Context, identifier table.Identifier, pr
 	}
 
 	ctx = utils.WithAwsConfig(ctx, c.awsCfg)
-	// TODO: consider providing a way to directly access the S3 iofs to enable testing of the catalog.
-	iofs, err := io.LoadFS(ctx, props, location)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load table %s.%s: %w", database, tableName, err)
-	}
 
-	icebergTable, err := table.NewFromLocation(identifier, location, iofs, c)
+	icebergTable, err := table.NewFromLocation(
+		ctx,
+		identifier,
+		location,
+		io.LoadFSFunc(props, location),
+		c,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create table from location %s.%s: %w", database, tableName, err)
 	}
@@ -255,7 +256,11 @@ func (c *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
 		return nil, err
 	}
 
-	wfs, ok := staged.FS().(io.WriteFileIO)
+	afs, err := staged.FS(ctx)
+	if err != nil {
+		return nil, err
+	}
+	wfs, ok := afs.(io.WriteFileIO)
 	if !ok {
 		return nil, errors.New("loaded filesystem IO does not support writing")
 	}
@@ -317,12 +322,14 @@ func (c *Catalog) RegisterTable(ctx context.Context, identifier table.Identifier
 	}
 	// Load the metadata file to get table properties
 	ctx = utils.WithAwsConfig(ctx, c.awsCfg)
-	iofs, err := io.LoadFS(ctx, nil, metadataLocation)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load metadata file at %s: %w", metadataLocation, err)
-	}
 	// Read the metadata file
-	metadata, err := table.NewFromLocation([]string{tableName}, metadataLocation, iofs, c)
+	metadata, err := table.NewFromLocation(
+		ctx,
+		[]string{tableName},
+		metadataLocation,
+		io.LoadFSFunc(nil, metadataLocation),
+		c,
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read table metadata from %s: %w", metadataLocation, err)
 	}
diff --git a/catalog/internal/utils.go b/catalog/internal/utils.go
index 7e4c8ca..81b8cc3 100644
--- a/catalog/internal/utils.go
+++ b/catalog/internal/utils.go
@@ -136,13 +136,15 @@ func CreateStagedTable(ctx context.Context, catprops iceberg.Properties, nsprops
 
 	ioProps := maps.Clone(catprops)
 	maps.Copy(ioProps, cfg.Properties)
-	fs, err := io.LoadFS(ctx, ioProps, metadataLoc)
-	if err != nil {
-		return table.StagedTable{}, err
-	}
 
 	return table.StagedTable{
-		Table: table.New(ident, metadata, metadataLoc, fs, nil),
+		Table: table.New(
+			ident,
+			metadata,
+			metadataLoc,
+			io.LoadFSFunc(ioProps, metadataLoc),
+			nil,
+		),
 	}, nil
 }
 
@@ -239,10 +241,13 @@ func UpdateAndStageTable(ctx context.Context, current *table.Table, ident table.
 		return nil, err
 	}
 
-	fs, err := io.LoadFS(ctx, updated.Properties(), newLocation)
-	if err != nil {
-		return nil, err
-	}
-
-	return &table.StagedTable{Table: table.New(ident, updated, newLocation, fs, cat)}, nil
+	return &table.StagedTable{
+		Table: table.New(
+			ident,
+			updated,
+			newLocation,
+			io.LoadFSFunc(updated.Properties(), newLocation),
+			cat,
+		),
+	}, nil
 }
diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go
index 9491649..93c88db 100644
--- a/catalog/rest/rest.go
+++ b/catalog/rest/rest.go
@@ -653,12 +653,13 @@ func checkValidNamespace(ident table.Identifier) error {
 }
 
 func (r *Catalog) tableFromResponse(ctx context.Context, identifier []string, metadata table.Metadata, loc string, config iceberg.Properties) (*table.Table, error) {
-	iofs, err := iceio.LoadFS(ctx, config, loc)
-	if err != nil {
-		return nil, err
-	}
-
-	return table.New(identifier, metadata, loc, iofs, r), nil
+	return table.New(
+		identifier,
+		metadata,
+		loc,
+		iceio.LoadFSFunc(config, loc),
+		r,
+	), nil
 }
 
 func (r *Catalog) ListTables(ctx context.Context, namespace table.Identifier) iter.Seq2[table.Identifier, error] {
diff --git a/catalog/rest/rest_integration_test.go b/catalog/rest/rest_integration_test.go
index edba0d5..4bd2807 100644
--- a/catalog/rest/rest_integration_test.go
+++ b/catalog/rest/rest_integration_test.go
@@ -33,6 +33,7 @@ import (
 	"github.com/apache/iceberg-go/catalog/rest"
 	"github.com/apache/iceberg-go/io"
 	"github.com/apache/iceberg-go/table"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 )
 
@@ -254,11 +255,11 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() {
 	pqfile, err := url.JoinPath(location, "data", "test_commit_table_data", "test.parquet")
 	s.Require().NoError(err)
 
-	fw, err := tbl.FS().(io.WriteFileIO).Create(pqfile)
+	fw, err := mustFS(s.T(), tbl).(io.WriteFileIO).Create(pqfile)
 	s.Require().NoError(err)
 	s.Require().NoError(pqarrow.WriteTable(table, fw, table.NumRows(),
 		nil, pqarrow.DefaultWriterProps()))
-	defer tbl.FS().Remove(pqfile)
+	defer mustFS(s.T(), tbl).Remove(pqfile)
 
 	txn := tbl.NewTransaction()
 	s.Require().NoError(txn.AddFiles(s.ctx, []string{pqfile}, nil, false))
@@ -266,7 +267,7 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() {
 	s.Require().NoError(err)
 
 	mf := []iceberg.ManifestFile{}
-	for m, err := range updated.AllManifests() {
+	for m, err := range updated.AllManifests(s.ctx) {
 		s.Require().NoError(err)
 		s.Require().NotNil(m)
 		mf = append(mf, m)
@@ -274,13 +275,19 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() {
 
 	s.Len(mf, 1)
 	s.EqualValues(1, mf[0].AddedDataFiles())
-	entries, err := mf[0].FetchEntries(updated.FS(), false)
+	entries, err := mf[0].FetchEntries(mustFS(s.T(), updated), false)
 	s.Require().NoError(err)
 
 	s.Len(entries, 1)
 	s.Equal(pqfile, entries[0].DataFile().FilePath())
 }
 
+func mustFS(t *testing.T, tbl *table.Table) io.IO {
+	r, err := tbl.FS(context.Background())
+	require.NoError(t, err)
+	return r
+}
+
 func TestRestIntegration(t *testing.T) {
 	suite.Run(t, new(RestIntegrationSuite))
 }
diff --git a/catalog/sql/sql.go b/catalog/sql/sql.go
index bb8a083..762a643 100644
--- a/catalog/sql/sql.go
+++ b/catalog/sql/sql.go
@@ -283,7 +283,11 @@ func (c *Catalog) CreateTable(ctx context.Context, ident table.Identifier, sc *i
 		return nil, fmt.Errorf("%w: %s", catalog.ErrNoSuchNamespace, ns)
 	}
 
-	wfs, ok := staged.FS().(io.WriteFileIO)
+	afs, err := staged.FS(ctx)
+	if err != nil {
+		return nil, err
+	}
+	wfs, ok := afs.(io.WriteFileIO)
 	if !ok {
 		return nil, errors.New("loaded filesystem IO does not support writing")
 	}
@@ -416,12 +420,13 @@ func (c *Catalog) LoadTable(ctx context.Context, identifier table.Identifier, pr
 	tblProps := maps.Clone(c.props)
 	maps.Copy(props, tblProps)
 
-	iofs, err := io.LoadFS(ctx, tblProps, result.MetadataLocation.String)
-	if err != nil {
-		return nil, err
-	}
-
-	return table.NewFromLocation(identifier, result.MetadataLocation.String, iofs, c)
+	return table.NewFromLocation(
+		ctx,
+		identifier,
+		result.MetadataLocation.String,
+		io.LoadFSFunc(tblProps, result.MetadataLocation.String),
+		c,
+	)
 }
 
 func (c *Catalog) DropTable(ctx context.Context, identifier table.Identifier) error {
diff --git a/cmd/iceberg/output.go b/cmd/iceberg/output.go
index cc31b6d..695ba28 100644
--- a/cmd/iceberg/output.go
+++ b/cmd/iceberg/output.go
@@ -18,6 +18,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -134,7 +135,12 @@ func (t textOutput) Files(tbl *table.Table, history bool) {
 				snap.SnapshotID, *snap.SchemaID, manifest),
 		})
 
-		manifestList, err := snap.Manifests(tbl.FS())
+		afs, err := tbl.FS(context.TODO())
+		if err != nil {
+			t.Error(err)
+			os.Exit(1)
+		}
+		manifestList, err := snap.Manifests(afs)
 		if err != nil {
 			t.Error(err)
 			os.Exit(1)
@@ -144,7 +150,7 @@ func (t textOutput) Files(tbl *table.Table, history bool) {
 			snapshotTree = append(snapshotTree, pterm.LeveledListItem{
 				Level: 1, Text: "Manifest: " + m.FilePath(),
 			})
-			datafiles, err := m.FetchEntries(tbl.FS(), false)
+			datafiles, err := m.FetchEntries(afs, false)
 			if err != nil {
 				t.Error(err)
 				os.Exit(1)
diff --git a/io/io.go b/io/io.go
index 523f7c5..248dfed 100644
--- a/io/io.go
+++ b/io/io.go
@@ -294,3 +294,15 @@ func LoadFS(ctx context.Context, props map[string]string, location string) (IO,
 
 	return iofs, nil
 }
+
+// LoadFSFunc is a helper function to create IO Func factory for later usage
+func LoadFSFunc(props map[string]string, location string) func(ctx context.Context) (IO, error) {
+	return func(ctx context.Context) (IO, error) {
+		iofs, err := LoadFS(ctx, props, location)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load metadata file at %s: %w", location, err)
+		}
+
+		return iofs, nil
+	}
+}
diff --git a/table/scanner.go b/table/scanner.go
index 14a3811..93dd052 100644
--- a/table/scanner.go
+++ b/table/scanner.go
@@ -154,7 +154,7 @@ func openManifest(io io.IO, manifest iceberg.ManifestFile,
 
 type Scan struct {
 	metadata       Metadata
-	io             io.IO
+	ioF            FSysF
 	rowFilter      iceberg.BooleanExpression
 	selectedFields []string
 	caseSensitive  bool
@@ -299,14 +299,18 @@ func matchDeletesToData(entry iceberg.ManifestEntry, positionalDeletes []iceberg
 
 // fetchPartitionSpecFilteredManifests retrieves the table's current snapshot,
 // fetches its manifest files, and applies partition-spec filters to remove irrelevant manifests.
-func (scan *Scan) fetchPartitionSpecFilteredManifests() ([]iceberg.ManifestFile, error) {
+func (scan *Scan) fetchPartitionSpecFilteredManifests(ctx context.Context) ([]iceberg.ManifestFile, error) {
 	snap := scan.Snapshot()
 	if snap == nil {
 		return nil, nil
 	}
 
+	afs, err := scan.ioF(ctx)
+	if err != nil {
+		return nil, err
+	}
 	// Fetch all manifests for the current snapshot.
-	manifestList, err := snap.Manifests(scan.io)
+	manifestList, err := snap.Manifests(afs)
 	if err != nil {
 		return nil, err
 	}
@@ -354,8 +358,12 @@ func (scan *Scan) collectManifestEntries(
 		}
 
 		g.Go(func() error {
+			fs, err := scan.ioF(ctx)
+			if err != nil {
+				return err
+			}
 			partEval := partitionEvaluators.Get(int(mf.PartitionSpecID()))
-			manifestEntries, err := openManifest(scan.io, mf, partEval, metricsEval)
+			manifestEntries, err := openManifest(fs, mf, partEval, metricsEval)
 			if err != nil {
 				return err
 			}
@@ -390,7 +398,7 @@ func (scan *Scan) collectManifestEntries(
 // building a list of FileScanTasks that match the current Scan criteria.
 func (scan *Scan) PlanFiles(ctx context.Context) ([]FileScanTask, error) {
 	// Step 1: Retrieve filtered manifests based on snapshot and partition specs.
-	manifestList, err := scan.fetchPartitionSpecFilteredManifests()
+	manifestList, err := scan.fetchPartitionSpecFilteredManifests(ctx)
 	if err != nil || len(manifestList) == 0 {
 		return nil, err
 	}
@@ -456,9 +464,14 @@ func (scan *Scan) ToArrowRecords(ctx context.Context) (*arrow.Schema, iter.Seq2[
 		return nil, nil, err
 	}
 
+	fs, err := scan.ioF(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
 	return (&arrowScan{
 		metadata:        scan.metadata,
-		fs:              scan.io,
+		fs:              fs,
 		projectedSchema: schema,
 		boundRowFilter:  boundFilter,
 		caseSensitive:   scan.caseSensitive,
diff --git a/table/table.go b/table/table.go
index c91fdfd..b49b240 100644
--- a/table/table.go
+++ b/table/table.go
@@ -33,6 +33,8 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
+type FSysF func(ctx context.Context) (io.IO, error)
+
 type Identifier = []string
 
 type CatalogIO interface {
@@ -44,8 +46,8 @@ type Table struct {
 	identifier       Identifier
 	metadata         Metadata
 	metadataLocation string
-	fs               io.IO
 	cat              CatalogIO
+	fsF              FSysF
 }
 
 func (t Table) Equals(other Table) bool {
@@ -54,19 +56,19 @@ func (t Table) Equals(other Table) bool {
 		t.metadata.Equals(other.metadata)
 }
 
-func (t Table) Identifier() Identifier               { return t.identifier }
-func (t Table) Metadata() Metadata                   { return t.metadata }
-func (t Table) MetadataLocation() string             { return t.metadataLocation }
-func (t Table) FS() io.IO                            { return t.fs }
-func (t Table) Schema() *iceberg.Schema              { return t.metadata.CurrentSchema() }
-func (t Table) Spec() iceberg.PartitionSpec          { return t.metadata.PartitionSpec() }
-func (t Table) SortOrder() SortOrder                 { return t.metadata.SortOrder() }
-func (t Table) Properties() iceberg.Properties       { return t.metadata.Properties() }
-func (t Table) NameMapping() iceberg.NameMapping     { return t.metadata.NameMapping() }
-func (t Table) Location() string                     { return t.metadata.Location() }
-func (t Table) CurrentSnapshot() *Snapshot           { return t.metadata.CurrentSnapshot() }
-func (t Table) SnapshotByID(id int64) *Snapshot      { return t.metadata.SnapshotByID(id) }
-func (t Table) SnapshotByName(name string) *Snapshot { return t.metadata.SnapshotByName(name) }
+func (t Table) Identifier() Identifier                { return t.identifier }
+func (t Table) Metadata() Metadata                    { return t.metadata }
+func (t Table) MetadataLocation() string              { return t.metadataLocation }
+func (t Table) FS(ctx context.Context) (io.IO, error) { return t.fsF(ctx) }
+func (t Table) Schema() *iceberg.Schema               { return t.metadata.CurrentSchema() }
+func (t Table) Spec() iceberg.PartitionSpec           { return t.metadata.PartitionSpec() }
+func (t Table) SortOrder() SortOrder                  { return t.metadata.SortOrder() }
+func (t Table) Properties() iceberg.Properties        { return t.metadata.Properties() }
+func (t Table) NameMapping() iceberg.NameMapping      { return t.metadata.NameMapping() }
+func (t Table) Location() string                      { return t.metadata.Location() }
+func (t Table) CurrentSnapshot() *Snapshot            { return t.metadata.CurrentSnapshot() }
+func (t Table) SnapshotByID(id int64) *Snapshot       { return t.metadata.SnapshotByID(id) }
+func (t Table) SnapshotByName(name string) *Snapshot  { return t.metadata.SnapshotByName(name) }
 func (t Table) Schemas() map[int]*iceberg.Schema {
 	m := make(map[int]*iceberg.Schema)
 	for _, s := range t.metadata.Schemas() {
@@ -110,15 +112,23 @@ func (t Table) Append(ctx context.Context, rdr array.RecordReader, snapshotProps
 	return txn.Commit(ctx)
 }
 
-func (t Table) AllManifests() iter.Seq2[iceberg.ManifestFile, error] {
+func (t Table) AllManifests(ctx context.Context) iter.Seq2[iceberg.ManifestFile, error] {
+	fs, err := t.fsF(ctx)
+	if err != nil {
+		return func(yield func(iceberg.ManifestFile, error) bool) {
+			yield(nil, err)
+		}
+	}
+
 	type list = tblutils.Enumerated[[]iceberg.ManifestFile]
 	g := errgroup.Group{}
 
 	n := len(t.metadata.Snapshots())
 	ch := make(chan list, n)
+
 	for i, sn := range t.metadata.Snapshots() {
 		g.Go(func() error {
-			manifests, err := sn.Manifests(t.fs)
+			manifests, err := sn.Manifests(fs)
 			if err != nil {
 				return err
 			}
@@ -195,10 +205,13 @@ func (t Table) doCommit(ctx context.Context, updates []Update, reqs []Requiremen
 	if err != nil {
 		return nil, err
 	}
+	fs, err := t.fsF(ctx)
+	if err != nil {
+		return nil, err
+	}
+	deleteOldMetadata(fs, t.metadata, newMeta)
 
-	deleteOldMetadata(t.fs, t.metadata, newMeta)
-
-	return New(t.identifier, newMeta, newLoc, t.fs, t.cat), nil
+	return New(t.identifier, newMeta, newLoc, t.fsF, t.cat), nil
 }
 
 func getFiles(it iter.Seq[MetadataLogEntry]) iter.Seq[string] {
@@ -310,7 +323,7 @@ func WithOptions(opts iceberg.Properties) ScanOption {
 func (t Table) Scan(opts ...ScanOption) *Scan {
 	s := &Scan{
 		metadata:       t.metadata,
-		io:             t.fs,
+		ioF:            t.fsF,
 		rowFilter:      iceberg.AlwaysTrue{},
 		selectedFields: []string{"*"},
 		caseSensitive:  true,
@@ -327,19 +340,29 @@ func (t Table) Scan(opts ...ScanOption) *Scan {
 	return s
 }
 
-func New(ident Identifier, meta Metadata, metadataLocation string, fs io.IO, cat CatalogIO) *Table {
+func New(ident Identifier, meta Metadata, metadataLocation string, fsF FSysF, cat CatalogIO) *Table {
 	return &Table{
 		identifier:       ident,
 		metadata:         meta,
 		metadataLocation: metadataLocation,
-		fs:               fs,
+		fsF:              fsF,
 		cat:              cat,
 	}
 }
 
-func NewFromLocation(ident Identifier, metalocation string, fsys io.IO, cat CatalogIO) (*Table, error) {
+func NewFromLocation(
+	ctx context.Context,
+	ident Identifier,
+	metalocation string,
+	fsysF FSysF,
+	cat CatalogIO,
+) (*Table, error) {
 	var meta Metadata
 
+	fsys, err := fsysF(ctx)
+	if err != nil {
+		return nil, err
+	}
 	if rf, ok := fsys.(io.ReadFileIO); ok {
 		data, err := rf.ReadFile(metalocation)
 		if err != nil {
@@ -361,5 +384,5 @@ func NewFromLocation(ident Identifier, metalocation string, fsys io.IO, cat Cata
 		}
 	}
 
-	return New(ident, meta, metalocation, fsys, cat), nil
+	return New(ident, meta, metalocation, fsysF, cat), nil
 }
diff --git a/table/table_test.go b/table/table_test.go
index 0b00a5a..cef39a5 100644
--- a/table/table_test.go
+++ b/table/table_test.go
@@ -59,6 +59,13 @@ func TestTable(t *testing.T) {
 	suite.Run(t, new(TableTestSuite))
 }
 
+func mustFS(t *testing.T, tbl *table.Table) iceio.IO {
+	r, err := tbl.FS(context.Background())
+	require.NoError(t, err)
+
+	return r
+}
+
 func (t *TableTestSuite) SetupSuite() {
 	var mockfs internal.MockFS
 	mockfs.Test(t.T())
@@ -66,13 +73,21 @@ func (t *TableTestSuite) SetupSuite() {
 		Return(&internal.MockFile{Contents: bytes.NewReader([]byte(table.ExampleTableMetadataV2))}, nil)
 	defer mockfs.AssertExpectations(t.T())
 
-	tbl, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfs, nil)
+	tbl, err := table.NewFromLocation(
+		context.Background(),
+		[]string{"foo"},
+		"s3://bucket/test/location/uuid.metadata.json",
+		func(ctx context.Context) (iceio.IO, error) {
+			return &mockfs, nil
+		},
+		nil,
+	)
 	t.Require().NoError(err)
 	t.Require().NotNil(tbl)
 
 	t.Equal([]string{"foo"}, tbl.Identifier())
 	t.Equal("s3://bucket/test/location/uuid.metadata.json", tbl.MetadataLocation())
-	t.Equal(&mockfs, tbl.FS())
+	t.Equal(&mockfs, mustFS(t.T(), tbl))
 
 	t.tbl = tbl
 }
@@ -84,7 +99,15 @@ func (t *TableTestSuite) TestNewTableFromReadFile() {
 		Return([]byte(table.ExampleTableMetadataV2), nil)
 	defer mockfsReadFile.AssertExpectations(t.T())
 
-	tbl2, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfsReadFile, nil)
+	tbl2, err := table.NewFromLocation(
+		t.T().Context(),
+		[]string{"foo"},
+		"s3://bucket/test/location/uuid.metadata.json",
+		func(ctx context.Context) (iceio.IO, error) {
+			return &mockfsReadFile, nil
+		},
+		nil,
+	)
 	t.Require().NoError(err)
 	t.Require().NotNil(tbl2)
 
@@ -298,7 +321,15 @@ func (t *TableWritingTestSuite) createTable(identifier table.Identifier, formatV
 		t.location, iceberg.Properties{"format-version": strconv.Itoa(formatVersion)})
 	t.Require().NoError(err)
 
-	return table.New(identifier, meta, t.getMetadataLoc(), iceio.LocalFS{}, nil)
+	return table.New(
+		identifier,
+		meta,
+		t.getMetadataLoc(),
+		func(ctx context.Context) (iceio.IO, error) {
+			return iceio.LocalFS{}, nil
+		},
+		nil,
+	)
 }
 
 func (t *TableWritingTestSuite) TestAddFilesUnpartitioned() {
@@ -311,7 +342,7 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitioned() {
 	files := make([]string, 0)
 	for i := range 5 {
 		filePath := fmt.Sprintf("%s/unpartitioned/test-%d.parquet", t.location, i)
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
 		files = append(files, filePath)
 	}
 
@@ -358,7 +389,7 @@ func (t *TableWritingTestSuite) TestAddFilesFileNotFound() {
 	files := make([]string, 0)
 	for i := range 5 {
 		filePath := fmt.Sprintf("%s/unpartitioned_file_not_found/test-%d.parquet", t.location, i)
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
 		files = append(files, filePath)
 	}
 
@@ -379,7 +410,7 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitionedHasFieldIDs() {
 	files := make([]string, 0)
 	for i := range 5 {
 		filePath := fmt.Sprintf("%s/unpartitioned_with_ids/test-%d.parquet", t.location, i)
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTblWithIDs)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTblWithIDs)
 		files = append(files, filePath)
 	}
 
@@ -408,7 +439,7 @@ func (t *TableWritingTestSuite) TestAddFilesFailsSchemaMismatch() {
 		  {"foo": false, "bar": "bar_string", "baz": "456", "qux": "2024-03-07"}]`,
 	})
 	t.Require().NoError(err)
-	t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, mismatchTable)
+	t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, mismatchTable)
 
 	files := []string{filePath}
 
@@ -451,7 +482,7 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() {
 		t.Require().NoError(err)
 		defer table.Release()
 
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, table)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, table)
 		files = append(files, filePath)
 	}
 
@@ -479,11 +510,11 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() {
 			},
 		})
 
-	m, err := stagedTbl.CurrentSnapshot().Manifests(tbl.FS())
+	m, err := stagedTbl.CurrentSnapshot().Manifests(mustFS(t.T(), tbl))
 	t.Require().NoError(err)
 
 	for _, manifest := range m {
-		entries, err := manifest.FetchEntries(tbl.FS(), false)
+		entries, err := manifest.FetchEntries(mustFS(t.T(), tbl), false)
 		t.Require().NoError(err)
 
 		for _, e := range entries {
@@ -509,7 +540,7 @@ func (t *TableWritingTestSuite) TestAddFilesToBucketPartitionedTableFails() {
 		t.Require().NoError(err)
 		defer table.Release()
 
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, table)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, table)
 		files = append(files, filePath)
 	}
 
@@ -537,7 +568,7 @@ func (t *TableWritingTestSuite) TestAddFilesToPartitionedTableFailsLowerAndUpper
 		t.Require().NoError(err)
 		defer table.Release()
 
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, table)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, table)
 		files = append(files, filePath)
 	}
 
@@ -576,8 +607,8 @@ func (t *TableWritingTestSuite) TestAddFilesWithLargeAndRegular() {
 	t.Require().NoError(err)
 	defer arrTableLarge.Release()
 
-	t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, arrTable)
-	t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePathLarge, arrTableLarge)
+	t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, arrTable)
+	t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePathLarge, arrTableLarge)
 
 	tx := tbl.NewTransaction()
 	t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath, filePathLarge}, nil, false))
@@ -600,7 +631,7 @@ func (t *TableWritingTestSuite) TestAddFilesValidUpcast() {
 	tbl := t.createTable(ident, t.formatVersion, *iceberg.UnpartitionedSpec, t.tableSchemaPromotedTypes)
 
 	filePath := fmt.Sprintf("%s/test_table_with_valid_upcast_v%d/test.parquet", t.location, t.formatVersion)
-	t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTablePromotedTypes)
+	t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTablePromotedTypes)
 
 	tx := tbl.NewTransaction()
 	t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false))
@@ -648,7 +679,7 @@ func (t *TableWritingTestSuite) TestAddFilesSubsetOfSchema() {
 	filePath := fmt.Sprintf("%s/test_table_with_subset_of_schema_v%d/test.parquet", t.location, t.formatVersion)
 	withoutCol := dropColFromTable(0, t.arrTbl)
 	defer withoutCol.Release()
-	t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, withoutCol)
+	t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, withoutCol)
 
 	tx := tbl.NewTransaction()
 	t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false))
@@ -683,7 +714,7 @@ func (t *TableWritingTestSuite) TestAddFilesDuplicateFilesInFilePaths() {
 	tbl := t.createTable(ident, t.formatVersion, *iceberg.UnpartitionedSpec, t.tableSchema)
 
 	filePath := fmt.Sprintf("%s/test_table_with_duplicate_files_v%d/test.parquet", t.location, t.formatVersion)
-	t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+	t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
 
 	tx := tbl.NewTransaction()
 	err := tx.AddFiles(t.ctx, []string{filePath, filePath}, nil, false)
@@ -701,7 +732,7 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedByCurrentSnapshot() {
 	files := make([]string, 0)
 	for i := range 5 {
 		filePath := fmt.Sprintf("%s/add_files_referenced/test-%d.parquet", t.location, i)
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
 		files = append(files, filePath)
 	}
 
@@ -723,7 +754,7 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedCurrentSnapshotIgnoreDupli
 	files := make([]string, 0)
 	for i := range 5 {
 		filePath := fmt.Sprintf("%s/add_files_referenced/test-%d.parquet", t.location, i)
-		t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+		t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
 		files = append(files, filePath)
 	}
 
@@ -735,7 +766,7 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedCurrentSnapshotIgnoreDupli
 	t.Require().NoError(err)
 
 	added, existing, deleted := []int32{}, []int32{}, []int32{}
-	for m, err := range staged.AllManifests() {
+	for m, err := range staged.AllManifests(t.T().Context()) {
 		t.Require().NoError(err)
 		added = append(added, m.AddedDataFiles())
 		existing = append(existing, m.ExistingDataFiles())
@@ -790,7 +821,15 @@ func (t *TableWritingTestSuite) TestReplaceDataFiles() {
 
 	ctx := context.Background()
 
-	tbl := table.New(ident, meta, t.getMetadataLoc(), fs, &mockedCatalog{})
+	tbl := table.New(
+		ident,
+		meta,
+		t.getMetadataLoc(),
+		func(ctx context.Context) (iceio.IO, error) {
+			return fs, nil
+		},
+		&mockedCatalog{},
+	)
 	for i := range 5 {
 		tx := tbl.NewTransaction()
 		t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false))
@@ -798,7 +837,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFiles() {
 		t.Require().NoError(err)
 	}
 
-	mflist, err := tbl.CurrentSnapshot().Manifests(tbl.FS())
+	mflist, err := tbl.CurrentSnapshot().Manifests(mustFS(t.T(), tbl))
 	t.Require().NoError(err)
 	t.Len(mflist, 5)
 
@@ -1105,12 +1144,12 @@ func (t *TableWritingTestSuite) TestMergeManifests() {
 	tblC, err = tblC.AppendTable(t.ctx, arrTable, 1, nil)
 	t.Require().NoError(err)
 
-	manifestList, err := tblA.CurrentSnapshot().Manifests(tblA.FS())
+	manifestList, err := tblA.CurrentSnapshot().Manifests(mustFS(t.T(), tblA))
 	t.Require().NoError(err)
 	t.Len(manifestList, 1)
-	t.validateManifestFileLength(tblA.FS(), manifestList[0])
+	t.validateManifestFileLength(mustFS(t.T(), tblA), manifestList[0])
 
-	entries, err := manifestList[0].FetchEntries(tblA.FS(), false)
+	entries, err := manifestList[0].FetchEntries(mustFS(t.T(), tblA), false)
 	t.Require().NoError(err)
 	t.Len(entries, 3)
 
@@ -1124,20 +1163,20 @@ func (t *TableWritingTestSuite) TestMergeManifests() {
 		}
 	}
 
-	manifestList, err = tblB.CurrentSnapshot().Manifests(tblB.FS())
+	manifestList, err = tblB.CurrentSnapshot().Manifests(mustFS(t.T(), tblB))
 	t.Require().NoError(err)
 	t.Len(manifestList, 3)
 
 	for _, m := range manifestList {
-		t.validateManifestFileLength(tblB.FS(), m)
+		t.validateManifestFileLength(mustFS(t.T(), tblB), m)
 	}
 
-	manifestList, err = tblC.CurrentSnapshot().Manifests(tblC.FS())
+	manifestList, err = tblC.CurrentSnapshot().Manifests(mustFS(t.T(), tblC))
 	t.Require().NoError(err)
 	t.Len(manifestList, 3)
 
 	for _, m := range manifestList {
-		t.validateManifestFileLength(tblC.FS(), m)
+		t.validateManifestFileLength(mustFS(t.T(), tblC), m)
 	}
 
 	resultA, err := tblA.Scan().ToArrowTable(t.ctx)
@@ -1204,9 +1243,8 @@ func TestNullableStructRequiredField(t *testing.T) {
 	sc, err := table.ArrowSchemaToIcebergWithFreshIDs(arrowSchema, false)
 	require.NoError(t, err)
 
-	ctx := context.TODO()
-	require.NoError(t, cat.CreateNamespace(ctx, table.Identifier{"testing"}, nil))
-	tbl, err := cat.CreateTable(ctx, table.Identifier{"testing", "nullable_struct_required_field"}, sc,
+	require.NoError(t, cat.CreateNamespace(t.Context(), table.Identifier{"testing"}, nil))
+	tbl, err := cat.CreateTable(t.Context(), table.Identifier{"testing", "nullable_struct_required_field"}, sc,
 		catalog.WithProperties(iceberg.Properties{"format-version": "2"}),
 		catalog.WithLocation("file://"+loc))
 	require.NoError(t, err)
@@ -1226,7 +1264,7 @@ func TestNullableStructRequiredField(t *testing.T) {
 	defer arrTable.Release()
 
 	tx := tbl.NewTransaction()
-	require.NoError(t, tx.AppendTable(ctx, arrTable, N, nil))
+	require.NoError(t, tx.AppendTable(t.Context(), arrTable, N, nil))
 	stagedTbl, err := tx.StagedTable()
 	require.NoError(t, err)
 	require.NotNil(t, stagedTbl)
@@ -1303,7 +1341,9 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataLogsErrorOnFileNotFound() {
 		table.UnsortedSortOrder, t.location, iceberg.Properties{"format-version": strconv.Itoa(t.formatVersion), "write.metadata.delete-after-commit.enabled": "true"})
 	t.Require().NoError(err)
 
-	tbl := table.New(ident, meta, t.getMetadataLoc(), fs, &DeleteOldMetadataMockedCatalog{})
+	tbl := table.New(ident, meta, t.getMetadataLoc(), func(ctx context.Context) (iceio.IO, error) {
+		return fs, nil
+	}, &DeleteOldMetadataMockedCatalog{})
 	ctx := context.Background()
 
 	// transaction 1 to create metadata file
@@ -1347,7 +1387,15 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataNoErrorLogsOnFileFound() {
 	meta, err := table.NewMetadata(t.tableSchemaPromotedTypes, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, t.location, iceberg.Properties{"format-version": strconv.Itoa(t.formatVersion), "write.metadata.delete-after-commit.enabled": "true"})
 	t.Require().NoError(err)
 
-	tbl := table.New(ident, meta, t.getMetadataLoc(), fs, &DeleteOldMetadataMockedCatalog{})
+	tbl := table.New(
+		ident,
+		meta,
+		t.getMetadataLoc(),
+		func(ctx context.Context) (iceio.IO, error) {
+			return fs, nil
+		},
+		&DeleteOldMetadataMockedCatalog{},
+	)
 
 	ctx := context.Background()
 
diff --git a/table/transaction.go b/table/transaction.go
index 26fa844..6379089 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -116,9 +116,9 @@ func (t *Transaction) apply(updates []Update, reqs []Requirement) error {
 	return nil
 }
 
-func (t *Transaction) appendSnapshotProducer(props iceberg.Properties) *snapshotProducer {
+func (t *Transaction) appendSnapshotProducer(afs io.IO, props iceberg.Properties) *snapshotProducer {
 	manifestMerge := t.meta.props.GetBool(ManifestMergeEnabledKey, ManifestMergeEnabledDefault)
-	updateSnapshot := t.updateSnapshot(props)
+	updateSnapshot := t.updateSnapshot(afs, props)
 	if manifestMerge {
 		return updateSnapshot.mergeAppend()
 	}
@@ -126,10 +126,10 @@ func (t *Transaction) appendSnapshotProducer(props iceberg.Properties) *snapshot
 	return updateSnapshot.fastAppend()
 }
 
-func (t *Transaction) updateSnapshot(props iceberg.Properties) snapshotUpdate {
+func (t *Transaction) updateSnapshot(fs io.IO, props iceberg.Properties) snapshotUpdate {
 	return snapshotUpdate{
 		txn:           t,
-		io:            t.tbl.fs.(io.WriteFileIO),
+		io:            fs.(io.WriteFileIO),
 		snapshotProps: props,
 	}
 }
@@ -150,12 +150,15 @@ func (t *Transaction) AppendTable(ctx context.Context, tbl arrow.Table, batchSiz
 }
 
 func (t *Transaction) Append(ctx context.Context, rdr array.RecordReader, snapshotProps iceberg.Properties) error {
-	appendFiles := t.appendSnapshotProducer(snapshotProps)
-
+	fs, err := t.tbl.fsF(ctx)
+	if err != nil {
+		return err
+	}
+	appendFiles := t.appendSnapshotProducer(fs, snapshotProps)
 	itr := recordsToDataFiles(ctx, t.tbl.Location(), t.meta, recordWritingArgs{
 		sc:        rdr.Schema(),
 		itr:       array.IterFromReader(rdr),
-		fs:        t.tbl.fs.(io.WriteFileIO),
+		fs:        fs.(io.WriteFileIO),
 		writeUUID: &appendFiles.commitUuid,
 	})
 
@@ -215,8 +218,12 @@ func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, files
 		return fmt.Errorf("%w: cannot replace files in a table without an existing snapshot", ErrInvalidOperation)
 	}
 
+	fs, err := t.tbl.fsF(ctx)
+	if err != nil {
+		return err
+	}
 	markedForDeletion := make([]iceberg.DataFile, 0, len(setToDelete))
-	for df, err := range s.dataFiles(t.tbl.fs, nil) {
+	for df, err := range s.dataFiles(fs, nil) {
 		if err != nil {
 			return err
 		}
@@ -247,13 +254,13 @@ func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, files
 	}
 
 	commitUUID := uuid.New()
-	updater := t.updateSnapshot(snapshotProps).mergeOverwrite(&commitUUID)
+	updater := t.updateSnapshot(fs, snapshotProps).mergeOverwrite(&commitUUID)
 
 	for _, df := range markedForDeletion {
 		updater.deleteDataFile(df)
 	}
 
-	dataFiles := filesToDataFiles(ctx, t.tbl.fs, t.meta, slices.Values(filesToAdd))
+	dataFiles := filesToDataFiles(ctx, fs, t.meta, slices.Values(filesToAdd))
 	for df, err := range dataFiles {
 		if err != nil {
 			return err
@@ -282,7 +289,11 @@ func (t *Transaction) AddFiles(ctx context.Context, files []string, snapshotProp
 	if !ignoreDuplicates {
 		if s := t.meta.currentSnapshot(); s != nil {
 			referenced := make([]string, 0)
-			for df, err := range s.dataFiles(t.tbl.fs, nil) {
+			fs, err := t.tbl.fsF(ctx)
+			if err != nil {
+				return err
+			}
+			for df, err := range s.dataFiles(fs, nil) {
 				if err != nil {
 					return err
 				}
@@ -309,9 +320,14 @@ func (t *Transaction) AddFiles(ctx context.Context, files []string, snapshotProp
 		}
 	}
 
-	updater := t.updateSnapshot(snapshotProps).fastAppend()
+	fs, err := t.tbl.fsF(ctx)
+	if err != nil {
+		return err
+	}
+
+	updater := t.updateSnapshot(fs, snapshotProps).fastAppend()
 
-	dataFiles := filesToDataFiles(ctx, t.tbl.fs, t.meta, slices.Values(files))
+	dataFiles := filesToDataFiles(ctx, fs, t.meta, slices.Values(files))
 	for df, err := range dataFiles {
 		if err != nil {
 			return err
@@ -335,7 +351,7 @@ func (t *Transaction) Scan(opts ...ScanOption) (*Scan, error) {
 
 	s := &Scan{
 		metadata:       updatedMeta,
-		io:             t.tbl.fs,
+		ioF:            t.tbl.fsF,
 		rowFilter:      iceberg.AlwaysTrue{},
 		selectedFields: []string{"*"},
 		caseSensitive:  true,
@@ -358,8 +374,15 @@ func (t *Transaction) StagedTable() (*StagedTable, error) {
 		return nil, err
 	}
 
-	return &StagedTable{Table: New(t.tbl.identifier, updatedMeta,
-		updatedMeta.Location(), t.tbl.fs, t.tbl.cat)}, nil
+	return &StagedTable{
+		Table: New(
+			t.tbl.identifier,
+			updatedMeta,
+			updatedMeta.Location(),
+			t.tbl.fsF,
+			t.tbl.cat,
+		),
+	}, nil
 }
 
 func (t *Transaction) Commit(ctx context.Context) (*Table, error) {
diff --git a/table/transaction_test.go b/table/transaction_test.go
index bef954e..40f015c 100644
--- a/table/transaction_test.go
+++ b/table/transaction_test.go
@@ -97,7 +97,7 @@ func (s *SparkIntegrationTestSuite) TestAddFile() {
 	rec := bldr.NewRecord()
 	defer rec.Release()
 
-	fw, err := tbl.FS().(iceio.WriteFileIO).Create(filename)
+	fw, err := mustFS(s.T(), tbl).(iceio.WriteFileIO).Create(filename)
 	if err != nil {
 		panic(err)
 	}