You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2025/06/10 22:32:51 UTC
(iceberg-go) branch main updated: refactor: pass context into actual IO (#452)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new 237da8f refactor: pass context into actual IO (#452)
237da8f is described below
commit 237da8fd2034d772cd5969a372d2e411e3dfc66d
Author: Andrei Tserakhau <la...@gmail.com>
AuthorDate: Wed Jun 11 00:32:46 2025 +0200
refactor: pass context into actual IO (#452)
related https://github.com/apache/iceberg-go/issues/437
All I/O operations must respect the context passed to PlanFiles(ctx) and
simmilar methods.
To support that, we stop passing pre-constructed I/O objects and instead
construct them ad hoc within PlanFiles, using the provided context
---
catalog/glue/glue.go | 31 +++++----
catalog/internal/utils.go | 27 ++++----
catalog/rest/rest.go | 13 ++--
catalog/rest/rest_integration_test.go | 15 +++--
catalog/sql/sql.go | 19 ++++--
cmd/iceberg/output.go | 10 ++-
io/io.go | 12 ++++
table/scanner.go | 25 +++++--
table/table.go | 71 +++++++++++++-------
table/table_test.go | 120 ++++++++++++++++++++++++----------
table/transaction.go | 55 +++++++++++-----
table/transaction_test.go | 2 +-
12 files changed, 275 insertions(+), 125 deletions(-)
diff --git a/catalog/glue/glue.go b/catalog/glue/glue.go
index f3d14c5..65b3899 100644
--- a/catalog/glue/glue.go
+++ b/catalog/glue/glue.go
@@ -223,13 +223,14 @@ func (c *Catalog) LoadTable(ctx context.Context, identifier table.Identifier, pr
}
ctx = utils.WithAwsConfig(ctx, c.awsCfg)
- // TODO: consider providing a way to directly access the S3 iofs to enable testing of the catalog.
- iofs, err := io.LoadFS(ctx, props, location)
- if err != nil {
- return nil, fmt.Errorf("failed to load table %s.%s: %w", database, tableName, err)
- }
- icebergTable, err := table.NewFromLocation(identifier, location, iofs, c)
+ icebergTable, err := table.NewFromLocation(
+ ctx,
+ identifier,
+ location,
+ io.LoadFSFunc(props, location),
+ c,
+ )
if err != nil {
return nil, fmt.Errorf("failed to create table from location %s.%s: %w", database, tableName, err)
}
@@ -255,7 +256,11 @@ func (c *Catalog) CreateTable(ctx context.Context, identifier table.Identifier,
return nil, err
}
- wfs, ok := staged.FS().(io.WriteFileIO)
+ afs, err := staged.FS(ctx)
+ if err != nil {
+ return nil, err
+ }
+ wfs, ok := afs.(io.WriteFileIO)
if !ok {
return nil, errors.New("loaded filesystem IO does not support writing")
}
@@ -317,12 +322,14 @@ func (c *Catalog) RegisterTable(ctx context.Context, identifier table.Identifier
}
// Load the metadata file to get table properties
ctx = utils.WithAwsConfig(ctx, c.awsCfg)
- iofs, err := io.LoadFS(ctx, nil, metadataLocation)
- if err != nil {
- return nil, fmt.Errorf("failed to load metadata file at %s: %w", metadataLocation, err)
- }
// Read the metadata file
- metadata, err := table.NewFromLocation([]string{tableName}, metadataLocation, iofs, c)
+ metadata, err := table.NewFromLocation(
+ ctx,
+ []string{tableName},
+ metadataLocation,
+ io.LoadFSFunc(nil, metadataLocation),
+ c,
+ )
if err != nil {
return nil, fmt.Errorf("failed to read table metadata from %s: %w", metadataLocation, err)
}
diff --git a/catalog/internal/utils.go b/catalog/internal/utils.go
index 7e4c8ca..81b8cc3 100644
--- a/catalog/internal/utils.go
+++ b/catalog/internal/utils.go
@@ -136,13 +136,15 @@ func CreateStagedTable(ctx context.Context, catprops iceberg.Properties, nsprops
ioProps := maps.Clone(catprops)
maps.Copy(ioProps, cfg.Properties)
- fs, err := io.LoadFS(ctx, ioProps, metadataLoc)
- if err != nil {
- return table.StagedTable{}, err
- }
return table.StagedTable{
- Table: table.New(ident, metadata, metadataLoc, fs, nil),
+ Table: table.New(
+ ident,
+ metadata,
+ metadataLoc,
+ io.LoadFSFunc(ioProps, metadataLoc),
+ nil,
+ ),
}, nil
}
@@ -239,10 +241,13 @@ func UpdateAndStageTable(ctx context.Context, current *table.Table, ident table.
return nil, err
}
- fs, err := io.LoadFS(ctx, updated.Properties(), newLocation)
- if err != nil {
- return nil, err
- }
-
- return &table.StagedTable{Table: table.New(ident, updated, newLocation, fs, cat)}, nil
+ return &table.StagedTable{
+ Table: table.New(
+ ident,
+ updated,
+ newLocation,
+ io.LoadFSFunc(updated.Properties(), newLocation),
+ cat,
+ ),
+ }, nil
}
diff --git a/catalog/rest/rest.go b/catalog/rest/rest.go
index 9491649..93c88db 100644
--- a/catalog/rest/rest.go
+++ b/catalog/rest/rest.go
@@ -653,12 +653,13 @@ func checkValidNamespace(ident table.Identifier) error {
}
func (r *Catalog) tableFromResponse(ctx context.Context, identifier []string, metadata table.Metadata, loc string, config iceberg.Properties) (*table.Table, error) {
- iofs, err := iceio.LoadFS(ctx, config, loc)
- if err != nil {
- return nil, err
- }
-
- return table.New(identifier, metadata, loc, iofs, r), nil
+ return table.New(
+ identifier,
+ metadata,
+ loc,
+ iceio.LoadFSFunc(config, loc),
+ r,
+ ), nil
}
func (r *Catalog) ListTables(ctx context.Context, namespace table.Identifier) iter.Seq2[table.Identifier, error] {
diff --git a/catalog/rest/rest_integration_test.go b/catalog/rest/rest_integration_test.go
index edba0d5..4bd2807 100644
--- a/catalog/rest/rest_integration_test.go
+++ b/catalog/rest/rest_integration_test.go
@@ -33,6 +33,7 @@ import (
"github.com/apache/iceberg-go/catalog/rest"
"github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/table"
+ "github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
@@ -254,11 +255,11 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() {
pqfile, err := url.JoinPath(location, "data", "test_commit_table_data", "test.parquet")
s.Require().NoError(err)
- fw, err := tbl.FS().(io.WriteFileIO).Create(pqfile)
+ fw, err := mustFS(s.T(), tbl).(io.WriteFileIO).Create(pqfile)
s.Require().NoError(err)
s.Require().NoError(pqarrow.WriteTable(table, fw, table.NumRows(),
nil, pqarrow.DefaultWriterProps()))
- defer tbl.FS().Remove(pqfile)
+ defer mustFS(s.T(), tbl).Remove(pqfile)
txn := tbl.NewTransaction()
s.Require().NoError(txn.AddFiles(s.ctx, []string{pqfile}, nil, false))
@@ -266,7 +267,7 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() {
s.Require().NoError(err)
mf := []iceberg.ManifestFile{}
- for m, err := range updated.AllManifests() {
+ for m, err := range updated.AllManifests(s.ctx) {
s.Require().NoError(err)
s.Require().NotNil(m)
mf = append(mf, m)
@@ -274,13 +275,19 @@ func (s *RestIntegrationSuite) TestWriteCommitTable() {
s.Len(mf, 1)
s.EqualValues(1, mf[0].AddedDataFiles())
- entries, err := mf[0].FetchEntries(updated.FS(), false)
+ entries, err := mf[0].FetchEntries(mustFS(s.T(), updated), false)
s.Require().NoError(err)
s.Len(entries, 1)
s.Equal(pqfile, entries[0].DataFile().FilePath())
}
+func mustFS(t *testing.T, tbl *table.Table) io.IO {
+ r, err := tbl.FS(context.Background())
+ require.NoError(t, err)
+ return r
+}
+
func TestRestIntegration(t *testing.T) {
suite.Run(t, new(RestIntegrationSuite))
}
diff --git a/catalog/sql/sql.go b/catalog/sql/sql.go
index bb8a083..762a643 100644
--- a/catalog/sql/sql.go
+++ b/catalog/sql/sql.go
@@ -283,7 +283,11 @@ func (c *Catalog) CreateTable(ctx context.Context, ident table.Identifier, sc *i
return nil, fmt.Errorf("%w: %s", catalog.ErrNoSuchNamespace, ns)
}
- wfs, ok := staged.FS().(io.WriteFileIO)
+ afs, err := staged.FS(ctx)
+ if err != nil {
+ return nil, err
+ }
+ wfs, ok := afs.(io.WriteFileIO)
if !ok {
return nil, errors.New("loaded filesystem IO does not support writing")
}
@@ -416,12 +420,13 @@ func (c *Catalog) LoadTable(ctx context.Context, identifier table.Identifier, pr
tblProps := maps.Clone(c.props)
maps.Copy(props, tblProps)
- iofs, err := io.LoadFS(ctx, tblProps, result.MetadataLocation.String)
- if err != nil {
- return nil, err
- }
-
- return table.NewFromLocation(identifier, result.MetadataLocation.String, iofs, c)
+ return table.NewFromLocation(
+ ctx,
+ identifier,
+ result.MetadataLocation.String,
+ io.LoadFSFunc(tblProps, result.MetadataLocation.String),
+ c,
+ )
}
func (c *Catalog) DropTable(ctx context.Context, identifier table.Identifier) error {
diff --git a/cmd/iceberg/output.go b/cmd/iceberg/output.go
index cc31b6d..695ba28 100644
--- a/cmd/iceberg/output.go
+++ b/cmd/iceberg/output.go
@@ -18,6 +18,7 @@
package main
import (
+ "context"
"encoding/json"
"fmt"
"log"
@@ -134,7 +135,12 @@ func (t textOutput) Files(tbl *table.Table, history bool) {
snap.SnapshotID, *snap.SchemaID, manifest),
})
- manifestList, err := snap.Manifests(tbl.FS())
+ afs, err := tbl.FS(context.TODO())
+ if err != nil {
+ t.Error(err)
+ os.Exit(1)
+ }
+ manifestList, err := snap.Manifests(afs)
if err != nil {
t.Error(err)
os.Exit(1)
@@ -144,7 +150,7 @@ func (t textOutput) Files(tbl *table.Table, history bool) {
snapshotTree = append(snapshotTree, pterm.LeveledListItem{
Level: 1, Text: "Manifest: " + m.FilePath(),
})
- datafiles, err := m.FetchEntries(tbl.FS(), false)
+ datafiles, err := m.FetchEntries(afs, false)
if err != nil {
t.Error(err)
os.Exit(1)
diff --git a/io/io.go b/io/io.go
index 523f7c5..248dfed 100644
--- a/io/io.go
+++ b/io/io.go
@@ -294,3 +294,15 @@ func LoadFS(ctx context.Context, props map[string]string, location string) (IO,
return iofs, nil
}
+
+// LoadFSFunc is a helper function to create IO Func factory for later usage
+func LoadFSFunc(props map[string]string, location string) func(ctx context.Context) (IO, error) {
+ return func(ctx context.Context) (IO, error) {
+ iofs, err := LoadFS(ctx, props, location)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load metadata file at %s: %w", location, err)
+ }
+
+ return iofs, nil
+ }
+}
diff --git a/table/scanner.go b/table/scanner.go
index 14a3811..93dd052 100644
--- a/table/scanner.go
+++ b/table/scanner.go
@@ -154,7 +154,7 @@ func openManifest(io io.IO, manifest iceberg.ManifestFile,
type Scan struct {
metadata Metadata
- io io.IO
+ ioF FSysF
rowFilter iceberg.BooleanExpression
selectedFields []string
caseSensitive bool
@@ -299,14 +299,18 @@ func matchDeletesToData(entry iceberg.ManifestEntry, positionalDeletes []iceberg
// fetchPartitionSpecFilteredManifests retrieves the table's current snapshot,
// fetches its manifest files, and applies partition-spec filters to remove irrelevant manifests.
-func (scan *Scan) fetchPartitionSpecFilteredManifests() ([]iceberg.ManifestFile, error) {
+func (scan *Scan) fetchPartitionSpecFilteredManifests(ctx context.Context) ([]iceberg.ManifestFile, error) {
snap := scan.Snapshot()
if snap == nil {
return nil, nil
}
+ afs, err := scan.ioF(ctx)
+ if err != nil {
+ return nil, err
+ }
// Fetch all manifests for the current snapshot.
- manifestList, err := snap.Manifests(scan.io)
+ manifestList, err := snap.Manifests(afs)
if err != nil {
return nil, err
}
@@ -354,8 +358,12 @@ func (scan *Scan) collectManifestEntries(
}
g.Go(func() error {
+ fs, err := scan.ioF(ctx)
+ if err != nil {
+ return err
+ }
partEval := partitionEvaluators.Get(int(mf.PartitionSpecID()))
- manifestEntries, err := openManifest(scan.io, mf, partEval, metricsEval)
+ manifestEntries, err := openManifest(fs, mf, partEval, metricsEval)
if err != nil {
return err
}
@@ -390,7 +398,7 @@ func (scan *Scan) collectManifestEntries(
// building a list of FileScanTasks that match the current Scan criteria.
func (scan *Scan) PlanFiles(ctx context.Context) ([]FileScanTask, error) {
// Step 1: Retrieve filtered manifests based on snapshot and partition specs.
- manifestList, err := scan.fetchPartitionSpecFilteredManifests()
+ manifestList, err := scan.fetchPartitionSpecFilteredManifests(ctx)
if err != nil || len(manifestList) == 0 {
return nil, err
}
@@ -456,9 +464,14 @@ func (scan *Scan) ToArrowRecords(ctx context.Context) (*arrow.Schema, iter.Seq2[
return nil, nil, err
}
+ fs, err := scan.ioF(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+
return (&arrowScan{
metadata: scan.metadata,
- fs: scan.io,
+ fs: fs,
projectedSchema: schema,
boundRowFilter: boundFilter,
caseSensitive: scan.caseSensitive,
diff --git a/table/table.go b/table/table.go
index c91fdfd..b49b240 100644
--- a/table/table.go
+++ b/table/table.go
@@ -33,6 +33,8 @@ import (
"golang.org/x/sync/errgroup"
)
+type FSysF func(ctx context.Context) (io.IO, error)
+
type Identifier = []string
type CatalogIO interface {
@@ -44,8 +46,8 @@ type Table struct {
identifier Identifier
metadata Metadata
metadataLocation string
- fs io.IO
cat CatalogIO
+ fsF FSysF
}
func (t Table) Equals(other Table) bool {
@@ -54,19 +56,19 @@ func (t Table) Equals(other Table) bool {
t.metadata.Equals(other.metadata)
}
-func (t Table) Identifier() Identifier { return t.identifier }
-func (t Table) Metadata() Metadata { return t.metadata }
-func (t Table) MetadataLocation() string { return t.metadataLocation }
-func (t Table) FS() io.IO { return t.fs }
-func (t Table) Schema() *iceberg.Schema { return t.metadata.CurrentSchema() }
-func (t Table) Spec() iceberg.PartitionSpec { return t.metadata.PartitionSpec() }
-func (t Table) SortOrder() SortOrder { return t.metadata.SortOrder() }
-func (t Table) Properties() iceberg.Properties { return t.metadata.Properties() }
-func (t Table) NameMapping() iceberg.NameMapping { return t.metadata.NameMapping() }
-func (t Table) Location() string { return t.metadata.Location() }
-func (t Table) CurrentSnapshot() *Snapshot { return t.metadata.CurrentSnapshot() }
-func (t Table) SnapshotByID(id int64) *Snapshot { return t.metadata.SnapshotByID(id) }
-func (t Table) SnapshotByName(name string) *Snapshot { return t.metadata.SnapshotByName(name) }
+func (t Table) Identifier() Identifier { return t.identifier }
+func (t Table) Metadata() Metadata { return t.metadata }
+func (t Table) MetadataLocation() string { return t.metadataLocation }
+func (t Table) FS(ctx context.Context) (io.IO, error) { return t.fsF(ctx) }
+func (t Table) Schema() *iceberg.Schema { return t.metadata.CurrentSchema() }
+func (t Table) Spec() iceberg.PartitionSpec { return t.metadata.PartitionSpec() }
+func (t Table) SortOrder() SortOrder { return t.metadata.SortOrder() }
+func (t Table) Properties() iceberg.Properties { return t.metadata.Properties() }
+func (t Table) NameMapping() iceberg.NameMapping { return t.metadata.NameMapping() }
+func (t Table) Location() string { return t.metadata.Location() }
+func (t Table) CurrentSnapshot() *Snapshot { return t.metadata.CurrentSnapshot() }
+func (t Table) SnapshotByID(id int64) *Snapshot { return t.metadata.SnapshotByID(id) }
+func (t Table) SnapshotByName(name string) *Snapshot { return t.metadata.SnapshotByName(name) }
func (t Table) Schemas() map[int]*iceberg.Schema {
m := make(map[int]*iceberg.Schema)
for _, s := range t.metadata.Schemas() {
@@ -110,15 +112,23 @@ func (t Table) Append(ctx context.Context, rdr array.RecordReader, snapshotProps
return txn.Commit(ctx)
}
-func (t Table) AllManifests() iter.Seq2[iceberg.ManifestFile, error] {
+func (t Table) AllManifests(ctx context.Context) iter.Seq2[iceberg.ManifestFile, error] {
+ fs, err := t.fsF(ctx)
+ if err != nil {
+ return func(yield func(iceberg.ManifestFile, error) bool) {
+ yield(nil, err)
+ }
+ }
+
type list = tblutils.Enumerated[[]iceberg.ManifestFile]
g := errgroup.Group{}
n := len(t.metadata.Snapshots())
ch := make(chan list, n)
+
for i, sn := range t.metadata.Snapshots() {
g.Go(func() error {
- manifests, err := sn.Manifests(t.fs)
+ manifests, err := sn.Manifests(fs)
if err != nil {
return err
}
@@ -195,10 +205,13 @@ func (t Table) doCommit(ctx context.Context, updates []Update, reqs []Requiremen
if err != nil {
return nil, err
}
+ fs, err := t.fsF(ctx)
+ if err != nil {
+ return nil, err
+ }
+ deleteOldMetadata(fs, t.metadata, newMeta)
- deleteOldMetadata(t.fs, t.metadata, newMeta)
-
- return New(t.identifier, newMeta, newLoc, t.fs, t.cat), nil
+ return New(t.identifier, newMeta, newLoc, t.fsF, t.cat), nil
}
func getFiles(it iter.Seq[MetadataLogEntry]) iter.Seq[string] {
@@ -310,7 +323,7 @@ func WithOptions(opts iceberg.Properties) ScanOption {
func (t Table) Scan(opts ...ScanOption) *Scan {
s := &Scan{
metadata: t.metadata,
- io: t.fs,
+ ioF: t.fsF,
rowFilter: iceberg.AlwaysTrue{},
selectedFields: []string{"*"},
caseSensitive: true,
@@ -327,19 +340,29 @@ func (t Table) Scan(opts ...ScanOption) *Scan {
return s
}
-func New(ident Identifier, meta Metadata, metadataLocation string, fs io.IO, cat CatalogIO) *Table {
+func New(ident Identifier, meta Metadata, metadataLocation string, fsF FSysF, cat CatalogIO) *Table {
return &Table{
identifier: ident,
metadata: meta,
metadataLocation: metadataLocation,
- fs: fs,
+ fsF: fsF,
cat: cat,
}
}
-func NewFromLocation(ident Identifier, metalocation string, fsys io.IO, cat CatalogIO) (*Table, error) {
+func NewFromLocation(
+ ctx context.Context,
+ ident Identifier,
+ metalocation string,
+ fsysF FSysF,
+ cat CatalogIO,
+) (*Table, error) {
var meta Metadata
+ fsys, err := fsysF(ctx)
+ if err != nil {
+ return nil, err
+ }
if rf, ok := fsys.(io.ReadFileIO); ok {
data, err := rf.ReadFile(metalocation)
if err != nil {
@@ -361,5 +384,5 @@ func NewFromLocation(ident Identifier, metalocation string, fsys io.IO, cat Cata
}
}
- return New(ident, meta, metalocation, fsys, cat), nil
+ return New(ident, meta, metalocation, fsysF, cat), nil
}
diff --git a/table/table_test.go b/table/table_test.go
index 0b00a5a..cef39a5 100644
--- a/table/table_test.go
+++ b/table/table_test.go
@@ -59,6 +59,13 @@ func TestTable(t *testing.T) {
suite.Run(t, new(TableTestSuite))
}
+func mustFS(t *testing.T, tbl *table.Table) iceio.IO {
+ r, err := tbl.FS(context.Background())
+ require.NoError(t, err)
+
+ return r
+}
+
func (t *TableTestSuite) SetupSuite() {
var mockfs internal.MockFS
mockfs.Test(t.T())
@@ -66,13 +73,21 @@ func (t *TableTestSuite) SetupSuite() {
Return(&internal.MockFile{Contents: bytes.NewReader([]byte(table.ExampleTableMetadataV2))}, nil)
defer mockfs.AssertExpectations(t.T())
- tbl, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfs, nil)
+ tbl, err := table.NewFromLocation(
+ context.Background(),
+ []string{"foo"},
+ "s3://bucket/test/location/uuid.metadata.json",
+ func(ctx context.Context) (iceio.IO, error) {
+ return &mockfs, nil
+ },
+ nil,
+ )
t.Require().NoError(err)
t.Require().NotNil(tbl)
t.Equal([]string{"foo"}, tbl.Identifier())
t.Equal("s3://bucket/test/location/uuid.metadata.json", tbl.MetadataLocation())
- t.Equal(&mockfs, tbl.FS())
+ t.Equal(&mockfs, mustFS(t.T(), tbl))
t.tbl = tbl
}
@@ -84,7 +99,15 @@ func (t *TableTestSuite) TestNewTableFromReadFile() {
Return([]byte(table.ExampleTableMetadataV2), nil)
defer mockfsReadFile.AssertExpectations(t.T())
- tbl2, err := table.NewFromLocation([]string{"foo"}, "s3://bucket/test/location/uuid.metadata.json", &mockfsReadFile, nil)
+ tbl2, err := table.NewFromLocation(
+ t.T().Context(),
+ []string{"foo"},
+ "s3://bucket/test/location/uuid.metadata.json",
+ func(ctx context.Context) (iceio.IO, error) {
+ return &mockfsReadFile, nil
+ },
+ nil,
+ )
t.Require().NoError(err)
t.Require().NotNil(tbl2)
@@ -298,7 +321,15 @@ func (t *TableWritingTestSuite) createTable(identifier table.Identifier, formatV
t.location, iceberg.Properties{"format-version": strconv.Itoa(formatVersion)})
t.Require().NoError(err)
- return table.New(identifier, meta, t.getMetadataLoc(), iceio.LocalFS{}, nil)
+ return table.New(
+ identifier,
+ meta,
+ t.getMetadataLoc(),
+ func(ctx context.Context) (iceio.IO, error) {
+ return iceio.LocalFS{}, nil
+ },
+ nil,
+ )
}
func (t *TableWritingTestSuite) TestAddFilesUnpartitioned() {
@@ -311,7 +342,7 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitioned() {
files := make([]string, 0)
for i := range 5 {
filePath := fmt.Sprintf("%s/unpartitioned/test-%d.parquet", t.location, i)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
files = append(files, filePath)
}
@@ -358,7 +389,7 @@ func (t *TableWritingTestSuite) TestAddFilesFileNotFound() {
files := make([]string, 0)
for i := range 5 {
filePath := fmt.Sprintf("%s/unpartitioned_file_not_found/test-%d.parquet", t.location, i)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
files = append(files, filePath)
}
@@ -379,7 +410,7 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitionedHasFieldIDs() {
files := make([]string, 0)
for i := range 5 {
filePath := fmt.Sprintf("%s/unpartitioned_with_ids/test-%d.parquet", t.location, i)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTblWithIDs)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTblWithIDs)
files = append(files, filePath)
}
@@ -408,7 +439,7 @@ func (t *TableWritingTestSuite) TestAddFilesFailsSchemaMismatch() {
{"foo": false, "bar": "bar_string", "baz": "456", "qux": "2024-03-07"}]`,
})
t.Require().NoError(err)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, mismatchTable)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, mismatchTable)
files := []string{filePath}
@@ -451,7 +482,7 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() {
t.Require().NoError(err)
defer table.Release()
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, table)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, table)
files = append(files, filePath)
}
@@ -479,11 +510,11 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() {
},
})
- m, err := stagedTbl.CurrentSnapshot().Manifests(tbl.FS())
+ m, err := stagedTbl.CurrentSnapshot().Manifests(mustFS(t.T(), tbl))
t.Require().NoError(err)
for _, manifest := range m {
- entries, err := manifest.FetchEntries(tbl.FS(), false)
+ entries, err := manifest.FetchEntries(mustFS(t.T(), tbl), false)
t.Require().NoError(err)
for _, e := range entries {
@@ -509,7 +540,7 @@ func (t *TableWritingTestSuite) TestAddFilesToBucketPartitionedTableFails() {
t.Require().NoError(err)
defer table.Release()
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, table)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, table)
files = append(files, filePath)
}
@@ -537,7 +568,7 @@ func (t *TableWritingTestSuite) TestAddFilesToPartitionedTableFailsLowerAndUpper
t.Require().NoError(err)
defer table.Release()
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, table)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, table)
files = append(files, filePath)
}
@@ -576,8 +607,8 @@ func (t *TableWritingTestSuite) TestAddFilesWithLargeAndRegular() {
t.Require().NoError(err)
defer arrTableLarge.Release()
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, arrTable)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePathLarge, arrTableLarge)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, arrTable)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePathLarge, arrTableLarge)
tx := tbl.NewTransaction()
t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath, filePathLarge}, nil, false))
@@ -600,7 +631,7 @@ func (t *TableWritingTestSuite) TestAddFilesValidUpcast() {
tbl := t.createTable(ident, t.formatVersion, *iceberg.UnpartitionedSpec, t.tableSchemaPromotedTypes)
filePath := fmt.Sprintf("%s/test_table_with_valid_upcast_v%d/test.parquet", t.location, t.formatVersion)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTablePromotedTypes)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTablePromotedTypes)
tx := tbl.NewTransaction()
t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false))
@@ -648,7 +679,7 @@ func (t *TableWritingTestSuite) TestAddFilesSubsetOfSchema() {
filePath := fmt.Sprintf("%s/test_table_with_subset_of_schema_v%d/test.parquet", t.location, t.formatVersion)
withoutCol := dropColFromTable(0, t.arrTbl)
defer withoutCol.Release()
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, withoutCol)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, withoutCol)
tx := tbl.NewTransaction()
t.Require().NoError(tx.AddFiles(t.ctx, []string{filePath}, nil, false))
@@ -683,7 +714,7 @@ func (t *TableWritingTestSuite) TestAddFilesDuplicateFilesInFilePaths() {
tbl := t.createTable(ident, t.formatVersion, *iceberg.UnpartitionedSpec, t.tableSchema)
filePath := fmt.Sprintf("%s/test_table_with_duplicate_files_v%d/test.parquet", t.location, t.formatVersion)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
tx := tbl.NewTransaction()
err := tx.AddFiles(t.ctx, []string{filePath, filePath}, nil, false)
@@ -701,7 +732,7 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedByCurrentSnapshot() {
files := make([]string, 0)
for i := range 5 {
filePath := fmt.Sprintf("%s/add_files_referenced/test-%d.parquet", t.location, i)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
files = append(files, filePath)
}
@@ -723,7 +754,7 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedCurrentSnapshotIgnoreDupli
files := make([]string, 0)
for i := range 5 {
filePath := fmt.Sprintf("%s/add_files_referenced/test-%d.parquet", t.location, i)
- t.writeParquet(tbl.FS().(iceio.WriteFileIO), filePath, t.arrTbl)
+ t.writeParquet(mustFS(t.T(), tbl).(iceio.WriteFileIO), filePath, t.arrTbl)
files = append(files, filePath)
}
@@ -735,7 +766,7 @@ func (t *TableWritingTestSuite) TestAddFilesReferencedCurrentSnapshotIgnoreDupli
t.Require().NoError(err)
added, existing, deleted := []int32{}, []int32{}, []int32{}
- for m, err := range staged.AllManifests() {
+ for m, err := range staged.AllManifests(t.T().Context()) {
t.Require().NoError(err)
added = append(added, m.AddedDataFiles())
existing = append(existing, m.ExistingDataFiles())
@@ -790,7 +821,15 @@ func (t *TableWritingTestSuite) TestReplaceDataFiles() {
ctx := context.Background()
- tbl := table.New(ident, meta, t.getMetadataLoc(), fs, &mockedCatalog{})
+ tbl := table.New(
+ ident,
+ meta,
+ t.getMetadataLoc(),
+ func(ctx context.Context) (iceio.IO, error) {
+ return fs, nil
+ },
+ &mockedCatalog{},
+ )
for i := range 5 {
tx := tbl.NewTransaction()
t.Require().NoError(tx.AddFiles(ctx, files[i:i+1], nil, false))
@@ -798,7 +837,7 @@ func (t *TableWritingTestSuite) TestReplaceDataFiles() {
t.Require().NoError(err)
}
- mflist, err := tbl.CurrentSnapshot().Manifests(tbl.FS())
+ mflist, err := tbl.CurrentSnapshot().Manifests(mustFS(t.T(), tbl))
t.Require().NoError(err)
t.Len(mflist, 5)
@@ -1105,12 +1144,12 @@ func (t *TableWritingTestSuite) TestMergeManifests() {
tblC, err = tblC.AppendTable(t.ctx, arrTable, 1, nil)
t.Require().NoError(err)
- manifestList, err := tblA.CurrentSnapshot().Manifests(tblA.FS())
+ manifestList, err := tblA.CurrentSnapshot().Manifests(mustFS(t.T(), tblA))
t.Require().NoError(err)
t.Len(manifestList, 1)
- t.validateManifestFileLength(tblA.FS(), manifestList[0])
+ t.validateManifestFileLength(mustFS(t.T(), tblA), manifestList[0])
- entries, err := manifestList[0].FetchEntries(tblA.FS(), false)
+ entries, err := manifestList[0].FetchEntries(mustFS(t.T(), tblA), false)
t.Require().NoError(err)
t.Len(entries, 3)
@@ -1124,20 +1163,20 @@ func (t *TableWritingTestSuite) TestMergeManifests() {
}
}
- manifestList, err = tblB.CurrentSnapshot().Manifests(tblB.FS())
+ manifestList, err = tblB.CurrentSnapshot().Manifests(mustFS(t.T(), tblB))
t.Require().NoError(err)
t.Len(manifestList, 3)
for _, m := range manifestList {
- t.validateManifestFileLength(tblB.FS(), m)
+ t.validateManifestFileLength(mustFS(t.T(), tblB), m)
}
- manifestList, err = tblC.CurrentSnapshot().Manifests(tblC.FS())
+ manifestList, err = tblC.CurrentSnapshot().Manifests(mustFS(t.T(), tblC))
t.Require().NoError(err)
t.Len(manifestList, 3)
for _, m := range manifestList {
- t.validateManifestFileLength(tblC.FS(), m)
+ t.validateManifestFileLength(mustFS(t.T(), tblC), m)
}
resultA, err := tblA.Scan().ToArrowTable(t.ctx)
@@ -1204,9 +1243,8 @@ func TestNullableStructRequiredField(t *testing.T) {
sc, err := table.ArrowSchemaToIcebergWithFreshIDs(arrowSchema, false)
require.NoError(t, err)
- ctx := context.TODO()
- require.NoError(t, cat.CreateNamespace(ctx, table.Identifier{"testing"}, nil))
- tbl, err := cat.CreateTable(ctx, table.Identifier{"testing", "nullable_struct_required_field"}, sc,
+ require.NoError(t, cat.CreateNamespace(t.Context(), table.Identifier{"testing"}, nil))
+ tbl, err := cat.CreateTable(t.Context(), table.Identifier{"testing", "nullable_struct_required_field"}, sc,
catalog.WithProperties(iceberg.Properties{"format-version": "2"}),
catalog.WithLocation("file://"+loc))
require.NoError(t, err)
@@ -1226,7 +1264,7 @@ func TestNullableStructRequiredField(t *testing.T) {
defer arrTable.Release()
tx := tbl.NewTransaction()
- require.NoError(t, tx.AppendTable(ctx, arrTable, N, nil))
+ require.NoError(t, tx.AppendTable(t.Context(), arrTable, N, nil))
stagedTbl, err := tx.StagedTable()
require.NoError(t, err)
require.NotNil(t, stagedTbl)
@@ -1303,7 +1341,9 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataLogsErrorOnFileNotFound() {
table.UnsortedSortOrder, t.location, iceberg.Properties{"format-version": strconv.Itoa(t.formatVersion), "write.metadata.delete-after-commit.enabled": "true"})
t.Require().NoError(err)
- tbl := table.New(ident, meta, t.getMetadataLoc(), fs, &DeleteOldMetadataMockedCatalog{})
+ tbl := table.New(ident, meta, t.getMetadataLoc(), func(ctx context.Context) (iceio.IO, error) {
+ return fs, nil
+ }, &DeleteOldMetadataMockedCatalog{})
ctx := context.Background()
// transaction 1 to create metadata file
@@ -1347,7 +1387,15 @@ func (t *TableWritingTestSuite) TestDeleteOldMetadataNoErrorLogsOnFileFound() {
meta, err := table.NewMetadata(t.tableSchemaPromotedTypes, iceberg.UnpartitionedSpec, table.UnsortedSortOrder, t.location, iceberg.Properties{"format-version": strconv.Itoa(t.formatVersion), "write.metadata.delete-after-commit.enabled": "true"})
t.Require().NoError(err)
- tbl := table.New(ident, meta, t.getMetadataLoc(), fs, &DeleteOldMetadataMockedCatalog{})
+ tbl := table.New(
+ ident,
+ meta,
+ t.getMetadataLoc(),
+ func(ctx context.Context) (iceio.IO, error) {
+ return fs, nil
+ },
+ &DeleteOldMetadataMockedCatalog{},
+ )
ctx := context.Background()
diff --git a/table/transaction.go b/table/transaction.go
index 26fa844..6379089 100644
--- a/table/transaction.go
+++ b/table/transaction.go
@@ -116,9 +116,9 @@ func (t *Transaction) apply(updates []Update, reqs []Requirement) error {
return nil
}
-func (t *Transaction) appendSnapshotProducer(props iceberg.Properties) *snapshotProducer {
+func (t *Transaction) appendSnapshotProducer(afs io.IO, props iceberg.Properties) *snapshotProducer {
manifestMerge := t.meta.props.GetBool(ManifestMergeEnabledKey, ManifestMergeEnabledDefault)
- updateSnapshot := t.updateSnapshot(props)
+ updateSnapshot := t.updateSnapshot(afs, props)
if manifestMerge {
return updateSnapshot.mergeAppend()
}
@@ -126,10 +126,10 @@ func (t *Transaction) appendSnapshotProducer(props iceberg.Properties) *snapshot
return updateSnapshot.fastAppend()
}
-func (t *Transaction) updateSnapshot(props iceberg.Properties) snapshotUpdate {
+func (t *Transaction) updateSnapshot(fs io.IO, props iceberg.Properties) snapshotUpdate {
return snapshotUpdate{
txn: t,
- io: t.tbl.fs.(io.WriteFileIO),
+ io: fs.(io.WriteFileIO),
snapshotProps: props,
}
}
@@ -150,12 +150,15 @@ func (t *Transaction) AppendTable(ctx context.Context, tbl arrow.Table, batchSiz
}
func (t *Transaction) Append(ctx context.Context, rdr array.RecordReader, snapshotProps iceberg.Properties) error {
- appendFiles := t.appendSnapshotProducer(snapshotProps)
-
+ fs, err := t.tbl.fsF(ctx)
+ if err != nil {
+ return err
+ }
+ appendFiles := t.appendSnapshotProducer(fs, snapshotProps)
itr := recordsToDataFiles(ctx, t.tbl.Location(), t.meta, recordWritingArgs{
sc: rdr.Schema(),
itr: array.IterFromReader(rdr),
- fs: t.tbl.fs.(io.WriteFileIO),
+ fs: fs.(io.WriteFileIO),
writeUUID: &appendFiles.commitUuid,
})
@@ -215,8 +218,12 @@ func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, files
return fmt.Errorf("%w: cannot replace files in a table without an existing snapshot", ErrInvalidOperation)
}
+ fs, err := t.tbl.fsF(ctx)
+ if err != nil {
+ return err
+ }
markedForDeletion := make([]iceberg.DataFile, 0, len(setToDelete))
- for df, err := range s.dataFiles(t.tbl.fs, nil) {
+ for df, err := range s.dataFiles(fs, nil) {
if err != nil {
return err
}
@@ -247,13 +254,13 @@ func (t *Transaction) ReplaceDataFiles(ctx context.Context, filesToDelete, files
}
commitUUID := uuid.New()
- updater := t.updateSnapshot(snapshotProps).mergeOverwrite(&commitUUID)
+ updater := t.updateSnapshot(fs, snapshotProps).mergeOverwrite(&commitUUID)
for _, df := range markedForDeletion {
updater.deleteDataFile(df)
}
- dataFiles := filesToDataFiles(ctx, t.tbl.fs, t.meta, slices.Values(filesToAdd))
+ dataFiles := filesToDataFiles(ctx, fs, t.meta, slices.Values(filesToAdd))
for df, err := range dataFiles {
if err != nil {
return err
@@ -282,7 +289,11 @@ func (t *Transaction) AddFiles(ctx context.Context, files []string, snapshotProp
if !ignoreDuplicates {
if s := t.meta.currentSnapshot(); s != nil {
referenced := make([]string, 0)
- for df, err := range s.dataFiles(t.tbl.fs, nil) {
+ fs, err := t.tbl.fsF(ctx)
+ if err != nil {
+ return err
+ }
+ for df, err := range s.dataFiles(fs, nil) {
if err != nil {
return err
}
@@ -309,9 +320,14 @@ func (t *Transaction) AddFiles(ctx context.Context, files []string, snapshotProp
}
}
- updater := t.updateSnapshot(snapshotProps).fastAppend()
+ fs, err := t.tbl.fsF(ctx)
+ if err != nil {
+ return err
+ }
+
+ updater := t.updateSnapshot(fs, snapshotProps).fastAppend()
- dataFiles := filesToDataFiles(ctx, t.tbl.fs, t.meta, slices.Values(files))
+ dataFiles := filesToDataFiles(ctx, fs, t.meta, slices.Values(files))
for df, err := range dataFiles {
if err != nil {
return err
@@ -335,7 +351,7 @@ func (t *Transaction) Scan(opts ...ScanOption) (*Scan, error) {
s := &Scan{
metadata: updatedMeta,
- io: t.tbl.fs,
+ ioF: t.tbl.fsF,
rowFilter: iceberg.AlwaysTrue{},
selectedFields: []string{"*"},
caseSensitive: true,
@@ -358,8 +374,15 @@ func (t *Transaction) StagedTable() (*StagedTable, error) {
return nil, err
}
- return &StagedTable{Table: New(t.tbl.identifier, updatedMeta,
- updatedMeta.Location(), t.tbl.fs, t.tbl.cat)}, nil
+ return &StagedTable{
+ Table: New(
+ t.tbl.identifier,
+ updatedMeta,
+ updatedMeta.Location(),
+ t.tbl.fsF,
+ t.tbl.cat,
+ ),
+ }, nil
}
func (t *Transaction) Commit(ctx context.Context) (*Table, error) {
diff --git a/table/transaction_test.go b/table/transaction_test.go
index bef954e..40f015c 100644
--- a/table/transaction_test.go
+++ b/table/transaction_test.go
@@ -97,7 +97,7 @@ func (s *SparkIntegrationTestSuite) TestAddFile() {
rec := bldr.NewRecord()
defer rec.Release()
- fw, err := tbl.FS().(iceio.WriteFileIO).Create(filename)
+ fw, err := mustFS(s.T(), tbl).(iceio.WriteFileIO).Create(filename)
if err != nil {
panic(err)
}