You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/30 17:49:48 UTC
(iceberg-go) branch main updated: fix(table): use Abort() on error path to prevent panic in stream() (#962)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new d9714931 fix(table): use Abort() on error path to prevent panic in stream() (#962)
d9714931 is described below
commit d9714931dc2aef95049c6244bbc37ba979ac773c
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Thu Apr 30 10:49:42 2026 -0700
fix(table): use Abort() on error path to prevent panic in stream() (#962)
When stream() exits on an error path after opening a file writer but
before writing any rows, Close() tries to finalize the Parquet footer
which panics in the Arrow library. Add Abort() to FileWriter that closes
only the file handle without finalizing, and use it in the deferred
cleanup. Also remove dead default branch in closeAndWait().
Fixes #961
---
table/internal/interfaces.go | 5 +++
table/internal/parquet_files.go | 4 +++
table/rolling_data_writer.go | 16 +++-------
table/rolling_data_writer_test.go | 67 +++++++++++++++++++++++++++++++++++++++
4 files changed, 81 insertions(+), 11 deletions(-)
diff --git a/table/internal/interfaces.go b/table/internal/interfaces.go
index a38d70be..a3ebf490 100644
--- a/table/internal/interfaces.go
+++ b/table/internal/interfaces.go
@@ -80,6 +80,11 @@ type FileWriter interface {
Write(arrow.RecordBatch) error
BytesWritten() int64
Close() (iceberg.DataFile, error)
+ // Abort closes the underlying file handle without finalizing the
+ // file format (e.g. Parquet footer). It is safe to call regardless
+ // of how many rows have been written and should be used on error
+ // paths where Close() may panic or produce an invalid file.
+ Abort() error
}
type FileFormat interface {
diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go
index 72e0ef1c..d9f7bb58 100644
--- a/table/internal/parquet_files.go
+++ b/table/internal/parquet_files.go
@@ -395,6 +395,10 @@ func (w *ParquetFileWriter) Close() (_ iceberg.DataFile, err error) {
}), nil
}
+func (w *ParquetFileWriter) Abort() error {
+ return w.fileCloser.Close()
+}
+
type decAsIntAgg[T int32 | int64] struct {
*statsAggregator[T]
}
diff --git a/table/rolling_data_writer.go b/table/rolling_data_writer.go
index 3bec7cbb..11520a2b 100644
--- a/table/rolling_data_writer.go
+++ b/table/rolling_data_writer.go
@@ -309,7 +309,7 @@ func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) {
var currentWriter tblutils.FileWriter
defer func() {
if currentWriter != nil {
- currentWriter.Close()
+ _ = currentWriter.Abort()
}
}()
@@ -391,17 +391,11 @@ func (r *RollingDataWriter) closeAndWait() error {
r.factory.writers.Delete(r.partitionKey)
r.wg.Wait()
- select {
- case err := <-r.errorCh:
- if err != nil {
- return fmt.Errorf("error in rolling data writer: %w", err)
- }
-
- return nil
- default:
-
- return nil
+ if err := <-r.errorCh; err != nil {
+ return fmt.Errorf("error in rolling data writer: %w", err)
}
+
+ return nil
}
func (w *writerFactory) closeAll() error {
diff --git a/table/rolling_data_writer_test.go b/table/rolling_data_writer_test.go
index fc5c0061..43f45f56 100644
--- a/table/rolling_data_writer_test.go
+++ b/table/rolling_data_writer_test.go
@@ -92,6 +92,31 @@ func (s *RollingDataWriterTestSuite) buildRecord(arrSchema *arrow.Schema, numRow
return bldr.NewRecordBatch()
}
+func (s *RollingDataWriterTestSuite) createFileWriter(loc string, arrSchema *arrow.Schema, fileName string) tblutils.FileWriter {
+ s.T().Helper()
+
+ icebergSchema, err := ArrowSchemaToIcebergWithFreshIDs(arrSchema, false)
+ s.Require().NoError(err)
+
+ spec := iceberg.NewPartitionSpec()
+ format := tblutils.GetFileFormat(iceberg.ParquetFile)
+ writeProps := format.GetWriteProperties(iceberg.Properties{})
+
+ statsCols, err := computeStatsPlan(icebergSchema, iceberg.Properties{})
+ s.Require().NoError(err)
+
+ fw, err := format.NewFileWriter(s.ctx, iceio.LocalFS{}, nil, tblutils.WriteFileInfo{
+ FileSchema: icebergSchema,
+ FileName: filepath.Join(loc, fileName),
+ StatsCols: statsCols,
+ WriteProps: writeProps,
+ Spec: spec,
+ }, arrSchema)
+ s.Require().NoError(err)
+
+ return fw
+}
+
func (s *RollingDataWriterTestSuite) TestSingleFileUnderTarget() {
arrSchema := arrow.NewSchema([]arrow.Field{
{Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
@@ -290,6 +315,48 @@ func (s *RollingDataWriterTestSuite) TestWriterFactoryPropagatesSortOrderID() {
s.Equal(expectedSortOrderID, *dataFiles[0].SortOrderID())
}
+func (s *RollingDataWriterTestSuite) TestAbortWithZeroRowsWritten() {
+ arrSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+ {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+
+ loc := filepath.ToSlash(s.T().TempDir())
+ fw := s.createFileWriter(loc, arrSchema, "test-abort-zero-rows.parquet")
+
+ // Abort without writing any rows must not panic.
+ s.Require().NoError(fw.Abort())
+}
+
+func (s *RollingDataWriterTestSuite) TestStreamErrorPathUsesAbort() {
+ writerSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+ {Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+ }, nil)
+
+ loc := filepath.ToSlash(s.T().TempDir())
+ factory, _ := s.createWriterFactory(loc, writerSchema, 1024*1024)
+ defer factory.closeAll()
+
+ outputCh := make(chan iceberg.DataFile, 10)
+ writer := factory.newRollingDataWriter(s.ctx, nil, "", nil, outputCh)
+
+ // Send a record with an incompatible schema to trigger a
+ // ToRequestedSchema error, exercising the deferred Abort() path.
+ badSchema := arrow.NewSchema([]arrow.Field{
+ {Name: "x", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
+ }, nil)
+
+ bldr := array.NewRecordBuilder(s.mem, badSchema)
+ bldr.Field(0).(*array.Float64Builder).Append(1.0)
+ badRecord := bldr.NewRecordBatch()
+ bldr.Release()
+ defer badRecord.Release()
+
+ s.Require().NoError(writer.Add(badRecord))
+ s.Require().Error(writer.closeAndWait())
+}
+
func (s *RollingDataWriterTestSuite) TestBytesWrittenReflectsCompressedSize() {
arrSchema := arrow.NewSchema([]arrow.Field{
{Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},