You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/04/30 17:49:48 UTC

(iceberg-go) branch main updated: fix(table): use Abort() on error path to prevent panic in stream() (#962)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new d9714931 fix(table): use Abort() on error path to prevent panic in stream() (#962)
d9714931 is described below

commit d9714931dc2aef95049c6244bbc37ba979ac773c
Author: Tanmay Rauth <t_...@apple.com>
AuthorDate: Thu Apr 30 10:49:42 2026 -0700

    fix(table): use Abort() on error path to prevent panic in stream() (#962)
    
    When stream() exits on an error path after opening a file writer but
    before writing any rows, Close() tries to finalize the Parquet footer
    which panics in the Arrow library. Add Abort() to FileWriter that closes
    only the file handle without finalizing, and use it in the deferred
    cleanup. Also remove dead default branch in closeAndWait().
    
    Fixes #961
---
 table/internal/interfaces.go      |  5 +++
 table/internal/parquet_files.go   |  4 +++
 table/rolling_data_writer.go      | 16 +++-------
 table/rolling_data_writer_test.go | 67 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 81 insertions(+), 11 deletions(-)

diff --git a/table/internal/interfaces.go b/table/internal/interfaces.go
index a38d70be..a3ebf490 100644
--- a/table/internal/interfaces.go
+++ b/table/internal/interfaces.go
@@ -80,6 +80,11 @@ type FileWriter interface {
 	Write(arrow.RecordBatch) error
 	BytesWritten() int64
 	Close() (iceberg.DataFile, error)
+	// Abort closes the underlying file handle without finalizing the
+	// file format (e.g. Parquet footer). It is safe to call regardless
+	// of how many rows have been written and should be used on error
+	// paths where Close() may panic or produce an invalid file.
+	Abort() error
 }
 
 type FileFormat interface {
diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go
index 72e0ef1c..d9f7bb58 100644
--- a/table/internal/parquet_files.go
+++ b/table/internal/parquet_files.go
@@ -395,6 +395,10 @@ func (w *ParquetFileWriter) Close() (_ iceberg.DataFile, err error) {
 	}), nil
 }
 
+func (w *ParquetFileWriter) Abort() error {
+	return w.fileCloser.Close()
+}
+
 type decAsIntAgg[T int32 | int64] struct {
 	*statsAggregator[T]
 }
diff --git a/table/rolling_data_writer.go b/table/rolling_data_writer.go
index 3bec7cbb..11520a2b 100644
--- a/table/rolling_data_writer.go
+++ b/table/rolling_data_writer.go
@@ -309,7 +309,7 @@ func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) {
 	var currentWriter tblutils.FileWriter
 	defer func() {
 		if currentWriter != nil {
-			currentWriter.Close()
+			_ = currentWriter.Abort()
 		}
 	}()
 
@@ -391,17 +391,11 @@ func (r *RollingDataWriter) closeAndWait() error {
 	r.factory.writers.Delete(r.partitionKey)
 	r.wg.Wait()
 
-	select {
-	case err := <-r.errorCh:
-		if err != nil {
-			return fmt.Errorf("error in rolling data writer: %w", err)
-		}
-
-		return nil
-	default:
-
-		return nil
+	if err := <-r.errorCh; err != nil {
+		return fmt.Errorf("error in rolling data writer: %w", err)
 	}
+
+	return nil
 }
 
 func (w *writerFactory) closeAll() error {
diff --git a/table/rolling_data_writer_test.go b/table/rolling_data_writer_test.go
index fc5c0061..43f45f56 100644
--- a/table/rolling_data_writer_test.go
+++ b/table/rolling_data_writer_test.go
@@ -92,6 +92,31 @@ func (s *RollingDataWriterTestSuite) buildRecord(arrSchema *arrow.Schema, numRow
 	return bldr.NewRecordBatch()
 }
 
+func (s *RollingDataWriterTestSuite) createFileWriter(loc string, arrSchema *arrow.Schema, fileName string) tblutils.FileWriter {
+	s.T().Helper()
+
+	icebergSchema, err := ArrowSchemaToIcebergWithFreshIDs(arrSchema, false)
+	s.Require().NoError(err)
+
+	spec := iceberg.NewPartitionSpec()
+	format := tblutils.GetFileFormat(iceberg.ParquetFile)
+	writeProps := format.GetWriteProperties(iceberg.Properties{})
+
+	statsCols, err := computeStatsPlan(icebergSchema, iceberg.Properties{})
+	s.Require().NoError(err)
+
+	fw, err := format.NewFileWriter(s.ctx, iceio.LocalFS{}, nil, tblutils.WriteFileInfo{
+		FileSchema: icebergSchema,
+		FileName:   filepath.Join(loc, fileName),
+		StatsCols:  statsCols,
+		WriteProps: writeProps,
+		Spec:       spec,
+	}, arrSchema)
+	s.Require().NoError(err)
+
+	return fw
+}
+
 func (s *RollingDataWriterTestSuite) TestSingleFileUnderTarget() {
 	arrSchema := arrow.NewSchema([]arrow.Field{
 		{Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
@@ -290,6 +315,48 @@ func (s *RollingDataWriterTestSuite) TestWriterFactoryPropagatesSortOrderID() {
 	s.Equal(expectedSortOrderID, *dataFiles[0].SortOrderID())
 }
 
+func (s *RollingDataWriterTestSuite) TestAbortWithZeroRowsWritten() {
+	arrSchema := arrow.NewSchema([]arrow.Field{
+		{Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+		{Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+	}, nil)
+
+	loc := filepath.ToSlash(s.T().TempDir())
+	fw := s.createFileWriter(loc, arrSchema, "test-abort-zero-rows.parquet")
+
+	// Abort without writing any rows must not panic.
+	s.Require().NoError(fw.Abort())
+}
+
+func (s *RollingDataWriterTestSuite) TestStreamErrorPathUsesAbort() {
+	writerSchema := arrow.NewSchema([]arrow.Field{
+		{Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
+		{Name: "name", Type: arrow.BinaryTypes.String, Nullable: true},
+	}, nil)
+
+	loc := filepath.ToSlash(s.T().TempDir())
+	factory, _ := s.createWriterFactory(loc, writerSchema, 1024*1024)
+	defer factory.closeAll()
+
+	outputCh := make(chan iceberg.DataFile, 10)
+	writer := factory.newRollingDataWriter(s.ctx, nil, "", nil, outputCh)
+
+	// Send a record with an incompatible schema to trigger a
+	// ToRequestedSchema error, exercising the deferred Abort() path.
+	badSchema := arrow.NewSchema([]arrow.Field{
+		{Name: "x", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
+	}, nil)
+
+	bldr := array.NewRecordBuilder(s.mem, badSchema)
+	bldr.Field(0).(*array.Float64Builder).Append(1.0)
+	badRecord := bldr.NewRecordBatch()
+	bldr.Release()
+	defer badRecord.Release()
+
+	s.Require().NoError(writer.Add(badRecord))
+	s.Require().Error(writer.closeAndWait())
+}
+
 func (s *RollingDataWriterTestSuite) TestBytesWrittenReflectsCompressedSize() {
 	arrSchema := arrow.NewSchema([]arrow.Field{
 		{Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true},