You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/08/21 16:24:32 UTC

(iceberg-go) branch main updated: fix(manifest): inherit writer content when creating manifest files (#1853) (#1854)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new a6cf20efb fix(manifest): inherit writer content when creating manifest files (#1853) (#1854)
a6cf20efb is described below

commit a6cf20efb1ff675da2fa50ad0bba90b68921419d
Author: Badal Prasad Singh <ba...@datazip.io>
AuthorDate: Fri Aug 21 21:54:26 2026 +0530

    fix(manifest): inherit writer content when creating manifest files (#1853) (#1854)
    
    fix: inherit the writer content
    
    Signed-off-by: badalprasadsingh <ba...@datazip.io>
---
 manifest.go      |  15 ++++-
 manifest_test.go | 166 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 178 insertions(+), 3 deletions(-)

diff --git a/manifest.go b/manifest.go
index 73f1d13e5..3d4b23ce5 100644
--- a/manifest.go
+++ b/manifest.go
@@ -1476,8 +1476,10 @@ func (w *ManifestWriter) Close() error {
 
 type ManifestFileOption func(mf *manifestFile)
 
-// WithManifestFileContent overrides the ManifestContent of a new manifest file with the provided value
-// Default: ManifestContentData
+// WithManifestFileContent sets the ManifestContent of a new manifest file.
+// The default is the content the ManifestWriter was created with, so this is
+// only needed to state that content explicitly. Passing a value that disagrees
+// with the writer makes ToManifestFile fail.
 func WithManifestFileContent(content ManifestContent) ManifestFileOption {
 	return func(mf *manifestFile) {
 		mf.Content = content
@@ -1499,7 +1501,7 @@ func (w *ManifestWriter) ToManifestFile(location string, length int64, opts ...M
 		Path:               location,
 		Len:                length,
 		SpecID:             int32(w.spec.id),
-		Content:            ManifestContentData,
+		Content:            w.content,
 		SeqNumber:          -1,
 		MinSeqNumber:       w.minSeqNum,
 		AddedSnapshotID:    w.snapshotID,
@@ -1516,6 +1518,13 @@ func (w *ManifestWriter) ToManifestFile(location string, length int64, opts ...M
 		apply(&mf)
 	}
 
+	// The writer already stamped w.content into the manifest's own avro metadata,
+	// so a different content here would contradict the file it describes.
+	if mf.Content != w.content {
+		return nil, fmt.Errorf("%w: manifest file content '%s' does not match writer content '%s'",
+			ErrInvalidArgument, mf.Content, w.content)
+	}
+
 	if mf.Content == ManifestContentDeletes && mf.FirstRowIDValue != nil {
 		return nil, errors.New("first_row_id must not be set on delete manifests")
 	}
diff --git a/manifest_test.go b/manifest_test.go
index 17a3721eb..b83b8cd85 100644
--- a/manifest_test.go
+++ b/manifest_test.go
@@ -33,6 +33,8 @@ import (
 	"github.com/apache/arrow-go/v18/arrow/decimal128"
 	"github.com/apache/iceberg-go/internal"
 	iceio "github.com/apache/iceberg-go/io"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 	"github.com/twmb/avro"
 	"github.com/twmb/avro/atype"
@@ -4330,3 +4332,167 @@ func (m *ManifestTestSuite) TestEntriesCloseErrorAsFinalPair() {
 		"terminal error must equal or wrap the simulated close error")
 	m.Equal(1, file.closeCount, "file must be closed exactly once even when Close returns an error")
 }
+
+func contentTestSpec() PartitionSpec { return NewPartitionSpecID(1) }
+
+func contentTestSchema() *Schema {
+	return NewSchema(0, NestedField{ID: 1, Name: "id", Type: PrimitiveTypes.Int64, Required: true})
+}
+
+func contentTestEntry(t *testing.T, spec PartitionSpec, content ManifestEntryContent, path string) ManifestEntry {
+	t.Helper()
+
+	builder, err := NewDataFileBuilder(spec, content, path, ParquetFile, map[int]any{}, nil, nil, 1, 1024)
+	require.NoError(t, err)
+
+	snapshotID, seqNum := int64(1234), int64(1)
+
+	return NewManifestEntry(EntryStatusADDED, &snapshotID, &seqNum, &seqNum, builder.Build())
+}
+
+func writeContentManifest(t *testing.T, version int, entryContent ManifestEntryContent,
+	writerOpts []ManifestWriterOption, fileOpts []ManifestFileOption,
+) ([]byte, ManifestFile, error) {
+	t.Helper()
+
+	schema, spec := contentTestSchema(), contentTestSpec()
+
+	var buf bytes.Buffer
+	cnt := &internal.CountingWriter{W: &buf}
+	writer, err := NewManifestWriter(version, cnt, spec, schema, 1234, writerOpts...)
+	require.NoError(t, err)
+
+	require.NoError(t, writer.Add(contentTestEntry(t, spec, entryContent, "s3://bucket/ns/tbl/data/f.parquet")))
+	require.NoError(t, writer.Close())
+
+	mf, err := writer.ToManifestFile("s3://bucket/ns/tbl/metadata/m.avro", cnt.Count, fileOpts...)
+
+	return buf.Bytes(), mf, err
+}
+
+func TestToManifestFileInheritsWriterContent(t *testing.T) {
+	tests := []struct {
+		name         string
+		writerOpts   []ManifestWriterOption
+		fileOpts     []ManifestFileOption
+		entryContent ManifestEntryContent
+		want         ManifestContent
+	}{
+		{
+			name:         "defaults to data",
+			entryContent: EntryContentData,
+			want:         ManifestContentData,
+		},
+		{
+			name:         "inherits deletes without redundant option",
+			writerOpts:   []ManifestWriterOption{WithManifestWriterContent(ManifestContentDeletes)},
+			entryContent: EntryContentPosDeletes,
+			want:         ManifestContentDeletes,
+		},
+		{
+			name:         "explicit matching deletes option",
+			writerOpts:   []ManifestWriterOption{WithManifestWriterContent(ManifestContentDeletes)},
+			fileOpts:     []ManifestFileOption{WithManifestFileContent(ManifestContentDeletes)},
+			entryContent: EntryContentPosDeletes,
+			want:         ManifestContentDeletes,
+		},
+		{
+			name:         "explicit matching data option",
+			fileOpts:     []ManifestFileOption{WithManifestFileContent(ManifestContentData)},
+			entryContent: EntryContentData,
+			want:         ManifestContentData,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			data, mf, err := writeContentManifest(t, 2, tt.entryContent, tt.writerOpts, tt.fileOpts)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, mf.ManifestContent())
+
+			// remaining fields must keep the values ToManifestFile derives from the writer
+			assert.Equal(t, 2, mf.Version())
+			assert.Equal(t, "s3://bucket/ns/tbl/metadata/m.avro", mf.FilePath())
+			assert.Equal(t, int64(len(data)), mf.Length())
+			assert.Equal(t, int32(1), mf.PartitionSpecID())
+			assert.Equal(t, int64(1234), mf.SnapshotID())
+			assert.Equal(t, int64(-1), mf.SequenceNum())
+			assert.Equal(t, int64(1), mf.MinSequenceNum())
+			assert.Equal(t, int32(1), mf.AddedDataFiles())
+			assert.Zero(t, mf.ExistingDataFiles())
+			assert.Zero(t, mf.DeletedDataFiles())
+			assert.Equal(t, int64(1), mf.AddedRows())
+			assert.Zero(t, mf.ExistingRows())
+			assert.Zero(t, mf.DeletedRows())
+			assert.Empty(t, mf.Partitions())
+			assert.Nil(t, mf.KeyMetadata())
+			assert.Nil(t, mf.FirstRowID())
+
+			entries, err := ReadManifest(mf, bytes.NewReader(data), false)
+			require.NoError(t, err)
+			require.Len(t, entries, 1)
+			assert.Equal(t, tt.entryContent, entries[0].DataFile().ContentType())
+		})
+	}
+}
+
+func TestToManifestFileRejectsContentMismatch(t *testing.T) {
+	tests := []struct {
+		name         string
+		version      int
+		writerOpts   []ManifestWriterOption
+		fileOpts     []ManifestFileOption
+		entryContent ManifestEntryContent
+	}{
+		{
+			name:         "data writer with deletes option",
+			version:      2,
+			fileOpts:     []ManifestFileOption{WithManifestFileContent(ManifestContentDeletes)},
+			entryContent: EntryContentData,
+		},
+		{
+			name:         "deletes writer with data option",
+			version:      2,
+			writerOpts:   []ManifestWriterOption{WithManifestWriterContent(ManifestContentDeletes)},
+			fileOpts:     []ManifestFileOption{WithManifestFileContent(ManifestContentData)},
+			entryContent: EntryContentPosDeletes,
+		},
+		{
+			// v1 has no delete manifests; NewManifestWriter rejects a deletes
+			// writer, and the file option must not smuggle one past that.
+			name:         "v1 writer with deletes option",
+			version:      1,
+			fileOpts:     []ManifestFileOption{WithManifestFileContent(ManifestContentDeletes)},
+			entryContent: EntryContentData,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, mf, err := writeContentManifest(t, tt.version, tt.entryContent, tt.writerOpts, tt.fileOpts)
+			require.ErrorIs(t, err, ErrInvalidArgument)
+			assert.ErrorContains(t, err, "does not match writer content")
+			assert.Nil(t, mf)
+		})
+	}
+}
+
+func TestManifestListRoundTripsDeleteManifestContent(t *testing.T) {
+	manifestData, mf, err := writeContentManifest(t, 2, EntryContentPosDeletes,
+		[]ManifestWriterOption{WithManifestWriterContent(ManifestContentDeletes)}, nil)
+	require.NoError(t, err)
+
+	var listBuf bytes.Buffer
+	seqNum := int64(1)
+	require.NoError(t, WriteManifestList(2, &listBuf, 1234, nil, &seqNum, 0, []ManifestFile{mf}))
+
+	files, err := ReadManifestList(bytes.NewReader(listBuf.Bytes()))
+	require.NoError(t, err)
+	require.Len(t, files, 1)
+	assert.Equal(t, ManifestContentDeletes, files[0].ManifestContent())
+
+	entries, err := ReadManifest(files[0], bytes.NewReader(manifestData), false)
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	assert.Equal(t, EntryContentPosDeletes, entries[0].DataFile().ContentType())
+}