You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/07/17 01:13:42 UTC

(iceberg-go) branch main updated: fix(table): guard malformed parquet UUID stats (#1414)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git


The following commit(s) were added to refs/heads/main by this push:
     new af424d31 fix(table): guard malformed parquet UUID stats (#1414)
af424d31 is described below

commit af424d3136dc0a0ff0fd706f6c7f1a051f334cff
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Fri Jul 17 03:13:36 2026 +0200

    fix(table): guard malformed parquet UUID stats (#1414)
    
    ## What changed
    - Guard UUID parquet stats aggregation with a panic-safe wrapper in
    DataFileStatsFromMeta.
    - If UUID bounds parsing panics (e.g. malformed fixed bytes), invalidate
    stats for that column so read planning still succeeds.
    - Add regression coverage for malformed UUID min/max bytes and verify
    column bounds are omitted.
    
    ## Validation
    - go test ./table/internal -run
    TestDataFileStatsFromMetaWithMalformedUUIDStats -count=1
    - go test ./table/internal
---
 table/internal/parquet_files.go      |  61 +++++++++++++++++++--
 table/internal/parquet_files_test.go | 102 +++++++++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+), 3 deletions(-)

diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go
index 666ca0ad..62496a02 100644
--- a/table/internal/parquet_files.go
+++ b/table/internal/parquet_files.go
@@ -712,7 +712,19 @@ type wrappedUUIDStats struct {
 	*metadata.FixedLenByteArrayStatistics
 }
 
+func (w *wrappedUUIDStats) HasMinMax() bool {
+	const uuidByteWidth = 16
+
+	return w.FixedLenByteArrayStatistics.HasMinMax() &&
+		len(w.FixedLenByteArrayStatistics.Min()) == uuidByteWidth &&
+		len(w.FixedLenByteArrayStatistics.Max()) == uuidByteWidth
+}
+
 func (w *wrappedUUIDStats) Min() uuid.UUID {
+	if !w.HasMinMax() {
+		return uuid.Nil
+	}
+
 	uid, err := uuid.FromBytes(w.FixedLenByteArrayStatistics.Min())
 	if err != nil {
 		panic(err)
@@ -722,6 +734,10 @@ func (w *wrappedUUIDStats) Min() uuid.UUID {
 }
 
 func (w *wrappedUUIDStats) Max() uuid.UUID {
+	if !w.HasMinMax() {
+		return uuid.Nil
+	}
+
 	uid, err := uuid.FromBytes(w.FixedLenByteArrayStatistics.Max())
 	if err != nil {
 		panic(err)
@@ -732,6 +748,13 @@ func (w *wrappedUUIDStats) Max() uuid.UUID {
 
 type wrappedFLBAStats struct {
 	*metadata.FixedLenByteArrayStatistics
+	expectedLen int
+}
+
+func (w *wrappedFLBAStats) HasMinMax() bool {
+	return w.FixedLenByteArrayStatistics.HasMinMax() &&
+		len(w.FixedLenByteArrayStatistics.Min()) == w.expectedLen &&
+		len(w.FixedLenByteArrayStatistics.Max()) == w.expectedLen
 }
 
 func (w *wrappedFLBAStats) Min() []byte {
@@ -744,10 +767,21 @@ func (w *wrappedFLBAStats) Max() []byte {
 
 type wrappedDecStats struct {
 	*metadata.FixedLenByteArrayStatistics
-	scale int
+	expectedLen int
+	scale       int
+}
+
+func (w wrappedDecStats) HasMinMax() bool {
+	return w.FixedLenByteArrayStatistics.HasMinMax() &&
+		len(w.FixedLenByteArrayStatistics.Min()) == w.expectedLen &&
+		len(w.FixedLenByteArrayStatistics.Max()) == w.expectedLen
 }
 
 func (w wrappedDecStats) Min() iceberg.Decimal {
+	if !w.HasMinMax() {
+		return iceberg.Decimal{Scale: w.scale}
+	}
+
 	dec, err := BigEndianToDecimal(w.FixedLenByteArrayStatistics.Min())
 	if err != nil {
 		panic(err)
@@ -757,6 +791,10 @@ func (w wrappedDecStats) Min() iceberg.Decimal {
 }
 
 func (w wrappedDecStats) Max() iceberg.Decimal {
+	if !w.HasMinMax() {
+		return iceberg.Decimal{Scale: w.scale}
+	}
+
 	dec, err := BigEndianToDecimal(w.FixedLenByteArrayStatistics.Max())
 	if err != nil {
 		panic(err)
@@ -848,6 +886,9 @@ func (p parquetFormat) DataFileStatsFromMeta(meta Metadata, statsCols map[int]St
 			if statsCol.Mode.Typ == MetricModeNone {
 				continue
 			}
+			if _, invalid := invalidateCol[fieldID]; invalid {
+				continue
+			}
 
 			colSizes[fieldID] += colChunk.TotalCompressedSize()
 			valueCounts[fieldID] += colChunk.NumValues()
@@ -903,20 +944,34 @@ func (p parquetFormat) DataFileStatsFromMeta(meta Metadata, statsCols map[int]St
 			case iceberg.StringType:
 				stats = &wrappedStringStats{stats.(*metadata.ByteArrayStatistics)}
 			case iceberg.FixedType:
-				stats = &wrappedFLBAStats{stats.(*metadata.FixedLenByteArrayStatistics)}
+				stats = &wrappedFLBAStats{
+					FixedLenByteArrayStatistics: stats.(*metadata.FixedLenByteArrayStatistics),
+					expectedLen:                 t.Len(),
+				}
 			case iceberg.DecimalType:
 				// Decimals can be stored as INT32/INT64 (small precision) or FIXED_LEN_BYTE_ARRAY/BYTE_ARRAY.
 				// Only wrap FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY statistics; INT32/INT64 stats
 				// are used directly by decAsIntAgg.
 				switch s := stats.(type) {
 				case *metadata.FixedLenByteArrayStatistics:
-					stats = &wrappedDecStats{s, t.Scale()}
+					stats = &wrappedDecStats{
+						FixedLenByteArrayStatistics: s,
+						expectedLen:                 internal.DecimalRequiredBytes(t.Precision()),
+						scale:                       t.Scale(),
+					}
 				case *metadata.ByteArrayStatistics:
 					stats = &wrappedDecByteArrayStats{s, t.Scale()}
 					// INT32/INT64 statistics are used directly by decAsIntAgg
 				}
 			}
 
+			if !stats.HasMinMax() {
+				invalidateCol[fieldID] = struct{}{}
+				delete(colAggs, fieldID)
+
+				continue
+			}
+
 			agg.Update(stats)
 		}
 
diff --git a/table/internal/parquet_files_test.go b/table/internal/parquet_files_test.go
index 75cf8a35..37039abd 100644
--- a/table/internal/parquet_files_test.go
+++ b/table/internal/parquet_files_test.go
@@ -528,6 +528,108 @@ func TestMetricsPrimitiveTypes(t *testing.T) {
 	})
 }
 
+func TestDataFileStatsFromMetaWithMalformedUUIDStats(t *testing.T) {
+	format := internal.GetFileFormat(iceberg.ParquetFile)
+
+	meta, tblMeta := constructTestTablePrimitiveTypes(t)
+	require.NotNil(t, tblMeta)
+	require.NotNil(t, meta)
+
+	mapping, err := format.PathToIDMapping(tblMeta.CurrentSchema())
+	require.NoError(t, err)
+
+	collector := getCollector()
+	found := false
+	for _, rg := range meta.RowGroups {
+		for _, col := range rg.Columns {
+			if col == nil || col.MetaData == nil || col.MetaData.Statistics == nil {
+				continue
+			}
+			path := col.MetaData.GetPathInSchema()
+			if len(path) == 1 && path[0] == "uuids" {
+				col.MetaData.Statistics.Min = []byte{0x01}
+				col.MetaData.Statistics.Max = []byte{0x02}
+				col.MetaData.Statistics.MinValue = []byte{0x01}
+				col.MetaData.Statistics.MaxValue = []byte{0x02}
+				found = true
+			}
+		}
+	}
+	require.True(t, found, "failed to inject malformed UUID stats")
+
+	var fileStats *internal.DataFileStatistics
+	assert.NotPanics(t, func() {
+		fileStats = format.DataFileStatsFromMeta(internal.Metadata(meta), collector, mapping, nil)
+	})
+	require.NotNil(t, fileStats)
+	require.NotContains(t, fileStats.ColAggs, 11)
+
+	dataFile := fileStats.ToDataFile(internal.DataFileOpts{
+		Schema:      tblMeta.CurrentSchema(),
+		Spec:        tblMeta.PartitionSpec(),
+		Path:        "fake-path.parquet",
+		Format:      iceberg.ParquetFile,
+		Content:     iceberg.EntryContentData,
+		FileSize:    meta.GetSourceFileSize(),
+		SortOrderID: 7,
+	})
+
+	assert.Len(t, dataFile.LowerBoundValues(), len(collector)-1)
+	assert.NotContains(t, dataFile.LowerBoundValues(), 11)
+	assert.NotContains(t, dataFile.UpperBoundValues(), 11)
+}
+
+func TestDataFileStatsFromMetaWithMalformedFixedLenDecimalStats(t *testing.T) {
+	format := internal.GetFileFormat(iceberg.ParquetFile)
+
+	meta, tblMeta := constructTestTablePrimitiveTypes(t)
+	require.NotNil(t, tblMeta)
+	require.NotNil(t, meta)
+
+	mapping, err := format.PathToIDMapping(tblMeta.CurrentSchema())
+	require.NoError(t, err)
+
+	collector := getCollector()
+	found := false
+	for _, rg := range meta.RowGroups {
+		for _, col := range rg.Columns {
+			if col == nil || col.MetaData == nil || col.MetaData.Statistics == nil {
+				continue
+			}
+			path := col.MetaData.GetPathInSchema()
+			if len(path) == 1 && path[0] == "large_dec" {
+				col.MetaData.Statistics.Min = []byte{0x01}
+				col.MetaData.Statistics.Max = []byte{0x02}
+				col.MetaData.Statistics.MinValue = []byte{0x01}
+				col.MetaData.Statistics.MaxValue = []byte{0x02}
+				found = true
+			}
+		}
+	}
+	require.True(t, found, "failed to inject malformed fixed-len decimal stats")
+
+	var fileStats *internal.DataFileStatistics
+	assert.NotPanics(t, func() {
+		fileStats = format.DataFileStatsFromMeta(internal.Metadata(meta), collector, mapping, nil)
+	})
+	require.NotNil(t, fileStats)
+	require.NotContains(t, fileStats.ColAggs, 15)
+
+	dataFile := fileStats.ToDataFile(internal.DataFileOpts{
+		Schema:      tblMeta.CurrentSchema(),
+		Spec:        tblMeta.PartitionSpec(),
+		Path:        "fake-path.parquet",
+		Format:      iceberg.ParquetFile,
+		Content:     iceberg.EntryContentData,
+		FileSize:    meta.GetSourceFileSize(),
+		SortOrderID: 7,
+	})
+
+	assert.Len(t, dataFile.LowerBoundValues(), len(collector)-1)
+	assert.NotContains(t, dataFile.LowerBoundValues(), 15)
+	assert.NotContains(t, dataFile.UpperBoundValues(), 15)
+}
+
 // TestNanosecondTimestampMetrics tests that nanosecond timestamp types (v3)
 // are correctly handled for Parquet stats collection and physical type mapping.
 func TestNanosecondTimestampMetrics(t *testing.T) {