You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ze...@apache.org on 2026/07/17 01:13:42 UTC
(iceberg-go) branch main updated: fix(table): guard malformed parquet UUID stats (#1414)
This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-go.git
The following commit(s) were added to refs/heads/main by this push:
new af424d31 fix(table): guard malformed parquet UUID stats (#1414)
af424d31 is described below
commit af424d3136dc0a0ff0fd706f6c7f1a051f334cff
Author: Minh Vu <vu...@gmail.com>
AuthorDate: Fri Jul 17 03:13:36 2026 +0200
fix(table): guard malformed parquet UUID stats (#1414)
## What changed
- Guard UUID parquet stats aggregation with a panic-safe wrapper in
DataFileStatsFromMeta.
- If UUID bounds parsing panics (e.g. malformed fixed bytes), invalidate
stats for that column so read planning still succeeds.
- Add regression coverage for malformed UUID min/max bytes and verify
column bounds are omitted.
## Validation
- go test ./table/internal -run
TestDataFileStatsFromMetaWithMalformedUUIDStats -count=1
- go test ./table/internal
---
table/internal/parquet_files.go | 61 +++++++++++++++++++--
table/internal/parquet_files_test.go | 102 +++++++++++++++++++++++++++++++++++
2 files changed, 160 insertions(+), 3 deletions(-)
diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go
index 666ca0ad..62496a02 100644
--- a/table/internal/parquet_files.go
+++ b/table/internal/parquet_files.go
@@ -712,7 +712,19 @@ type wrappedUUIDStats struct {
*metadata.FixedLenByteArrayStatistics
}
+func (w *wrappedUUIDStats) HasMinMax() bool {
+ const uuidByteWidth = 16
+
+ return w.FixedLenByteArrayStatistics.HasMinMax() &&
+ len(w.FixedLenByteArrayStatistics.Min()) == uuidByteWidth &&
+ len(w.FixedLenByteArrayStatistics.Max()) == uuidByteWidth
+}
+
func (w *wrappedUUIDStats) Min() uuid.UUID {
+ if !w.HasMinMax() {
+ return uuid.Nil
+ }
+
uid, err := uuid.FromBytes(w.FixedLenByteArrayStatistics.Min())
if err != nil {
panic(err)
@@ -722,6 +734,10 @@ func (w *wrappedUUIDStats) Min() uuid.UUID {
}
func (w *wrappedUUIDStats) Max() uuid.UUID {
+ if !w.HasMinMax() {
+ return uuid.Nil
+ }
+
uid, err := uuid.FromBytes(w.FixedLenByteArrayStatistics.Max())
if err != nil {
panic(err)
@@ -732,6 +748,13 @@ func (w *wrappedUUIDStats) Max() uuid.UUID {
type wrappedFLBAStats struct {
*metadata.FixedLenByteArrayStatistics
+ expectedLen int
+}
+
+func (w *wrappedFLBAStats) HasMinMax() bool {
+ return w.FixedLenByteArrayStatistics.HasMinMax() &&
+ len(w.FixedLenByteArrayStatistics.Min()) == w.expectedLen &&
+ len(w.FixedLenByteArrayStatistics.Max()) == w.expectedLen
}
func (w *wrappedFLBAStats) Min() []byte {
@@ -744,10 +767,21 @@ func (w *wrappedFLBAStats) Max() []byte {
type wrappedDecStats struct {
*metadata.FixedLenByteArrayStatistics
- scale int
+ expectedLen int
+ scale int
+}
+
+func (w wrappedDecStats) HasMinMax() bool {
+ return w.FixedLenByteArrayStatistics.HasMinMax() &&
+ len(w.FixedLenByteArrayStatistics.Min()) == w.expectedLen &&
+ len(w.FixedLenByteArrayStatistics.Max()) == w.expectedLen
}
func (w wrappedDecStats) Min() iceberg.Decimal {
+ if !w.HasMinMax() {
+ return iceberg.Decimal{Scale: w.scale}
+ }
+
dec, err := BigEndianToDecimal(w.FixedLenByteArrayStatistics.Min())
if err != nil {
panic(err)
@@ -757,6 +791,10 @@ func (w wrappedDecStats) Min() iceberg.Decimal {
}
func (w wrappedDecStats) Max() iceberg.Decimal {
+ if !w.HasMinMax() {
+ return iceberg.Decimal{Scale: w.scale}
+ }
+
dec, err := BigEndianToDecimal(w.FixedLenByteArrayStatistics.Max())
if err != nil {
panic(err)
@@ -848,6 +886,9 @@ func (p parquetFormat) DataFileStatsFromMeta(meta Metadata, statsCols map[int]St
if statsCol.Mode.Typ == MetricModeNone {
continue
}
+ if _, invalid := invalidateCol[fieldID]; invalid {
+ continue
+ }
colSizes[fieldID] += colChunk.TotalCompressedSize()
valueCounts[fieldID] += colChunk.NumValues()
@@ -903,20 +944,34 @@ func (p parquetFormat) DataFileStatsFromMeta(meta Metadata, statsCols map[int]St
case iceberg.StringType:
stats = &wrappedStringStats{stats.(*metadata.ByteArrayStatistics)}
case iceberg.FixedType:
- stats = &wrappedFLBAStats{stats.(*metadata.FixedLenByteArrayStatistics)}
+ stats = &wrappedFLBAStats{
+ FixedLenByteArrayStatistics: stats.(*metadata.FixedLenByteArrayStatistics),
+ expectedLen: t.Len(),
+ }
case iceberg.DecimalType:
// Decimals can be stored as INT32/INT64 (small precision) or FIXED_LEN_BYTE_ARRAY/BYTE_ARRAY.
// Only wrap FIXED_LEN_BYTE_ARRAY and BYTE_ARRAY statistics; INT32/INT64 stats
// are used directly by decAsIntAgg.
switch s := stats.(type) {
case *metadata.FixedLenByteArrayStatistics:
- stats = &wrappedDecStats{s, t.Scale()}
+ stats = &wrappedDecStats{
+ FixedLenByteArrayStatistics: s,
+ expectedLen: internal.DecimalRequiredBytes(t.Precision()),
+ scale: t.Scale(),
+ }
case *metadata.ByteArrayStatistics:
stats = &wrappedDecByteArrayStats{s, t.Scale()}
// INT32/INT64 statistics are used directly by decAsIntAgg
}
}
+ if !stats.HasMinMax() {
+ invalidateCol[fieldID] = struct{}{}
+ delete(colAggs, fieldID)
+
+ continue
+ }
+
agg.Update(stats)
}
diff --git a/table/internal/parquet_files_test.go b/table/internal/parquet_files_test.go
index 75cf8a35..37039abd 100644
--- a/table/internal/parquet_files_test.go
+++ b/table/internal/parquet_files_test.go
@@ -528,6 +528,108 @@ func TestMetricsPrimitiveTypes(t *testing.T) {
})
}
+func TestDataFileStatsFromMetaWithMalformedUUIDStats(t *testing.T) {
+ format := internal.GetFileFormat(iceberg.ParquetFile)
+
+ meta, tblMeta := constructTestTablePrimitiveTypes(t)
+ require.NotNil(t, tblMeta)
+ require.NotNil(t, meta)
+
+ mapping, err := format.PathToIDMapping(tblMeta.CurrentSchema())
+ require.NoError(t, err)
+
+ collector := getCollector()
+ found := false
+ for _, rg := range meta.RowGroups {
+ for _, col := range rg.Columns {
+ if col == nil || col.MetaData == nil || col.MetaData.Statistics == nil {
+ continue
+ }
+ path := col.MetaData.GetPathInSchema()
+ if len(path) == 1 && path[0] == "uuids" {
+ col.MetaData.Statistics.Min = []byte{0x01}
+ col.MetaData.Statistics.Max = []byte{0x02}
+ col.MetaData.Statistics.MinValue = []byte{0x01}
+ col.MetaData.Statistics.MaxValue = []byte{0x02}
+ found = true
+ }
+ }
+ }
+ require.True(t, found, "failed to inject malformed UUID stats")
+
+ var fileStats *internal.DataFileStatistics
+ assert.NotPanics(t, func() {
+ fileStats = format.DataFileStatsFromMeta(internal.Metadata(meta), collector, mapping, nil)
+ })
+ require.NotNil(t, fileStats)
+ require.NotContains(t, fileStats.ColAggs, 11)
+
+ dataFile := fileStats.ToDataFile(internal.DataFileOpts{
+ Schema: tblMeta.CurrentSchema(),
+ Spec: tblMeta.PartitionSpec(),
+ Path: "fake-path.parquet",
+ Format: iceberg.ParquetFile,
+ Content: iceberg.EntryContentData,
+ FileSize: meta.GetSourceFileSize(),
+ SortOrderID: 7,
+ })
+
+ assert.Len(t, dataFile.LowerBoundValues(), len(collector)-1)
+ assert.NotContains(t, dataFile.LowerBoundValues(), 11)
+ assert.NotContains(t, dataFile.UpperBoundValues(), 11)
+}
+
+func TestDataFileStatsFromMetaWithMalformedFixedLenDecimalStats(t *testing.T) {
+ format := internal.GetFileFormat(iceberg.ParquetFile)
+
+ meta, tblMeta := constructTestTablePrimitiveTypes(t)
+ require.NotNil(t, tblMeta)
+ require.NotNil(t, meta)
+
+ mapping, err := format.PathToIDMapping(tblMeta.CurrentSchema())
+ require.NoError(t, err)
+
+ collector := getCollector()
+ found := false
+ for _, rg := range meta.RowGroups {
+ for _, col := range rg.Columns {
+ if col == nil || col.MetaData == nil || col.MetaData.Statistics == nil {
+ continue
+ }
+ path := col.MetaData.GetPathInSchema()
+ if len(path) == 1 && path[0] == "large_dec" {
+ col.MetaData.Statistics.Min = []byte{0x01}
+ col.MetaData.Statistics.Max = []byte{0x02}
+ col.MetaData.Statistics.MinValue = []byte{0x01}
+ col.MetaData.Statistics.MaxValue = []byte{0x02}
+ found = true
+ }
+ }
+ }
+ require.True(t, found, "failed to inject malformed fixed-len decimal stats")
+
+ var fileStats *internal.DataFileStatistics
+ assert.NotPanics(t, func() {
+ fileStats = format.DataFileStatsFromMeta(internal.Metadata(meta), collector, mapping, nil)
+ })
+ require.NotNil(t, fileStats)
+ require.NotContains(t, fileStats.ColAggs, 15)
+
+ dataFile := fileStats.ToDataFile(internal.DataFileOpts{
+ Schema: tblMeta.CurrentSchema(),
+ Spec: tblMeta.PartitionSpec(),
+ Path: "fake-path.parquet",
+ Format: iceberg.ParquetFile,
+ Content: iceberg.EntryContentData,
+ FileSize: meta.GetSourceFileSize(),
+ SortOrderID: 7,
+ })
+
+ assert.Len(t, dataFile.LowerBoundValues(), len(collector)-1)
+ assert.NotContains(t, dataFile.LowerBoundValues(), 15)
+ assert.NotContains(t, dataFile.UpperBoundValues(), 15)
+}
+
// TestNanosecondTimestampMetrics tests that nanosecond timestamp types (v3)
// are correctly handled for Parquet stats collection and physical type mapping.
func TestNanosecondTimestampMetrics(t *testing.T) {