You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2026/08/31 12:40:09 UTC
(iceberg-python) branch main updated: fix: reject decimal promotion that changes the scale (#3613)
This is an automated email from the ASF dual-hosted git repository.
Fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 57a4c8e6e fix: reject decimal promotion that changes the scale (#3613)
57a4c8e6e is described below
commit 57a4c8e6e39a82337d0f056c1ceb800c813910c3
Author: Anas Khan <an...@gmail.com>
AuthorDate: Mon Aug 31 18:10:04 2026 +0530
fix: reject decimal promotion that changes the scale (#3613)
* fix: reject decimal promotion that changes the scale
The DecimalType handler in promote() guarded scale equality with
`file_type.scale == file_type.scale`, which compares the file scale to
itself and is always true. As a result any decimal-to-decimal promotion
with a widening precision was accepted regardless of the scale.
Per the Iceberg spec a decimal may only be promoted when the scale is
unchanged and the precision widens (decimal(P, S) to decimal(P2, S) with
P2 > P), matching TypeUtil.isPromotionAllowed in the Java implementation.
The bug affected both paths that use promote(): on read a differing-scale
promotion built a reader at the wrong scale and reinterpreted the stored
unscaled integers (silent data corruption), and on write it let a
DataFrame column with a different scale pass the compatibility check.
Compare the file scale to the read scale instead. The identical tautology
in the test oracle masked the defect, so fix it too and add a decimal with
a different scale to the promotion matrix, plus explicit regression tests
for the schema and read paths.
Signed-off-by: Anas Khan <83...@users.noreply.github.com>
* test: simplify decimal promotion coverage
Signed-off-by: Anas Khan <83...@users.noreply.github.com>
---------
Signed-off-by: Anas Khan <83...@users.noreply.github.com>
---
pyiceberg/schema.py | 2 +-
tests/avro/test_resolver.py | 9 +++++++++
tests/test_schema.py | 19 ++++++++++++++++++-
3 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/pyiceberg/schema.py b/pyiceberg/schema.py
index fd60eb8f9..a79b3ae95 100644
--- a/pyiceberg/schema.py
+++ b/pyiceberg/schema.py
@@ -1697,7 +1697,7 @@ def _(file_type: BinaryType, read_type: IcebergType) -> IcebergType:
@promote.register(DecimalType)
def _(file_type: DecimalType, read_type: IcebergType) -> IcebergType:
if isinstance(read_type, DecimalType):
- if file_type.precision <= read_type.precision and file_type.scale == file_type.scale:
+ if file_type.precision <= read_type.precision and file_type.scale == read_type.scale:
return read_type
else:
raise ResolveError(f"Cannot reduce precision from {file_type} to {read_type}")
diff --git a/tests/avro/test_resolver.py b/tests/avro/test_resolver.py
index f0742e946..fc47ffec0 100644
--- a/tests/avro/test_resolver.py
+++ b/tests/avro/test_resolver.py
@@ -256,6 +256,15 @@ def test_resolve_decimal_to_decimal_reduce_precision() -> None:
assert "Cannot reduce precision from decimal(19, 25) to decimal(10, 25)" in str(exc_info.value)
+def test_resolve_decimal_to_decimal_change_scale() -> None:
+ # Changing the scale is not a valid promotion, even when the precision widens.
+ # Allowing it would reinterpret the file's unscaled integers at the wrong scale.
+ with pytest.raises(ResolveError) as exc_info:
+ _ = resolve_reader(DecimalType(9, 2), DecimalType(18, 4))
+
+ assert "Cannot reduce precision from decimal(9, 2) to decimal(18, 4)" in str(exc_info.value)
+
+
def test_column_assignment() -> None:
int_schema = {
"type": "record",
diff --git a/tests/test_schema.py b/tests/test_schema.py
index aacdbc7d5..872e95ce3 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -868,6 +868,10 @@ def test_schema_select_cant_be_found(table_schema_nested: Schema) -> None:
assert "Could not find column: 'BAZ'" in str(exc_info.value)
+def can_promote_decimal(file_type: DecimalType, read_type: DecimalType) -> bool:
+ return file_type.precision <= read_type.precision and file_type.scale == read_type.scale
+
+
def should_promote(file_type: IcebergType, read_type: IcebergType) -> bool:
if isinstance(file_type, IntegerType) and isinstance(read_type, LongType):
return True
@@ -878,7 +882,7 @@ def should_promote(file_type: IcebergType, read_type: IcebergType) -> bool:
if isinstance(file_type, BinaryType) and isinstance(read_type, StringType):
return True
if isinstance(file_type, DecimalType) and isinstance(read_type, DecimalType):
- return file_type.precision <= read_type.precision and file_type.scale == file_type.scale
+ return can_promote_decimal(file_type, read_type)
if isinstance(file_type, FixedType) and isinstance(read_type, UUIDType) and len(file_type) == 16:
return True
return False
@@ -945,6 +949,19 @@ def test_promotion(file_type: IcebergType, read_type: IcebergType) -> None:
promote(file_type, read_type)
+def test_decimal_promotion() -> None:
+ # Widening precision while keeping the scale fixed is allowed.
+ assert promote(DecimalType(9, 2), DecimalType(18, 2)) == DecimalType(18, 2)
+ # Equal precision and scale resolves to the read type.
+ assert promote(DecimalType(9, 2), DecimalType(9, 2)) == DecimalType(9, 2)
+ # Changing the scale is never allowed, even when the precision widens.
+ with pytest.raises(ResolveError):
+ promote(DecimalType(9, 2), DecimalType(18, 4))
+ # Reducing the precision is not allowed.
+ with pytest.raises(ResolveError):
+ promote(DecimalType(18, 2), DecimalType(9, 2))
+
+
def test_unknown_type_promotion_to_primitive() -> None:
"""Test that UnknownType can be promoted to primitive types (V3+ behavior)"""
unknown_type = UnknownType()