You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by at...@apache.org on 2026/06/15 08:22:56 UTC

(iceberg-rust) branch main updated: fix: support decimal datum conversion by precision (#2508)

This is an automated email from the ASF dual-hosted git repository.

laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git


The following commit(s) were added to refs/heads/main by this push:
     new b10a8d8f4 fix: support decimal datum conversion by precision (#2508)
b10a8d8f4 is described below

commit b10a8d8f4bb7091802580679fe61a48d5226ff03
Author: Abanoub Doss <ab...@gmail.com>
AuthorDate: Mon Jun 15 03:22:50 2026 -0500

    fix: support decimal datum conversion by precision (#2508)
    
    ## Which issue does this PR close?
    
    <!--
    We generally require a GitHub issue to be filed for all bug fixes and
    enhancements and this helps us generate change logs for our releases.
    You can link an issue to this PR using the GitHub syntax. For example
    `Closes #123` indicates that this PR will close issue #123.
    -->
    
    - Closes #.
    
    ## What changes are included in this PR?
    
    <!--
    Provide a summary of the modifications in this PR. List the main changes
    such as new features, bug fixes, refactoring, or any other updates.
    -->
    
    This fixes decimal datum precision handling in two places.
    
    First, `Datum::to()` can now convert between decimal types with the same
    scale and different precision, as long as the value fits the target
    precision. This is needed when a decimal literal is created before the
    target column precision is known.
    
    Second, this fixes a correctness issue in
    `Datum::decimal_with_precision`. It previously checked whether the
    mantissa fit the storage byte width for the requested precision, but
    that could accept values that exceeded the
    declared decimal precision. It now checks the mantissa’s decimal digit
    count directly. For example, `0.42` has mantissa `42`, so it does not
    fit `decimal(1, 2)`.
    
    Scale conversion remains out of scope. Decimal conversions with
    different source and target scales still return `DataInvalid`.
    
    ## Are these changes tested?
    
    <!--
    Specify what test covers (unit test, integration test, etc.).
    
    If tests are not included in your PR, please explain why (for example,
    are they covered by existing tests)?
    -->
    
    Yes. This adds unit coverage for decimal precision calculation and
    decimal datum conversion.
    
    Validation run:
    
    ```bash
    cargo fmt --check
    cargo test -p iceberg test_decimal_precision
    cargo test -p iceberg test_datum_to_decimal_
    cargo test -p iceberg test_datum_decimal_with_precision_
    cargo clippy -p iceberg --tests -- -D warnings
    ```
    
    ##### Was generative AI tooling used to co-author this PR?
    
    Yes.
    
    ---------
    
    Co-authored-by: Abanoub Doss <ab...@gmail.com>
---
 crates/iceberg/src/spec/values/datum.rs         |  75 ++++++++----
 crates/iceberg/src/spec/values/decimal_utils.rs |  37 ++++++
 crates/iceberg/src/spec/values/tests.rs         | 155 +++++++++++++++++++++++-
 3 files changed, 244 insertions(+), 23 deletions(-)

diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs
index 68ea6b3d4..2040d4634 100644
--- a/crates/iceberg/src/spec/values/datum.rs
+++ b/crates/iceberg/src/spec/values/datum.rs
@@ -29,8 +29,8 @@ use serde::{Deserialize, Serialize};
 use serde_bytes::ByteBuf;
 
 use super::decimal_utils::{
-    Decimal, decimal_from_i128_with_scale, decimal_from_str_exact, decimal_mantissa, decimal_scale,
-    i128_from_be_bytes, i128_to_be_bytes_min,
+    Decimal, decimal_from_i128_with_scale, decimal_from_str_exact, decimal_mantissa,
+    decimal_precision, decimal_scale, i128_from_be_bytes, i128_to_be_bytes_min,
 };
 use super::literal::Literal;
 use super::primitive::PrimitiveLiteral;
@@ -1053,24 +1053,7 @@ impl Datum {
         let scale = decimal_scale(&value);
         let mantissa = decimal_mantissa(&value);
 
-        let available_bytes = Type::decimal_required_bytes(precision)? as usize;
-        let actual_bytes = i128_to_be_bytes_min(mantissa);
-        if actual_bytes.len() > available_bytes {
-            return Err(Error::new(
-                ErrorKind::DataInvalid,
-                format!("Decimal value {value} is too large for precision {precision}"),
-            ));
-        }
-
-        let r#type = Type::decimal(precision, scale)?;
-        if let Type::Primitive(p) = r#type {
-            Ok(Self {
-                r#type: p,
-                literal: PrimitiveLiteral::Int128(mantissa),
-            })
-        } else {
-            unreachable!("Decimal type must be primitive.")
-        }
+        Self::decimal_from_mantissa(mantissa, precision, scale)
     }
 
     fn i64_to_i32<T: Into<i64> + PartialOrd<i64>>(val: T) -> Datum {
@@ -1109,6 +1092,26 @@ impl Datum {
         })
     }
 
+    fn decimal_from_mantissa(mantissa: i128, precision: u32, scale: u32) -> Result<Self> {
+        let r#type = Type::decimal(precision, scale)?;
+        if decimal_precision(mantissa) > precision {
+            let value = decimal_from_i128_with_scale(mantissa, scale);
+            return Err(Error::new(
+                ErrorKind::DataInvalid,
+                format!("Decimal value {value} is too large for precision {precision}"),
+            ));
+        }
+
+        if let Type::Primitive(p) = r#type {
+            Ok(Self {
+                r#type: p,
+                literal: PrimitiveLiteral::Int128(mantissa),
+            })
+        } else {
+            unreachable!("Decimal type must be primitive.")
+        }
+    }
+
     /// Convert the datum to `target_type`.
     pub fn to(self, target_type: &Type) -> Result<Datum> {
         match target_type {
@@ -1130,7 +1133,37 @@ impl Datum {
                     (PrimitiveLiteral::Int128(val), _, PrimitiveType::Long) => {
                         Ok(Datum::i128_to_i64(*val))
                     }
-
+                    (
+                        PrimitiveLiteral::Int128(val),
+                        PrimitiveType::Decimal {
+                            scale: self_scale, ..
+                        },
+                        PrimitiveType::Decimal {
+                            precision,
+                            scale: target_scale,
+                        },
+                    ) if self_scale == target_scale => {
+                        Datum::decimal_from_mantissa(*val, *precision, *target_scale)
+                    }
+                    // Java's DecimalLiteral.to() is a no-op for decimal targets, even when scales
+                    // differ. We intentionally do not mirror that here: the same mantissa with a
+                    // different scale represents a different value, so scale conversion needs
+                    // explicit rescaling.
+                    (
+                        PrimitiveLiteral::Int128(_),
+                        PrimitiveType::Decimal {
+                            scale: self_scale, ..
+                        },
+                        PrimitiveType::Decimal {
+                            scale: target_scale,
+                            ..
+                        },
+                    ) => Err(Error::new(
+                        ErrorKind::DataInvalid,
+                        format!(
+                            "Decimal scale conversion is not supported: source scale {self_scale}, target scale {target_scale}"
+                        ),
+                    )),
                     (PrimitiveLiteral::String(val), _, PrimitiveType::Boolean) => {
                         Datum::bool_from_str(val)
                     }
diff --git a/crates/iceberg/src/spec/values/decimal_utils.rs b/crates/iceberg/src/spec/values/decimal_utils.rs
index 88e3f72f6..97225113c 100644
--- a/crates/iceberg/src/spec/values/decimal_utils.rs
+++ b/crates/iceberg/src/spec/values/decimal_utils.rs
@@ -120,6 +120,16 @@ pub fn decimal_mantissa(d: &Decimal) -> i128 {
     }
 }
 
+/// Get the decimal digit precision of a mantissa.
+///
+/// A mantissa of 0 has a precision of 1.
+pub fn decimal_precision(mantissa: i128) -> u32 {
+    mantissa
+        .unsigned_abs()
+        .checked_ilog10()
+        .map_or(1, |x| x + 1)
+}
+
 /// Get the scale (number of digits after decimal point).
 ///
 /// This is equivalent to rust_decimal's `decimal.scale()`.
@@ -232,6 +242,33 @@ mod tests {
         assert_eq!(decimal_mantissa(&d), -12345);
     }
 
+    #[test]
+    fn test_decimal_precision() {
+        assert_eq!(decimal_precision(0), 1);
+        assert_eq!(decimal_precision(5), 1);
+        assert_eq!(decimal_precision(42), 2);
+        assert_eq!(decimal_precision(-42), 2);
+        // power-of-10 boundaries, where off-by-one bugs in log10 logic live
+        assert_eq!(decimal_precision(9), 1);
+        assert_eq!(decimal_precision(10), 2);
+        assert_eq!(decimal_precision(99), 2);
+        assert_eq!(decimal_precision(100), 3);
+
+        // max Iceberg decimal precision
+        assert_eq!(
+            decimal_precision(99999999999999999999999999999999999999),
+            38
+        );
+        assert_eq!(
+            decimal_precision(-99999999999999999999999999999999999999),
+            38
+        );
+
+        // i128 extremes
+        assert_eq!(decimal_precision(i128::MAX), 39);
+        assert_eq!(decimal_precision(i128::MIN), 39);
+    }
+
     #[test]
     fn test_decimal_scale() {
         let d = decimal_from_i128_with_scale(12345, 2);
diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs
index d3e0a455c..bc8b011fd 100644
--- a/crates/iceberg/src/spec/values/tests.rs
+++ b/crates/iceberg/src/spec/values/tests.rs
@@ -460,8 +460,8 @@ fn avro_bytes_decimal() {
         (vec![251u8, 46u8], -1234, 2, 38),
         (vec![4u8, 210u8], 1234, 3, 38),
         (vec![251u8, 46u8], -1234, 3, 38),
-        (vec![42u8], 42, 2, 1),
-        (vec![214u8], -42, 2, 1),
+        (vec![42u8], 42, 2, 2),
+        (vec![214u8], -42, 2, 2),
     ];
 
     for (input_bytes, decimal_num, expect_scale, expect_precision) in cases {
@@ -1424,3 +1424,154 @@ fn test_date_from_json_as_number() {
 
     // Both formats should produce the same Literal value
 }
+
+#[test]
+fn test_datum_to_decimal_narrows_precision_when_scale_matches() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 9,
+        scale: 2,
+    });
+    let datum = Datum::decimal_from_str("123.45").unwrap();
+
+    let converted = datum.to(&target_type).unwrap();
+
+    assert_eq!(converted.data_type(), &PrimitiveType::Decimal {
+        precision: 9,
+        scale: 2,
+    });
+    assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(12345));
+}
+
+#[test]
+fn test_datum_to_decimal_widens_precision_when_scale_matches() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 38,
+        scale: 2,
+    });
+    let datum = Datum::decimal_with_precision(decimal_from_i128_with_scale(12345, 2), 9).unwrap();
+
+    let converted = datum.to(&target_type).unwrap();
+
+    assert_eq!(converted.data_type(), &PrimitiveType::Decimal {
+        precision: 38,
+        scale: 2,
+    });
+    assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(12345));
+}
+
+#[test]
+fn test_datum_to_decimal_accepts_zero_mantissa() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 1,
+        scale: 0,
+    });
+    let datum = Datum::decimal_with_precision(decimal_from_i128_with_scale(0, 0), 9).unwrap();
+
+    let converted = datum.to(&target_type).unwrap();
+
+    assert_eq!(converted.data_type(), &PrimitiveType::Decimal {
+        precision: 1,
+        scale: 0,
+    });
+    assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(0));
+}
+
+#[test]
+fn test_datum_to_decimal_accepts_negative_mantissa() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 2,
+        scale: 1,
+    });
+    let datum = Datum::decimal_from_str("-1.5").unwrap();
+
+    let converted = datum.to(&target_type).unwrap();
+
+    assert_eq!(converted.data_type(), &PrimitiveType::Decimal {
+        precision: 2,
+        scale: 1,
+    });
+    assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(-15));
+}
+
+#[test]
+fn test_datum_to_decimal_rejects_precision_too_narrow() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 1,
+        scale: 1,
+    });
+    let datum = Datum::decimal_from_str("1.5").unwrap();
+
+    let result = datum.to(&target_type);
+
+    assert!(result.is_err(), "expect error but got {result:?}");
+    assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid);
+}
+
+#[test]
+fn test_datum_to_decimal_rejects_value_that_fits_storage_bytes_but_not_precision() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 1,
+        scale: 1,
+    });
+    let datum = Datum::decimal_from_str("4.2").unwrap();
+
+    let result = datum.to(&target_type);
+
+    assert!(result.is_err(), "expect error but got {result:?}");
+    assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid);
+}
+
+#[test]
+fn test_datum_to_decimal_accepts_single_digit_mantissa_for_precision_one() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 1,
+        scale: 1,
+    });
+    let datum = Datum::decimal_from_str("0.5").unwrap();
+
+    let converted = datum.to(&target_type).unwrap();
+
+    assert_eq!(converted.data_type(), &PrimitiveType::Decimal {
+        precision: 1,
+        scale: 1,
+    });
+    assert_eq!(converted.literal(), &PrimitiveLiteral::Int128(5));
+}
+
+#[test]
+fn test_datum_decimal_with_precision_rejects_value_that_exceeds_digit_precision() {
+    let result = Datum::decimal_with_precision(decimal_from_i128_with_scale(42, 2), 1);
+
+    assert!(result.is_err(), "expect error but got {result:?}");
+    assert_eq!(result.unwrap_err().kind(), ErrorKind::DataInvalid);
+}
+
+#[test]
+fn test_datum_decimal_with_precision_accepts_value_that_fits_digit_precision() {
+    let datum = Datum::decimal_with_precision(decimal_from_i128_with_scale(5, 1), 1).unwrap();
+
+    assert_eq!(datum.data_type(), &PrimitiveType::Decimal {
+        precision: 1,
+        scale: 1,
+    });
+    assert_eq!(datum.literal(), &PrimitiveLiteral::Int128(5));
+}
+
+#[test]
+fn test_datum_to_decimal_rejects_scale_change() {
+    let target_type = Type::Primitive(PrimitiveType::Decimal {
+        precision: 9,
+        scale: 3,
+    });
+    let datum = Datum::decimal_from_str("123.45").unwrap();
+
+    let result = datum.to(&target_type);
+    assert!(result.is_err(), "expect error but got {result:?}");
+
+    let err = result.unwrap_err();
+    assert_eq!(err.kind(), ErrorKind::DataInvalid);
+    assert!(
+        err.to_string()
+            .contains("Decimal scale conversion is not supported")
+    );
+}