(iceberg-rust) branch main updated: fix: Add support for unsigned Arrow datatypes in schema conversion (#1617)

[xu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git


The following commit(s) were added to refs/heads/main by this push:
     new 8bc44a766 fix: Add support for unsigned Arrow datatypes in schema conversion (#1617)
8bc44a766 is described below

commit 8bc44a766949ef568061c12b444d752c83665dec
Author: Geeta Krishna Panda <ge...@amazon.com>
AuthorDate: Wed Sep 3 00:43:08 2025 -0700

    fix: Add support for unsigned Arrow datatypes in schema conversion (#1617)
    
    ## Which issue does this PR close?
    
    <!--
    We generally require a GitHub issue to be filed for all bug fixes and
    enhancements and this helps us generate change logs for our releases.
    You can link an issue to this PR using the GitHub syntax. For example
    `Closes #123` indicates that this PR will close issue #123.
    -->
    
    - Closes #675
    
    ## What changes are included in this PR?
    
    <!--
    Provide a summary of the modifications in this PR. List the main changes
    such as new features, bug fixes, refactoring, or any other updates.
    -->
    ### Bug Fixes
    - Fixed crash when ArrowSchemaConverter encounters unsigned datatypes
    - Resolved "Unsupported Arrow data type" errors for UInt8/16/32/64
    
    ### Features
    - Added casting support for unsigned Arrow types
    - UInt8/16 → Int32 (safe casting to larger signed type)
    - UInt32 → Int64 (safe casting to larger signed type)
    - UInt64 → Error (no safe casting option, explicit error with guidance)
    
    ### Code Changes
    - Enhanced ArrowSchemaConverter primitive() method with unsigned type
    handling
    - Added comprehensive test: test_unsigned_type_casting() for all
    unsigned variants
    
    ### Files Modified
    - `crates/iceberg/src/arrow/schema.rs`
    
    ## Impact
    ✅ No breaking changes - existing functionality preserved
    ✅ Safe type casting prevents overflow issues
    ✅ Clear error messages for unsupported UInt64 with alternatives
    ✅ Follows proven PyIceberg implementation approach
    
    ## Are these changes tested?
    - All existing schema tests pass
    - New comprehensive test covers UInt8, UInt16, UInt32, UInt64 conversion
    behavior
    - Test verifies proper casting: UInt8/16→Int32, UInt32→Int64,
    UInt64→Error
    <!--
    Specify what test covers (unit test, integration test, etc.).
    
    If tests are not included in your PR, please explain why (for example,
    are they covered by existing tests)?
    -->
---
 crates/iceberg/src/arrow/schema.rs | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/crates/iceberg/src/arrow/schema.rs b/crates/iceberg/src/arrow/schema.rs
index 07058657b..c214821dd 100644
--- a/crates/iceberg/src/arrow/schema.rs
+++ b/crates/iceberg/src/arrow/schema.rs
@@ -378,7 +378,16 @@ impl ArrowSchemaVisitor for ArrowSchemaConverter {
             DataType::Int8 | DataType::Int16 | DataType::Int32 => {
                 Ok(Type::Primitive(PrimitiveType::Int))
             }
+            DataType::UInt8 | DataType::UInt16 => Ok(Type::Primitive(PrimitiveType::Int)),
+            DataType::UInt32 => Ok(Type::Primitive(PrimitiveType::Long)),
             DataType::Int64 => Ok(Type::Primitive(PrimitiveType::Long)),
+            DataType::UInt64 => {
+                // Block uint64 - no safe casting option
+                Err(Error::new(
+                    ErrorKind::DataInvalid,
+                    "UInt64 is not supported. Use Int64 for values ≤ 9,223,372,036,854,775,807 or Decimal(20,0) for full uint64 range.",
+                ))
+            }
             DataType::Float32 => Ok(Type::Primitive(PrimitiveType::Float)),
             DataType::Float64 => Ok(Type::Primitive(PrimitiveType::Double)),
             DataType::Decimal128(p, s) => Type::decimal(*p as u32, *s as u32).map_err(|e| {
@@ -1717,6 +1726,49 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_unsigned_integer_type_conversion() {
+        let test_cases = vec![
+            (DataType::UInt8, PrimitiveType::Int),
+            (DataType::UInt16, PrimitiveType::Int),
+            (DataType::UInt32, PrimitiveType::Long),
+        ];
+
+        for (arrow_type, expected_iceberg_type) in test_cases {
+            let arrow_field = Field::new("test", arrow_type.clone(), false).with_metadata(
+                HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
+            );
+            let arrow_schema = ArrowSchema::new(vec![arrow_field]);
+
+            let iceberg_schema = arrow_schema_to_schema(&arrow_schema).unwrap();
+            let iceberg_field = iceberg_schema.as_struct().fields().first().unwrap();
+
+            assert!(
+                matches!(iceberg_field.field_type.as_ref(), Type::Primitive(t) if *t == expected_iceberg_type),
+                "Expected {:?} to map to {:?}",
+                arrow_type,
+                expected_iceberg_type
+            );
+        }
+
+        // Test UInt64 blocking
+        {
+            let arrow_field = Field::new("test", DataType::UInt64, false).with_metadata(
+                HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())]),
+            );
+            let arrow_schema = ArrowSchema::new(vec![arrow_field]);
+
+            let result = arrow_schema_to_schema(&arrow_schema);
+            assert!(result.is_err());
+            assert!(
+                result
+                    .unwrap_err()
+                    .to_string()
+                    .contains("UInt64 is not supported")
+            );
+        }
+    }
+
     #[test]
     fn test_datum_conversion() {
         {