(datafusion) 01/01: fix: reject LargeList offsets that exceed i32 (#25516)

[gi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25516-925d7f8ffdcb5584e26f2410facbcfc5b4234d20
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit 38a1865b7e18889f4645c5b081a44b414af3545c
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Sun Sep 20 00:46:12 2026 +0000

    fix: reject LargeList offsets that exceed i32 (#25516)
    
    ## Which issue does this PR close?
    
    N/A
    
    ## Rationale for this change
    
    Spark map functions accept `LargeList` inputs whose offsets are
    represented as
    `i64`. The map construction path converts these offsets to `i32`, but
    the
    existing unchecked cast silently wraps oversized values.
    
    For example, an offset of `2147483648` becomes `-2147483648`. Downstream
    code
    can then interpret the wrapped value as an invalid child-array index
    instead of
    reporting the unsupported input.
    
    ## What changes are included in this PR?
    
    - Replace the unchecked `i64` to `i32` conversion with a checked
    conversion.
    - Return an actionable execution error when an offset cannot be
    represented as
      `i32`.
    - Add a regression test using a sliced `LargeList` whose first offset
    exceeds
      `i32::MAX`.
    
    ## What is the testing strategy for this PR?
    
    Added `large_list_offsets_do_not_truncate`.
    
    Ablation testing confirmed that the test fails with the old
    implementation,
    which returns the truncated offsets `[-2147483648, -2147483647]`, and
    passes
    with the checked conversion.
    
    Also verified:
    
    - `cargo fmt --all -- --check`
    - `git diff --check`
    
    The full test suite is covered by CI.
    
    ## Are there any user-facing changes?
    
    Yes. Oversized `LargeList` offsets now produce a clear execution error
    instead
    of being silently truncated. There are no API changes.
---
 datafusion/spark/src/function/map/utils.rs | 32 ++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/datafusion/spark/src/function/map/utils.rs b/datafusion/spark/src/function/map/utils.rs
index cd8fa9ccdd..6f6fec5b3b 100644
--- a/datafusion/spark/src/function/map/utils.rs
+++ b/datafusion/spark/src/function/map/utils.rs
@@ -25,7 +25,7 @@ use arrow::array::{
 use arrow::buffer::{NullBuffer, OffsetBuffer};
 use arrow::compute::{filter, take};
 use arrow::datatypes::{DataType, Field, Fields};
-use datafusion_common::{Result, ScalarValue, exec_err};
+use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err};
 
 /// Helper function to get element [`DataType`]
 /// from [`List`](DataType::List)/[`LargeList`](DataType::LargeList)/[`FixedSizeList`](DataType::FixedSizeList)<br>
@@ -70,8 +70,14 @@ pub fn get_list_offsets(array: &ArrayRef) -> Result<Cow<'_, [i32]>> {
                 .as_list::<i64>()
                 .offsets()
                 .iter()
-                .map(|i| *i as i32)
-                .collect::<Vec<_>>(),
+                .map(|offset| {
+                    i32::try_from(*offset).map_err(|_| {
+                        exec_datafusion_err!(
+                            "get_list_offsets: LargeList offset {offset} cannot be represented as i32"
+                        )
+                    })
+                })
+                .collect::<Result<Vec<_>>>()?,
         )),
         DataType::FixedSizeList(_, size) => Ok(Cow::Owned(
             (0..=array.len() as i32).map(|i| size * i).collect(),
@@ -259,7 +265,7 @@ fn map_deduplicate_keys(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arrow::array::{Int32Array, StringArray};
+    use arrow::array::{Int32Array, LargeListArray, NullArray, StringArray};
     use arrow::datatypes::Int32Type;
 
     fn int32_utf8_inputs(
@@ -271,6 +277,24 @@ mod tests {
         (keys, values)
     }
 
+    #[test]
+    fn large_list_offsets_do_not_truncate() {
+        let start = i64::from(i32::MAX) + 1;
+        let values: ArrayRef = Arc::new(NullArray::new((start + 1) as usize));
+        let array: ArrayRef = Arc::new(
+            LargeListArray::new(
+                Arc::new(Field::new_list_field(DataType::Null, true)),
+                OffsetBuffer::new(vec![0, start, start + 1].into()),
+                values,
+                None,
+            )
+            .slice(1, 1),
+        );
+
+        let err = get_list_offsets(&array).unwrap_err().to_string();
+        assert!(err.contains("cannot be represented as i32"), "{err}");
+    }
+
     #[test]
     fn happy_path_two_rows_no_duplicates() {
         let (keys, values) =


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org