You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/07 23:27:23 UTC
(datafusion) branch gh-readonly-queue/main/pr-23306-1ca0073cc24d5224e65a0f033a007941d408b391 created (now 8272b4aa20)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23306-1ca0073cc24d5224e65a0f033a007941d408b391
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 8272b4aa20 fix: return execution error instead of capacity overflow panic in array_resize (#23306)
This branch includes the following new commits:
new 8272b4aa20 fix: return execution error instead of capacity overflow panic in array_resize (#23306)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: fix: return execution error instead of capacity overflow panic in array_resize (#23306)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23306-1ca0073cc24d5224e65a0f033a007941d408b391
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 8272b4aa20570f3a5a2fb8b8aa77c1b550fb6368
Author: Burak Şen <bu...@gmail.com>
AuthorDate: Wed Jul 8 02:27:06 2026 +0300
fix: return execution error instead of capacity overflow panic in array_resize (#23306)
## Which issue does this PR close?
- Closes #22227..
## Rationale for this change
Please check #22227
## What changes are included in this PR?
Instead of panicking we return execution error buy checking sizes in
array_resize.
Outcome is:
```
➜ datafusion git:(fix-array-resize-overflow-panic) cargo run -p datafusion-cli -- -c "SELECT array_resize(make_array(1), 9223372036854775807, 0)"
Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.54s
Running `target/debug/datafusion-cli -c 'SELECT array_resize(make_array(1), 9223372036854775807, 0)'`
DataFusion CLI v54.0.0
Error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size
```
in issue it was:
<img width="871" height="281" alt="image"
src="https://github.com/user-attachments/assets/6ebdda2c-5e74-4ca7-bc7d-51393a95fa24"
/>
## Are these changes tested?
yes. added new unit and slt tests
## Are there any user-facing changes?
instead of panic users will see execution error
---
datafusion/functions-nested/src/resize.rs | 97 +++++++++++++++++++++-
.../sqllogictest/test_files/array/array_resize.slt | 17 ++++
2 files changed, 111 insertions(+), 3 deletions(-)
diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rs
index d11064bf7e..e4fd8421fe 100644
--- a/datafusion/functions-nested/src/resize.rs
+++ b/datafusion/functions-nested/src/resize.rs
@@ -219,6 +219,14 @@ fn general_list_resize<O: OffsetSizeTrait + TryInto<i64>>(
}
}
+ if output_values_len > max_resize_values(&data_type)
+ || O::from_usize(output_values_len).is_none()
+ {
+ return exec_err!(
+ "array_resize: resulting array of {output_values_len} elements exceeds the maximum array size"
+ );
+ }
+
// The fast path is valid when at least one row grows and every row would
// use the same fill value.
let use_bulk_fill = max_extra > 0
@@ -342,12 +350,26 @@ where
)?))
}
+/// Largest element count whose eager value buffer stays within `isize::MAX`
+/// bytes, so `array_resize` rejects oversized results instead of panicking.
+/// Only primitive and `FixedSizeBinary` leaves are byte-exact.
+fn max_resize_values(value_type: &DataType) -> usize {
+ let element_width = match value_type {
+ DataType::FixedSizeBinary(size) if *size > 0 => *size as usize,
+ _ => value_type.primitive_width().unwrap_or(size_of::<u128>()),
+ };
+
+ (isize::MAX as usize) / element_width.max(1)
+}
+
#[cfg(test)]
mod tests {
use super::array_resize_inner;
- use arrow::array::{ArrayRef, AsArray, Int64Array, ListArray};
- use arrow::buffer::{NullBuffer, ScalarBuffer};
- use arrow::datatypes::Int32Type;
+ use arrow::array::{
+ ArrayRef, AsArray, FixedSizeBinaryArray, Int64Array, LargeListArray, ListArray,
+ };
+ use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
+ use arrow::datatypes::{DataType, Field, Int32Type, Int64Type};
use datafusion_common::Result;
use std::sync::Arc;
@@ -373,4 +395,73 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_array_resize_large_size_errors_without_panicking() {
+ let array: ArrayRef =
+ Arc::new(ListArray::from_iter_primitive::<Int64Type, _, _>(vec![
+ Some(vec![Some(1)]),
+ ]));
+ let size: ArrayRef = Arc::new(Int64Array::from(vec![i64::MAX]));
+ let fill: ArrayRef = Arc::new(Int64Array::from(vec![0]));
+
+ let err = array_resize_inner(&[array, size, fill]).unwrap_err();
+ assert!(
+ err.to_string().contains("exceeds the maximum array size"),
+ "unexpected error: {err}"
+ );
+ }
+
+ #[test]
+ fn test_array_resize_fixed_size_binary_large_size_errors_without_panicking() {
+ let values =
+ FixedSizeBinaryArray::try_from_iter(vec![vec![0u8; 32]].into_iter()).unwrap();
+ let elem_field =
+ Arc::new(Field::new_list_field(DataType::FixedSizeBinary(32), true));
+ let offsets = OffsetBuffer::<i64>::new(vec![0i64, 1].into());
+ let array: ArrayRef = Arc::new(LargeListArray::new(
+ elem_field,
+ offsets,
+ Arc::new(values) as ArrayRef,
+ None,
+ ));
+ // Passes the width-16 bound (isize::MAX / 16) but overflows at width 32.
+ let size: ArrayRef = Arc::new(Int64Array::from(vec![400_000_000_000_000_000i64]));
+
+ let err = array_resize_inner(&[array, size]).unwrap_err();
+ assert!(
+ err.to_string().contains("exceeds the maximum array size"),
+ "unexpected error: {err}"
+ );
+ }
+
+ #[test]
+ fn test_array_resize_accumulates_values_across_rows() {
+ // Each row's target (6e17) is individually under the width-8 cap
+ // (isize::MAX / 8), but their sum (1.2e18) exceeds it, so the guard
+ // must reject based on the accumulated total rather than per row.
+ let values = Int64Array::from(vec![1, 2]);
+ let offsets = OffsetBuffer::<i64>::new(vec![0i64, 1, 2].into());
+ let elem_field = Arc::new(Field::new_list_field(DataType::Int64, true));
+ let array: ArrayRef = Arc::new(LargeListArray::new(
+ elem_field,
+ offsets,
+ Arc::new(values) as ArrayRef,
+ None,
+ ));
+ let size: ArrayRef = Arc::new(Int64Array::from(vec![
+ 600_000_000_000_000_000i64,
+ 600_000_000_000_000_000i64,
+ ]));
+
+ let err = array_resize_inner(&[array, size]).unwrap_err();
+ assert!(
+ err.to_string().contains("1200000000000000000"),
+ "expected accumulated total in error: {err}"
+ );
+ assert!(
+ err.to_string().contains("exceeds the maximum array size"),
+ "unexpected error: {err}"
+ );
+ }
}
diff --git a/datafusion/sqllogictest/test_files/array/array_resize.slt b/datafusion/sqllogictest/test_files/array/array_resize.slt
index cb2ffa3a7e..37f8f2c693 100644
--- a/datafusion/sqllogictest/test_files/array/array_resize.slt
+++ b/datafusion/sqllogictest/test_files/array/array_resize.slt
@@ -64,6 +64,23 @@ select array_resize(arrow_cast(make_array(1, 2, 3), 'LargeList(Int64)'), 5, 4);
query error
select array_resize(make_array(1, 2, 3), -5, 2);
+# array_resize with a very large size should error instead of panicking (capacity overflow)
+query error DataFusion error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size
+select array_resize(make_array(1), 9223372036854775807, 0);
+
+query error DataFusion error: Execution error: array_resize: resulting array of 9223372036854775807 elements exceeds the maximum array size
+select array_resize(arrow_cast(make_array(1), 'LargeList(Int64)'), 9223372036854775807, 0);
+
+# List size above i32::MAX must error via the offset-type guard instead of
+# silently truncating the offsets or attempting a multi-GB allocation
+query error DataFusion error: Execution error: array_resize: resulting array of 3000000000 elements exceeds the maximum array size
+select array_resize(make_array(1), 3000000000, 0);
+
+# Non-primitive element types (e.g. Utf8) fall back to a conservative byte
+# width; a size above that cap must error instead of attempting a huge allocation.
+query error DataFusion error: Execution error: array_resize: resulting array of 600000000000000000 elements exceeds the maximum array size
+select array_resize(arrow_cast(make_array('a'), 'LargeList(Utf8)'), 600000000000000000);
+
# array_resize scalar function #5
query ?
select array_resize(make_array(1.1, 2.2, 3.3), 10, 9.9);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org