You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/24 15:36:50 UTC
(datafusion) 01/01: fix array_repeat capacity overflow on constant scalar with large count (#22305)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22305-94c58d086d1fc7f4ea6fb0a5f6d82805ef2b7a98
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 7ad8e2ce501e58f87bdd039cd252c2441bc22d6b
Author: Zhen Chen <cz...@163.com>
AuthorDate: Sun May 24 23:36:30 2026 +0800
fix array_repeat capacity overflow on constant scalar with large count (#22305)
## Which issue does this PR close?
- Closes #22228 .
## Rationale for this change
`array_repeat` still panics for oversized repeat counts in the
constant-scalar path. The simplest reproducer is:
```sql
SELECT array_repeat(1, 9223372036854775807)
```
Unlike the previously reported `array_repeat` overflow cases, this path
does not sum counts across rows and does not multiply nested list
lengths, but it still reaches an unchecked `Vec` preallocation and
panics with `capacity overflow`.
This change makes `array_repeat` reject oversized output lengths up
front and return a normal execution error instead of panicking.
## What changes are included in this PR?
This PR adds explicit bounds checks in repeat.rs so `array_repeat`
validates requested output sizes before allocating buffers.
The main changes are:
- Move repeat-length accumulation into shared checked helpers.
- Reject oversized output lengths with:
`array_repeat: requested length exceeds maximum array size`
- Guard both scalar and list repeat paths so they fail consistently
before hitting unchecked allocation or arithmetic overflow.
- Reuse precomputed outer offsets for the list path instead of
rebuilding them from unchecked lengths.
## Are these changes tested?
Yes.
This PR adds a regression test in repeat.rs covering the constant-scalar
reproducer with `i64::MAX` as the repeat count and verifies that
`array_repeat` returns an execution error rather than panicking.
Validated with:
```bash
cargo test -p datafusion-functions-nested scalar_count_exceeding_max_array_size_returns_error --lib
```
## Are there any user-facing changes?
Yes.
Previously, oversized `array_repeat` calls could panic the process.
After this change, they return a regular execution error:
```text
array_repeat: requested length exceeds maximum array size
```
---
datafusion/functions-nested/src/repeat.rs | 143 ++++++++++++++++-----
.../sqllogictest/test_files/array/array_repeat.slt | 3 +
2 files changed, 111 insertions(+), 35 deletions(-)
diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs
index ceec748a6e..825530923f 100644
--- a/datafusion/functions-nested/src/repeat.rs
+++ b/datafusion/functions-nested/src/repeat.rs
@@ -38,8 +38,12 @@ use datafusion_expr::{
};
use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
use datafusion_macros::user_doc;
+use std::mem::size_of;
use std::sync::Arc;
+const ARRAY_REPEAT_LENGTH_EXCEEDED: &str =
+ "array_repeat: requested length exceeds maximum array size";
+
make_udf_expr_and_func!(
ArrayRepeat,
array_repeat,
@@ -175,28 +179,12 @@ fn general_repeat<O: OffsetSizeTrait>(
array: &ArrayRef,
count_array: &Int64Array,
) -> Result<ArrayRef> {
- let total_repeated_values: usize = (0..count_array.len())
- .map(|i| get_count_with_validity(count_array, i))
- .sum();
+ let (offsets, total_repeated_values) = build_repeat_offsets::<O>(count_array)?;
let mut take_indices = Vec::with_capacity(total_repeated_values);
- let mut offsets = Vec::with_capacity(count_array.len() + 1);
- offsets.push(O::zero());
- let mut running_offset = 0usize;
for idx in 0..count_array.len() {
let count = get_count_with_validity(count_array, idx);
- running_offset = running_offset.checked_add(count).ok_or_else(|| {
- DataFusionError::Execution(
- "array_repeat: running_offset overflowed usize".to_string(),
- )
- })?;
- let offset = O::from_usize(running_offset).ok_or_else(|| {
- DataFusionError::Execution(format!(
- "array_repeat: offset {running_offset} exceeds the maximum value for offset type"
- ))
- })?;
- offsets.push(offset);
take_indices.extend(std::iter::repeat_n(idx as u64, count));
}
@@ -231,23 +219,23 @@ fn general_list_repeat<O: OffsetSizeTrait>(
count_array: &Int64Array,
) -> Result<ArrayRef> {
let list_offsets = list_array.value_offsets();
+ let (outer_offsets, outer_total) = build_repeat_offsets::<O>(count_array)?;
// calculate capacities for pre-allocation
- let mut outer_total = 0usize;
let mut inner_total = 0usize;
for i in 0..count_array.len() {
let count = get_count_with_validity(count_array, i);
- if count > 0 {
- outer_total += count;
- if list_array.is_valid(i) {
- let len = list_offsets[i + 1].to_usize().unwrap()
- - list_offsets[i].to_usize().unwrap();
- inner_total += len * count;
- }
+ if count > 0 && list_array.is_valid(i) {
+ let len = list_offsets[i + 1].to_usize().unwrap()
+ - list_offsets[i].to_usize().unwrap();
+ inner_total =
+ checked_repeat_len_add(inner_total, checked_repeat_len_mul(len, count)?)?;
+ ensure_array_repeat_output_len::<O>(inner_total)?;
}
}
// Build inner structures
+ ensure_vec_capacity::<O>(checked_repeat_len_add(outer_total, 1)?)?;
let mut inner_offsets = Vec::with_capacity(outer_total + 1);
let mut take_indices = Vec::with_capacity(inner_total);
let mut inner_nulls = BooleanBufferBuilder::new(outer_total);
@@ -262,11 +250,8 @@ fn general_list_repeat<O: OffsetSizeTrait>(
let row_len = end - start;
for _ in 0..count {
- inner_running = inner_running.checked_add(row_len).ok_or_else(|| {
- DataFusionError::Execution(
- "array_repeat: inner offset overflowed usize".to_string(),
- )
- })?;
+ inner_running = checked_repeat_len_add(inner_running, row_len)?;
+ ensure_array_repeat_output_len::<O>(inner_running)?;
let offset = O::from_usize(inner_running).ok_or_else(|| {
DataFusionError::Execution(format!(
"array_repeat: offset {inner_running} exceeds the maximum value for offset type"
@@ -299,16 +284,85 @@ fn general_list_repeat<O: OffsetSizeTrait>(
list_array.data_type().to_owned(),
true,
)),
- OffsetBuffer::<O>::from_lengths(
- count_array
- .iter()
- .map(|c| c.map(|v| if v > 0 { v as usize } else { 0 }).unwrap_or(0)),
- ),
+ OffsetBuffer::new(outer_offsets.into()),
Arc::new(inner_list),
count_array.nulls().cloned(),
)?))
}
+fn build_repeat_offsets<O: OffsetSizeTrait>(
+ count_array: &Int64Array,
+) -> Result<(Vec<O>, usize)> {
+ let mut offsets = Vec::with_capacity(count_array.len() + 1);
+ offsets.push(O::zero());
+ let mut running_offset = 0usize;
+
+ for idx in 0..count_array.len() {
+ let count = get_count_with_validity(count_array, idx);
+ running_offset = checked_repeat_len_add(running_offset, count)?;
+ ensure_array_repeat_output_len::<O>(running_offset)?;
+ let offset = O::from_usize(running_offset).ok_or_else(|| {
+ DataFusionError::Execution(format!(
+ "array_repeat: offset {running_offset} exceeds the maximum value for offset type"
+ ))
+ })?;
+ offsets.push(offset);
+ }
+
+ Ok((offsets, running_offset))
+}
+
+fn checked_repeat_len_add(lhs: usize, rhs: usize) -> Result<usize> {
+ lhs.checked_add(rhs).ok_or_else(|| {
+ DataFusionError::Execution(ARRAY_REPEAT_LENGTH_EXCEEDED.to_string())
+ })
+}
+
+fn checked_repeat_len_mul(lhs: usize, rhs: usize) -> Result<usize> {
+ lhs.checked_mul(rhs).ok_or_else(|| {
+ DataFusionError::Execution(ARRAY_REPEAT_LENGTH_EXCEEDED.to_string())
+ })
+}
+
+fn ensure_array_repeat_output_len<O: OffsetSizeTrait>(len: usize) -> Result<()> {
+ if len > max_array_repeat_output_len::<O>() {
+ return Err(DataFusionError::Execution(
+ ARRAY_REPEAT_LENGTH_EXCEEDED.to_string(),
+ ));
+ }
+
+ Ok(())
+}
+
+fn ensure_vec_capacity<T>(len: usize) -> Result<()> {
+ if len > max_vec_elements::<T>() {
+ return Err(DataFusionError::Execution(
+ ARRAY_REPEAT_LENGTH_EXCEEDED.to_string(),
+ ));
+ }
+
+ Ok(())
+}
+
+fn max_array_repeat_output_len<O: OffsetSizeTrait>() -> usize {
+ max_offset_elements::<O>().min(max_vec_elements::<u64>())
+}
+
+fn max_offset_elements<O: OffsetSizeTrait>() -> usize {
+ if size_of::<O>() == size_of::<i32>() {
+ i32::MAX as usize
+ } else {
+ i64::MAX as usize
+ }
+}
+
+fn max_vec_elements<T>() -> usize {
+ let element_size = size_of::<T>();
+ (isize::MAX as usize)
+ .checked_div(element_size)
+ .unwrap_or(usize::MAX)
+}
+
/// Helper function to get count from count_array at given index
/// Return 0 for null values or non-positive count.
#[inline]
@@ -320,3 +374,22 @@ fn get_count_with_validity(count_array: &Int64Array, idx: usize) -> usize {
if c > 0 { c as usize } else { 0 }
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::array_repeat_inner;
+ use arrow::array::{ArrayRef, Int64Array};
+ use std::sync::Arc;
+
+ #[test]
+ fn scalar_count_exceeding_max_array_size_returns_error() {
+ let element: ArrayRef = Arc::new(Int64Array::from(vec![1]));
+ let count: ArrayRef = Arc::new(Int64Array::from(vec![i64::MAX]));
+
+ let err = array_repeat_inner(&[element, count]).unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Execution error: array_repeat: requested length exceeds maximum array size"
+ );
+ }
+}
diff --git a/datafusion/sqllogictest/test_files/array/array_repeat.slt b/datafusion/sqllogictest/test_files/array/array_repeat.slt
index 8052f09cb3..9f17c449c8 100644
--- a/datafusion/sqllogictest/test_files/array/array_repeat.slt
+++ b/datafusion/sqllogictest/test_files/array/array_repeat.slt
@@ -43,6 +43,9 @@ select
----
[[1], [1], [1], [1], [1]] [[1.1, 2.2, 3.3], [1.1, 2.2, 3.3], [1.1, 2.2, 3.3]] [[NULL, NULL], [NULL, NULL], [NULL, NULL]] [[[1, 2], [3, 4]], [[1, 2], [3, 4]]]
+query error DataFusion error: Execution error: array_repeat: requested length exceeds maximum array size
+select array_repeat(1, 9223372036854775807);
+
query ????
select
array_repeat(arrow_cast([1], 'LargeList(Int64)'), 5),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org