You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by gi...@apache.org on 2026/10/18 03:44:52 UTC
(arrow-rs) branch gh-readonly-queue/main/pr-11209-c0970925a2bb29efe2f855305a76986959171685 created (now f53b221bd6)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-11209-c0970925a2bb29efe2f855305a76986959171685
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
at f53b221bd6 fix: handle zero-length null extension for empty unions (#11209)
This branch includes the following new commits:
new f53b221bd6 fix: handle zero-length null extension for empty unions (#11209)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(arrow-rs) 01/01: fix: handle zero-length null extension for empty unions (#11209)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-11209-c0970925a2bb29efe2f855305a76986959171685
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit f53b221bd6a002d6ca1ed4775b695e1df51d087f
Author: Efe <go...@gmail.com>
AuthorDate: Sun Oct 18 03:03:50 2026 +0000
fix: handle zero-length null extension for empty unions (#11209)
## Summary
`MutableArrayData::try_extend_nulls(0)` remains a no-op for empty
unions. For non-empty sparse and dense unions, null extension now uses a
nullable child field rather than assuming the first child is nullable.
Fixes #11198
## Testing
- `cargo test --locked --manifest-path arrow-data/Cargo.toml -p
arrow-data --lib` (64 passed)
- `cargo fmt --all -- --check`
- `git diff --check HEAD^ HEAD`
AI assistance was used during implementation.
---------
Signed-off-by: Efe Gökdemir <ef...@rexcode.co.uk>
---
arrow-data/src/transform/mod.rs | 106 +++++++++++++++++++++++++++++++++++++-
arrow-data/src/transform/union.rs | 36 +++++++------
2 files changed, 126 insertions(+), 16 deletions(-)
diff --git a/arrow-data/src/transform/mod.rs b/arrow-data/src/transform/mod.rs
index 2db03a42aa..c4cd2bee5b 100644
--- a/arrow-data/src/transform/mod.rs
+++ b/arrow-data/src/transform/mod.rs
@@ -843,6 +843,12 @@ impl<'a> MutableArrayData<'a> {
));
}
+ if len == 0 {
+ return Ok(());
+ }
+
+ (self.extend_nulls)(&mut self.data, len)?;
+
self.data.len += len;
let bit_len = bit_util::ceil(self.data.len, 8);
let nulls = self.data.null_buffer();
@@ -850,7 +856,6 @@ impl<'a> MutableArrayData<'a> {
.try_resize(bit_len, 0)
.map_err(|e| ArrowError::MemoryError(e.to_string()))?;
self.data.null_count += len;
- (self.extend_nulls)(&mut self.data, len)?;
Ok(())
}
@@ -1019,6 +1024,105 @@ mod test {
assert_eq!(mutable.len(), 1);
}
+ #[test]
+ fn test_try_extend_nulls_empty_union() {
+ use arrow_schema::{UnionFields, UnionMode};
+
+ for mode in [UnionMode::Sparse, UnionMode::Dense] {
+ let mut builder = ArrayData::builder(DataType::Union(UnionFields::empty(), mode))
+ .len(0)
+ .add_buffer(Buffer::from_vec(Vec::<i8>::new()));
+
+ if mode == UnionMode::Dense {
+ builder = builder.add_buffer(Buffer::from_vec(Vec::<i32>::new()));
+ }
+
+ let source = builder.build().unwrap();
+ let mut mutable = MutableArrayData::new(vec![&source], true, 0);
+
+ mutable.try_extend_nulls(0).unwrap();
+ assert_eq!(mutable.len(), 0);
+
+ let err = mutable.try_extend_nulls(1).unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Invalid argument error: Cannot extend a union with no fields with nulls"
+ );
+ assert_eq!(mutable.len(), 0);
+ }
+ }
+
+ #[test]
+ fn test_try_extend_nulls_union_uses_nullable_field() {
+ use arrow_schema::{Field, UnionFields};
+
+ let fields = UnionFields::try_new(
+ vec![0, 1],
+ vec![
+ Field::new("non_nullable", DataType::Int32, false),
+ Field::new("nullable", DataType::Int32, true),
+ ],
+ )
+ .unwrap();
+
+ for mode in [UnionMode::Sparse, UnionMode::Dense] {
+ let mut builder = ArrayData::builder(DataType::Union(fields.clone(), mode))
+ .len(0)
+ .add_buffer(Buffer::from_vec(Vec::<i8>::new()));
+
+ if mode == UnionMode::Dense {
+ builder = builder.add_buffer(Buffer::from_vec(Vec::<i32>::new()));
+ }
+
+ let source = builder
+ .child_data(vec![
+ ArrayData::new_empty(&DataType::Int32),
+ ArrayData::new_empty(&DataType::Int32),
+ ])
+ .build()
+ .unwrap();
+ let mut mutable = MutableArrayData::new(vec![&source], true, 1);
+
+ mutable.try_extend_nulls(1).unwrap();
+ let result = mutable.freeze();
+
+ assert_eq!(result.buffer::<i8>(0), &[1]);
+ assert_eq!(result.child_data()[1].null_count(), 1);
+ assert_eq!(result.child_data()[1].len(), 1);
+ if mode == UnionMode::Dense {
+ assert_eq!(result.buffer::<i32>(1), &[0]);
+ assert_eq!(result.child_data()[0].len(), 0);
+ } else {
+ assert_eq!(result.child_data()[0].len(), 1);
+ }
+ }
+ }
+
+ #[test]
+ fn test_try_extend_nulls_union_without_nullable_fields() {
+ use arrow_schema::{Field, UnionFields};
+
+ let fields = UnionFields::try_new(
+ vec![0],
+ vec![Field::new("non_nullable", DataType::Int32, false)],
+ )
+ .unwrap();
+ let source = ArrayData::builder(DataType::Union(fields, UnionMode::Sparse))
+ .len(0)
+ .add_buffer(Buffer::from_vec(Vec::<i8>::new()))
+ .child_data(vec![ArrayData::new_empty(&DataType::Int32)])
+ .build()
+ .unwrap();
+ let mut mutable = MutableArrayData::new(vec![&source], true, 0);
+
+ let err = mutable.try_extend_nulls(1).unwrap_err();
+ assert_eq!(
+ err.to_string(),
+ "Invalid argument error: Cannot extend a union with no nullable fields with nulls"
+ );
+ assert_eq!(mutable.len(), 0);
+ }
+
#[test]
fn test_list_append_with_capacities() {
let array = ArrayData::new_empty(&DataType::List(Arc::new(Field::new(
diff --git a/arrow-data/src/transform/union.rs b/arrow-data/src/transform/union.rs
index ce2c55c391..410941c97d 100644
--- a/arrow-data/src/transform/union.rs
+++ b/arrow-data/src/transform/union.rs
@@ -19,6 +19,20 @@ use super::{_MutableArrayData, Extend};
use crate::ArrayData;
use arrow_schema::{ArrowError, DataType};
+fn nullable_union_field(fields: &arrow_schema::UnionFields) -> Result<(usize, i8), ArrowError> {
+ fields
+ .iter()
+ .enumerate()
+ .find_map(|(index, (type_id, field))| field.is_nullable().then_some((index, type_id)))
+ .ok_or_else(|| {
+ ArrowError::InvalidArgumentError(if fields.is_empty() {
+ "Cannot extend a union with no fields with nulls".to_owned()
+ } else {
+ "Cannot extend a union with no nullable fields with nulls".to_owned()
+ })
+ })
+}
+
pub(super) fn build_extend_sparse(array: &ArrayData) -> Extend<'_> {
let type_ids = array.buffer::<i8>(0);
@@ -77,23 +91,19 @@ pub(super) fn extend_nulls_dense(
let DataType::Union(fields, _) = &mutable.data_type else {
unreachable!()
};
- let first_type_id = fields
- .iter()
- .next()
- .expect("union must have at least one field")
- .0;
+ let (child_index, null_type_id) = nullable_union_field(fields)?;
// Extend type_ids buffer
mutable
.buffer1
- .try_extend_from_slice(&vec![first_type_id; len])
+ .try_extend_from_slice(&vec![null_type_id; len])
.map_err(|e| ArrowError::MemoryError(e.to_string()))?;
- // Dense: extend offsets pointing into the first child, then extend nulls in that child
- let child_offset = mutable.child_data[0].len();
+ // Dense: point into a nullable child and extend its values with nulls.
+ let child_offset = mutable.child_data[child_index].len();
let (start, end) = (child_offset as i32, (child_offset + len) as i32);
mutable.buffer2.extend(start..end);
- mutable.child_data[0].try_extend_nulls(len)?;
+ mutable.child_data[child_index].try_extend_nulls(len)?;
Ok(())
}
@@ -104,16 +114,12 @@ pub(super) fn extend_nulls_sparse(
let DataType::Union(fields, _) = &mutable.data_type else {
unreachable!()
};
- let first_type_id = fields
- .iter()
- .next()
- .expect("union must have at least one field")
- .0;
+ let (_, null_type_id) = nullable_union_field(fields)?;
// Extend type_ids buffer
mutable
.buffer1
- .try_extend_from_slice(&vec![first_type_id; len])
+ .try_extend_from_slice(&vec![null_type_id; len])
.map_err(|e| ArrowError::MemoryError(e.to_string()))?;
// Sparse: extend nulls in ALL children