You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2026/10/22 15:39:42 UTC
(arrow-rs) branch 59_maintenance updated: [59_maintenance] Write empty validity buffers for arrays without nulls in IPC writer (#11820)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch 59_maintenance
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/59_maintenance by this push:
new dcca3342cd [59_maintenance] Write empty validity buffers for arrays without nulls in IPC writer (#11820)
dcca3342cd is described below
commit dcca3342cd6ecb1446baa18fd5ce20d2c3496e09
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Thu Oct 22 11:39:37 2026 -0400
[59_maintenance] Write empty validity buffers for arrays without nulls in IPC writer (#11820)
* Backport of #11694
* For inclusion in #11664
* 58 backport is #11817 (requested in
https://github.com/apache/arrow-rs/issues/11665#issuecomment-6342385224)
Cherry-pick of
[49219f8](https://github.com/apache/arrow-rs/commit/49219f867f77422525627a07b84cd7a87207dcbe)
The cherry-pick applied cleanly but did not compile: on main
`write_array_data` has a `null_count` local introduced by #10128, which
is not on `59_maintenance`. This PR adds that one-line local (`let
null_count = array_data.null_count();`) so the backported code matches
main.
Co-authored-by: Aleksandar Zecevic <al...@databricks.com>
---
arrow-flight/src/encode.rs | 12 +++++------
arrow-ipc/src/writer.rs | 54 +++++++++++++++++++++++++++++++++++-----------
2 files changed, 47 insertions(+), 19 deletions(-)
diff --git a/arrow-flight/src/encode.rs b/arrow-flight/src/encode.rs
index 6adf4153c0..897e538bc1 100644
--- a/arrow-flight/src/encode.rs
+++ b/arrow-flight/src/encode.rs
@@ -1942,7 +1942,7 @@ mod tests {
])
.unwrap();
- verify_encoded_split(batch, 120).await;
+ verify_encoded_split(batch, 88).await;
}
#[tokio::test]
@@ -1953,7 +1953,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 4312).await;
+ verify_encoded_split(batch, 4304).await;
}
#[tokio::test]
@@ -1989,7 +1989,7 @@ mod tests {
// 5k over limit (which is 2x larger than limit of 5k)
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 5808).await;
+ verify_encoded_split(batch, 5784).await;
}
#[tokio::test]
@@ -2019,7 +2019,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 3336).await;
+ verify_encoded_split(batch, 3328).await;
}
#[tokio::test]
@@ -2033,7 +2033,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 5288).await;
+ verify_encoded_split(batch, 5280).await;
}
#[tokio::test]
@@ -2058,7 +2058,7 @@ mod tests {
// overage is much higher than ideal
// https://github.com/apache/arrow-rs/issues/3478
- verify_encoded_split(batch, 4136).await;
+ verify_encoded_split(batch, 4112).await;
}
/// Return size, in memory of flight data
diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs
index 73994fad8e..cae1712908 100644
--- a/arrow-ipc/src/writer.rs
+++ b/arrow-ipc/src/writer.rs
@@ -36,7 +36,6 @@ use arrow_array::builder::BufferBuilder;
use arrow_array::cast::*;
use arrow_array::types::{Int16Type, Int32Type, Int64Type, RunEndIndexType};
use arrow_array::*;
-use arrow_buffer::bit_util;
use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ToByteSlice};
use arrow_data::{ArrayData, ArrayDataBuilder, BufferSpec, layout};
use arrow_schema::*;
@@ -2373,10 +2372,11 @@ fn write_array_data(
) -> Result<i64, ArrowError> {
let mut offset = offset;
let num_rows = array_data.len();
+ let null_count = array_data.null_count();
if !matches!(array_data.data_type(), DataType::Null) {
meta.nodes.push(crate::FieldNode::new(
num_rows as i64,
- array_data.null_count() as i64,
+ null_count as i64,
));
} else {
// NullArray's null_count equals to len, but ArrayData null_count is always 0.
@@ -2384,16 +2384,11 @@ fn write_array_data(
.push(crate::FieldNode::new(num_rows as i64, num_rows as i64));
}
if has_validity_bitmap(array_data.data_type(), write_options) {
- // write null buffer if exists
+ // Fields with null_count == 0 may write a 0-length validity buffer
+ // instead of a materialized bitmap (see FieldNode in Message.fbs)
let null_buffer = match array_data.nulls() {
- None => {
- // create a buffer and fill it with valid bits
- let num_bytes = bit_util::ceil(num_rows, 8);
- let buffer = MutableBuffer::new(num_bytes);
- let buffer = buffer.with_bitset(num_bytes, true);
- buffer.into()
- }
- Some(buffer) => buffer.inner().sliced(),
+ Some(buffer) if null_count > 0 => buffer.inner().sliced(),
+ _ => MutableBuffer::new(0).into(),
};
offset = encode_sink_buffer(
@@ -3466,6 +3461,37 @@ mod tests {
assert_eq!(record_batch_slice, deserialized_batch);
}
+ #[test]
+ fn test_write_empty_validity_buffer_without_nulls() {
+ let batch = RecordBatch::try_from_iter([
+ ("a", Arc::new(Int32Array::from(vec![1, 2])) as _),
+ // Keeps a null buffer, but the slice has no nulls
+ (
+ "b",
+ Arc::new(Int32Array::from(vec![None, Some(2), Some(3)]).slice(1, 2)) as _,
+ ),
+ ("c", Arc::new(Int32Array::from(vec![Some(1), None])) as _),
+ ])
+ .unwrap();
+ assert_eq!(batch.column(1).nulls().map(|n| n.null_count()), Some(0));
+
+ let (_, encoded) = IpcDataGenerator::default()
+ .encode(
+ &batch,
+ &mut DictionaryTracker::new(false),
+ &Default::default(),
+ &mut Default::default(),
+ )
+ .unwrap();
+ let message = crate::root_as_message(&encoded.ipc_message).unwrap();
+ let buffers = message.header_as_record_batch().unwrap().buffers().unwrap();
+ // Each column writes a validity buffer, then a values buffer
+ let validity_lengths: Vec<_> = buffers.iter().step_by(2).map(|b| b.length()).collect();
+ assert_eq!(validity_lengths, [0, 0, 1]);
+
+ assert_eq!(deserialize_stream(serialize_stream(&batch)), batch);
+ }
+
#[test]
fn truncate_ipc_dictionary_array() {
fn create_batch() -> RecordBatch {
@@ -4613,13 +4639,15 @@ mod tests {
const IPC_ALIGNMENT: usize = 8;
let num_cols = 2;
- let num_rows = 1;
let mut fields = Vec::new();
let mut arrays = Vec::new();
for i in 0..num_cols {
let field = Field::new(format!("col_{i}"), DataType::Decimal128(38, 10), true);
- let array = Decimal128Array::from(vec![num_cols as i128; num_rows]);
+ // The null makes the writer emit a 1-byte validity buffer. With IPC_ALIGNMENT = 8,
+ // the writer pads that buffer to 8 bytes, so the decimal values that follow start
+ // at offset 8 of the body, which is not a multiple of 16.
+ let array = Decimal128Array::from(vec![Some(num_cols as i128), None]);
fields.push(field);
arrays.push(Arc::new(array) as Arc<dyn Array>);
}