You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/24 22:47:06 UTC
(datafusion) branch gh-readonly-queue/main/pr-21789-7d5ddca187e2f37f234b5dfa0468a0b4e1671118 created (now 794f30e0d8)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-21789-7d5ddca187e2f37f234b5dfa0468a0b4e1671118
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 794f30e0d8 perf: Add bulk NULL-aware string builders, use in `lower` and `upper` (#21789)
This branch includes the following new commits:
new 794f30e0d8 perf: Add bulk NULL-aware string builders, use in `lower` and `upper` (#21789)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: perf: Add bulk NULL-aware string builders, use in `lower` and `upper` (#21789)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-21789-7d5ddca187e2f37f234b5dfa0468a0b4e1671118
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 794f30e0d8de487a65c1b810b42f305d9364dfd6
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Fri Apr 24 18:46:45 2026 -0400
perf: Add bulk NULL-aware string builders, use in `lower` and `upper` (#21789)
## Which issue does this PR close?
- Part of #21684
## Rationale for this change
Introduce three new string array builders with bulk null tracking:
- `StringArrayBuilder` (Utf8)
- `LargeStringArrayBuilder` (LargeUtf8)
- `StringViewArrayBuilder` (Utf8View)
Each builder has the following API:
- append_value(&str) -- add a non-NULL value (row)
- append_placeholder() -- add a NULL row placeholder
- finish(Option<NullBuffer>) -- finish the build, specify NULLs
These are the counterpart of Arrow's `GenericStringBuilder` /
`StringViewBuilder` but it
skips per-row NULL buffer maintenance, which lets callers compute the
NULL buffer in
bulk when possible.
This PR also switches `case_conversion` to use the new APIs, which is
used to
implement `lower`, `upper`, and the Spark equivalents. This improves
`lower` / `upper`
performance by 3-15% on microbenchmarks. More UDFs (~10) will be
converted to use
this API in future PRs.
## What changes are included in this PR?
* Add new builders
* Add unit tests
* Adopt builders in `case_conversion`
## Are these changes tested?
Yes.
## Are there any user-facing changes?
No.
---
datafusion/functions/src/string/common.rs | 63 +++--
datafusion/functions/src/strings.rs | 456 +++++++++++++++++++++++++++++-
2 files changed, 491 insertions(+), 28 deletions(-)
diff --git a/datafusion/functions/src/string/common.rs b/datafusion/functions/src/string/common.rs
index ddecd39ddb..70699c3653 100644
--- a/datafusion/functions/src/string/common.rs
+++ b/datafusion/functions/src/string/common.rs
@@ -19,10 +19,10 @@
use std::sync::Arc;
-use crate::strings::append_view;
+use crate::strings::{GenericStringArrayBuilder, StringViewArrayBuilder, append_view};
use arrow::array::{
- Array, ArrayRef, GenericStringArray, GenericStringBuilder, NullBufferBuilder,
- OffsetSizeTrait, StringViewArray, StringViewBuilder, new_null_array,
+ Array, ArrayRef, GenericStringArray, NullBufferBuilder, OffsetSizeTrait,
+ StringViewArray, new_null_array,
};
use arrow::buffer::{Buffer, OffsetBuffer, ScalarBuffer};
use arrow::datatypes::DataType;
@@ -349,18 +349,30 @@ where
>(array, op)?)),
DataType::Utf8View => {
let string_array = as_string_view_array(array)?;
- let mut string_builder =
- StringViewBuilder::with_capacity(string_array.len());
-
- for str in string_array.iter() {
- if let Some(str) = str {
- string_builder.append_value(op(str));
- } else {
- string_builder.append_null();
+ let item_len = string_array.len();
+ // Null-preserving: reuse the input null buffer as the output null buffer.
+ let nulls = string_array.nulls().cloned();
+ let mut builder = StringViewArrayBuilder::with_capacity(item_len);
+
+ if let Some(ref n) = nulls {
+ for i in 0..item_len {
+ if n.is_null(i) {
+ builder.append_placeholder();
+ } else {
+ // SAFETY: `n.is_null(i)` was false in the branch above.
+ let s = unsafe { string_array.value_unchecked(i) };
+ builder.append_value(&op(s));
+ }
+ }
+ } else {
+ for i in 0..item_len {
+ // SAFETY: no null buffer means every index is valid.
+ let s = unsafe { string_array.value_unchecked(i) };
+ builder.append_value(&op(s));
}
}
- Ok(ColumnarValue::Array(Arc::new(string_builder.finish())))
+ Ok(ColumnarValue::Array(Arc::new(builder.finish(nulls)?)))
}
other => exec_err!("Unsupported data type {other:?} for function {name}"),
},
@@ -400,17 +412,28 @@ where
let start = offsets.first().unwrap().as_usize();
let end = offsets.last().unwrap().as_usize();
let capacity = (end - start) + PRE_ALLOC_BYTES;
- let mut builder = GenericStringBuilder::<O>::with_capacity(item_len, capacity);
+ // Null-preserving: reuse the input null buffer as the output null buffer.
+ let nulls = string_array.nulls().cloned();
+ let mut builder = GenericStringArrayBuilder::<O>::with_capacity(item_len, capacity);
- if string_array.null_count() == 0 {
- let iter =
- (0..item_len).map(|i| Some(op(unsafe { string_array.value_unchecked(i) })));
- builder.extend(iter);
+ if let Some(ref n) = nulls {
+ for i in 0..item_len {
+ if n.is_null(i) {
+ builder.append_placeholder();
+ } else {
+ // SAFETY: `n.is_null(i)` was false in the branch above.
+ let s = unsafe { string_array.value_unchecked(i) };
+ builder.append_value(&op(s));
+ }
+ }
} else {
- let iter = string_array.iter().map(|string| string.map(&op));
- builder.extend(iter);
+ for i in 0..item_len {
+ // SAFETY: no null buffer means every index is valid.
+ let s = unsafe { string_array.value_unchecked(i) };
+ builder.append_value(&op(s));
+ }
}
- Ok(Arc::new(builder.finish()))
+ Ok(Arc::new(builder.finish(nulls)?))
}
/// Fast path for case conversion on an all-ASCII string array. ASCII case
diff --git a/datafusion/functions/src/strings.rs b/datafusion/functions/src/strings.rs
index f986ffd2e3..ee609767bb 100644
--- a/datafusion/functions/src/strings.rs
+++ b/datafusion/functions/src/strings.rs
@@ -15,13 +15,14 @@
// specific language governing permissions and limitations
// under the License.
+use std::marker::PhantomData;
use std::mem::size_of;
use datafusion_common::{Result, exec_datafusion_err, internal_err};
use arrow::array::{
- Array, ArrayAccessor, ArrayDataBuilder, BinaryArray, ByteView, LargeStringArray,
- StringArray, StringViewArray, make_view,
+ Array, ArrayAccessor, ArrayDataBuilder, BinaryArray, ByteView, GenericStringArray,
+ LargeStringArray, OffsetSizeTrait, StringArray, StringViewArray, make_view,
};
use arrow::buffer::{Buffer, MutableBuffer, NullBuffer, ScalarBuffer};
use arrow::datatypes::DataType;
@@ -29,9 +30,13 @@ use arrow::datatypes::DataType;
/// Builder used by `concat`/`concat_ws` to assemble a [`StringArray`] one row
/// at a time from multiple input columns.
///
-/// Each row is written via repeated `write` calls, followed by a single
-/// `append_offset` call to commit the row. The output null buffer is supplied
-/// by the caller at `finish` time.
+/// Each row is written via repeated `write` calls (one per input fragment)
+/// followed by a single `append_offset` to commit the row. The output null
+/// buffer is computed in bulk by the caller and supplied to `finish`, avoiding
+/// per-row NULL handling work.
+///
+/// For the common "produce one `&str` per row" pattern, prefer
+/// `GenericStringArrayBuilder` instead.
pub(crate) struct ConcatStringBuilder {
offsets_buffer: MutableBuffer,
value_buffer: MutableBuffer,
@@ -157,9 +162,13 @@ impl ConcatStringBuilder {
/// Builder used by `concat`/`concat_ws` to assemble a [`StringViewArray`] one
/// row at a time from multiple input columns.
///
-/// Each row is written via repeated `write` calls, followed by a single
-/// `append_offset` call to commit the row as a single string view. The output
-/// null buffer is supplied by the caller at `finish` time.
+/// Each row is written via repeated `write` calls (one per input
+/// fragment) followed by a single `append_offset` to commit the row
+/// as a single string view. The output null buffer is supplied by the caller
+/// at `finish` time, avoiding per-row NULL handling work.
+///
+/// For the common "produce one `&str` per row" pattern, prefer
+/// [`StringViewArrayBuilder`] instead.
pub(crate) struct ConcatStringViewBuilder {
views: Vec<u128>,
data: Vec<u8>,
@@ -292,6 +301,9 @@ impl ConcatStringViewBuilder {
/// Builder used by `concat`/`concat_ws` to assemble a [`LargeStringArray`] one
/// row at a time from multiple input columns. See [`ConcatStringBuilder`] for
/// details on the row-composition contract.
+///
+/// For the common "produce one `&str` per row" pattern, prefer
+/// `GenericStringArrayBuilder` instead.
pub(crate) struct ConcatLargeStringBuilder {
offsets_buffer: MutableBuffer,
value_buffer: MutableBuffer,
@@ -414,6 +426,242 @@ impl ConcatLargeStringBuilder {
}
}
+// ----------------------------------------------------------------------------
+// Bulk-nulls builders
+//
+// These builders are similar to Arrow's `GenericStringBuilder` and
+// `StringViewBuilder`, except that callers must pass the NULL bitmap to
+// `finish()`, rather than maintaining it iteratively (per-row). For callers
+// that can compute the NULL bitmap in bulk (which is true of many
+// string-related UDFs), this can be significantly more efficient.
+//
+// For a row known to be null, call `append_placeholder` to advance the row
+// count without touching the value buffer; the caller MUST ensure that the
+// corresponding bit is cleared (0 = null) in the null buffer passed to
+// `finish`.
+// ----------------------------------------------------------------------------
+
+/// Builder for a [`GenericStringArray<O>`] that defers null tracking to
+/// `finish`. Instantiate with `O = i32` for [`StringArray`] (Utf8) or
+/// `O = i64` for [`LargeStringArray`] (LargeUtf8).
+pub(crate) struct GenericStringArrayBuilder<O: OffsetSizeTrait> {
+ offsets_buffer: MutableBuffer,
+ value_buffer: MutableBuffer,
+ placeholder_count: usize,
+ _phantom: PhantomData<O>,
+}
+
+impl<O: OffsetSizeTrait> GenericStringArrayBuilder<O> {
+ pub fn with_capacity(item_capacity: usize, data_capacity: usize) -> Self {
+ let capacity = item_capacity
+ .checked_add(1)
+ .map(|i| i.saturating_mul(size_of::<O>()))
+ .expect("capacity integer overflow");
+
+ let mut offsets_buffer = MutableBuffer::with_capacity(capacity);
+ offsets_buffer.push(O::usize_as(0));
+ Self {
+ offsets_buffer,
+ value_buffer: MutableBuffer::with_capacity(data_capacity),
+ placeholder_count: 0,
+ _phantom: PhantomData,
+ }
+ }
+
+ /// Append `value` as the next row.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the cumulative byte length exceeds `O::MAX`.
+ pub fn append_value(&mut self, value: &str) {
+ self.value_buffer.extend_from_slice(value.as_bytes());
+ let next_offset =
+ O::from_usize(self.value_buffer.len()).expect("byte array offset overflow");
+ self.offsets_buffer.push(next_offset);
+ }
+
+ /// Append an empty placeholder row. The corresponding slot must be masked
+ /// as null by the null buffer passed to `finish`.
+ pub fn append_placeholder(&mut self) {
+ let next_offset =
+ O::from_usize(self.value_buffer.len()).expect("byte array offset overflow");
+ self.offsets_buffer.push(next_offset);
+ self.placeholder_count += 1;
+ }
+
+ /// Finalize into a [`GenericStringArray<O>`] using the caller-supplied
+ /// null buffer.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when `null_buffer.len()` does not match the number of
+ /// appended rows.
+ pub fn finish(
+ self,
+ null_buffer: Option<NullBuffer>,
+ ) -> Result<GenericStringArray<O>> {
+ let row_count = self.offsets_buffer.len() / size_of::<O>() - 1;
+ if let Some(ref n) = null_buffer
+ && n.len() != row_count
+ {
+ return internal_err!(
+ "Null buffer length ({}) must match row count ({row_count})",
+ n.len()
+ );
+ }
+ let null_count = null_buffer.as_ref().map_or(0, |n| n.null_count());
+ debug_assert!(
+ null_count >= self.placeholder_count,
+ "{} placeholder rows but null buffer has {null_count} nulls",
+ self.placeholder_count,
+ );
+ let array_data = ArrayDataBuilder::new(GenericStringArray::<O>::DATA_TYPE)
+ .len(row_count)
+ .add_buffer(self.offsets_buffer.into())
+ .add_buffer(self.value_buffer.into())
+ .nulls(null_buffer);
+ // SAFETY: every appended value came from a `&str`, so the value
+ // buffer is valid UTF-8 and offsets are monotonically non-decreasing.
+ let array_data = unsafe { array_data.build_unchecked() };
+ Ok(GenericStringArray::<O>::from(array_data))
+ }
+}
+
+/// Starting size for the long-string data block; matches Arrow's
+/// `GenericByteViewBuilder` default.
+const STARTING_BLOCK_SIZE: u32 = 8 * 1024;
+/// Maximum size each long-string data block grows to; matches Arrow's
+/// `GenericByteViewBuilder` default.
+const MAX_BLOCK_SIZE: u32 = 2 * 1024 * 1024;
+
+/// Builder for a [`StringViewArray`] that defers null tracking to `finish`.
+///
+/// Modeled on Arrow's [`arrow::array::builder::StringViewBuilder`] but
+/// without per-row [`arrow::array::builder::NullBufferBuilder`] maintenance.
+/// Short strings (≤ 12 bytes) are inlined into the view itself; long strings
+/// are appended into an in-progress data block. When the in-progress block
+/// fills up it is flushed into `completed` and a new block — double the size
+/// of the last, capped at [`MAX_BLOCK_SIZE`] — is started.
+pub(crate) struct StringViewArrayBuilder {
+ views: Vec<u128>,
+ in_progress: Vec<u8>,
+ completed: Vec<Buffer>,
+ /// Current block-size target; doubles each time a block is flushed, up to
+ /// [`MAX_BLOCK_SIZE`].
+ block_size: u32,
+ placeholder_count: usize,
+}
+
+impl StringViewArrayBuilder {
+ pub fn with_capacity(item_capacity: usize) -> Self {
+ Self {
+ views: Vec::with_capacity(item_capacity),
+ in_progress: Vec::new(),
+ completed: Vec::new(),
+ block_size: STARTING_BLOCK_SIZE,
+ placeholder_count: 0,
+ }
+ }
+
+ /// Doubles the block-size target (capped at [`MAX_BLOCK_SIZE`]) and
+ /// returns the new size. The first call returns `2 * STARTING_BLOCK_SIZE`.
+ fn next_block_size(&mut self) -> u32 {
+ if self.block_size < MAX_BLOCK_SIZE {
+ self.block_size = self.block_size.saturating_mul(2);
+ }
+ self.block_size
+ }
+
+ /// Append `value` as the next row.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the value length, the in-progress buffer offset, or the
+ /// number of completed buffers exceeds `i32::MAX`. The ByteView spec
+ /// uses signed 32-bit integers for these fields; exceeding `i32::MAX`
+ /// would produce an array that does not round-trip through Arrow IPC
+ /// (see <https://github.com/apache/arrow-rs/issues/6172>).
+ #[inline]
+ pub fn append_value(&mut self, value: &str) {
+ let v = value.as_bytes();
+ let length: u32 =
+ i32::try_from(v.len()).expect("value length exceeds i32::MAX") as u32;
+ if length <= 12 {
+ self.views.push(make_view(v, 0, 0));
+ return;
+ }
+
+ let required_cap = self.in_progress.len() + length as usize;
+ if self.in_progress.capacity() < required_cap {
+ self.flush_in_progress();
+ let to_reserve = (length as usize).max(self.next_block_size() as usize);
+ self.in_progress.reserve(to_reserve);
+ }
+
+ let buffer_index: u32 = i32::try_from(self.completed.len())
+ .expect("buffer count exceeds i32::MAX")
+ as u32;
+ let offset: u32 = i32::try_from(self.in_progress.len())
+ .expect("offset exceeds i32::MAX") as u32;
+ self.in_progress.extend_from_slice(v);
+ self.views.push(make_view(v, buffer_index, offset));
+ }
+
+ /// Append an empty placeholder row. The corresponding slot must be
+ /// masked as null by the null buffer passed to `finish`.
+ #[inline]
+ pub fn append_placeholder(&mut self) {
+ // Zero-length inline view — `length` field is 0, no buffer ref.
+ self.views.push(0);
+ self.placeholder_count += 1;
+ }
+
+ fn flush_in_progress(&mut self) {
+ if !self.in_progress.is_empty() {
+ let block = std::mem::take(&mut self.in_progress);
+ self.completed.push(Buffer::from_vec(block));
+ }
+ }
+
+ /// Finalize into a [`StringViewArray`] using the caller-supplied null
+ /// buffer.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when `null_buffer.len()` does not match the number of
+ /// appended rows.
+ pub fn finish(mut self, null_buffer: Option<NullBuffer>) -> Result<StringViewArray> {
+ if let Some(ref n) = null_buffer
+ && n.len() != self.views.len()
+ {
+ return internal_err!(
+ "Null buffer length ({}) must match row count ({})",
+ n.len(),
+ self.views.len()
+ );
+ }
+ let null_count = null_buffer.as_ref().map_or(0, |n| n.null_count());
+ debug_assert!(
+ null_count >= self.placeholder_count,
+ "{} placeholder rows but null buffer has {null_count} nulls",
+ self.placeholder_count,
+ );
+ self.flush_in_progress();
+ // SAFETY: every long-string view references bytes we wrote ourselves
+ // into `self.completed`, with prefixes derived from those same bytes.
+ // Inline views were built from valid `&str`. Placeholder views are
+ // zero-length with no buffer reference.
+ let array = unsafe {
+ StringViewArray::new_unchecked(
+ ScalarBuffer::from(self.views),
+ self.completed,
+ null_buffer,
+ )
+ };
+ Ok(array)
+ }
+}
+
/// Append a new view to the views buffer with the given substr.
///
/// Callers are responsible for their own null tracking.
@@ -512,4 +760,196 @@ mod tests {
fn test_overflow_concat_large_string_builder() {
let _builder = ConcatLargeStringBuilder::with_capacity(usize::MAX, usize::MAX);
}
+
+ #[test]
+ fn string_array_builder_empty() {
+ let builder = GenericStringArrayBuilder::<i32>::with_capacity(0, 0);
+ let array = builder.finish(None).unwrap();
+ assert_eq!(array.len(), 0);
+ }
+
+ #[test]
+ fn string_array_builder_no_nulls() {
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(3, 16);
+ builder.append_value("foo");
+ builder.append_value("");
+ builder.append_value("hello world");
+ let array = builder.finish(None).unwrap();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.value(0), "foo");
+ assert_eq!(array.value(1), "");
+ assert_eq!(array.value(2), "hello world");
+ assert_eq!(array.null_count(), 0);
+ }
+
+ #[test]
+ fn string_array_builder_with_nulls() {
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(3, 8);
+ builder.append_value("a");
+ builder.append_placeholder();
+ builder.append_value("c");
+ let nulls = NullBuffer::from(vec![true, false, true]);
+ let array = builder.finish(Some(nulls)).unwrap();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.value(0), "a");
+ assert!(array.is_null(1));
+ assert_eq!(array.value(2), "c");
+ }
+
+ #[test]
+ fn string_array_builder_null_buffer_length_mismatch() {
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(2, 4);
+ builder.append_value("a");
+ builder.append_value("b");
+ let nulls = NullBuffer::from(vec![true, false, true]);
+ assert!(builder.finish(Some(nulls)).is_err());
+ }
+
+ #[test]
+ #[cfg(debug_assertions)]
+ #[should_panic(expected = "placeholder rows")]
+ fn string_array_builder_placeholder_without_null_mask() {
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(2, 4);
+ builder.append_value("a");
+ builder.append_placeholder();
+ // Slot 1 is a placeholder but the null buffer doesn't mark it null.
+ let nulls = NullBuffer::from(vec![true, true]);
+ let _ = builder.finish(Some(nulls));
+ }
+
+ #[test]
+ #[cfg(debug_assertions)]
+ #[should_panic(expected = "placeholder rows")]
+ fn string_array_builder_placeholder_with_none_null_buffer() {
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(1, 4);
+ builder.append_placeholder();
+ let _ = builder.finish(None);
+ }
+
+ #[test]
+ fn string_array_builder_all_placeholders() {
+ let mut builder = GenericStringArrayBuilder::<i32>::with_capacity(3, 0);
+ builder.append_placeholder();
+ builder.append_placeholder();
+ builder.append_placeholder();
+ let nulls = NullBuffer::from(vec![false, false, false]);
+ let array = builder.finish(Some(nulls)).unwrap();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.null_count(), 3);
+ assert!((0..3).all(|i| array.is_null(i)));
+ }
+
+ #[test]
+ fn large_string_array_builder_with_nulls() {
+ let mut builder = GenericStringArrayBuilder::<i64>::with_capacity(3, 8);
+ builder.append_value("a");
+ builder.append_placeholder();
+ builder.append_value("c");
+ let nulls = NullBuffer::from(vec![true, false, true]);
+ let array = builder.finish(Some(nulls)).unwrap();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.value(0), "a");
+ assert!(array.is_null(1));
+ assert_eq!(array.value(2), "c");
+ }
+
+ #[test]
+ fn string_view_array_builder_empty() {
+ let builder = StringViewArrayBuilder::with_capacity(0);
+ let array = builder.finish(None).unwrap();
+ assert_eq!(array.len(), 0);
+ }
+
+ #[test]
+ fn string_view_array_builder_inline_and_buffer() {
+ let mut builder = StringViewArrayBuilder::with_capacity(3);
+ builder.append_value("short"); // ≤ 12 bytes, inline
+ builder.append_value("a string longer than twelve bytes");
+ builder.append_value("");
+ let array = builder.finish(None).unwrap();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.value(0), "short");
+ assert_eq!(array.value(1), "a string longer than twelve bytes");
+ assert_eq!(array.value(2), "");
+ }
+
+ #[test]
+ fn string_view_array_builder_with_nulls() {
+ let mut builder = StringViewArrayBuilder::with_capacity(4);
+ builder.append_value("a string longer than twelve bytes");
+ builder.append_placeholder();
+ builder.append_value("short");
+ builder.append_placeholder();
+ let nulls = NullBuffer::from(vec![true, false, true, false]);
+ let array = builder.finish(Some(nulls)).unwrap();
+ assert_eq!(array.len(), 4);
+ assert_eq!(array.value(0), "a string longer than twelve bytes");
+ assert!(array.is_null(1));
+ assert_eq!(array.value(2), "short");
+ assert!(array.is_null(3));
+ }
+
+ #[test]
+ fn string_view_array_builder_all_placeholders() {
+ let mut builder = StringViewArrayBuilder::with_capacity(3);
+ builder.append_placeholder();
+ builder.append_placeholder();
+ builder.append_placeholder();
+ let nulls = NullBuffer::from(vec![false, false, false]);
+ let array = builder.finish(Some(nulls)).unwrap();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.null_count(), 3);
+ assert!((0..3).all(|i| array.is_null(i)));
+ }
+
+ #[test]
+ fn string_view_array_builder_null_buffer_length_mismatch() {
+ let mut builder = StringViewArrayBuilder::with_capacity(2);
+ builder.append_value("a");
+ builder.append_value("b");
+ let nulls = NullBuffer::from(vec![true, false, true]);
+ assert!(builder.finish(Some(nulls)).is_err());
+ }
+
+ #[test]
+ #[cfg(debug_assertions)]
+ #[should_panic(expected = "placeholder rows")]
+ fn string_view_array_builder_placeholder_without_null_mask() {
+ let mut builder = StringViewArrayBuilder::with_capacity(2);
+ builder.append_value("a");
+ builder.append_placeholder();
+ let nulls = NullBuffer::from(vec![true, true]);
+ let _ = builder.finish(Some(nulls));
+ }
+
+ #[test]
+ #[cfg(debug_assertions)]
+ #[should_panic(expected = "placeholder rows")]
+ fn string_view_array_builder_placeholder_with_none_null_buffer() {
+ let mut builder = StringViewArrayBuilder::with_capacity(1);
+ builder.append_placeholder();
+ let _ = builder.finish(None);
+ }
+
+ #[test]
+ fn string_view_array_builder_flushes_full_blocks() {
+ // Each value is 300 bytes. The first data block is 2 × STARTING_BLOCK_SIZE
+ // = 16 KiB, so ~50 values saturate it and the rest spill into additional
+ // blocks.
+ let value = "x".repeat(300);
+ let mut builder = StringViewArrayBuilder::with_capacity(100);
+ for _ in 0..100 {
+ builder.append_value(&value);
+ }
+ let array = builder.finish(None).unwrap();
+ assert_eq!(array.len(), 100);
+ assert!(
+ array.data_buffers().len() > 1,
+ "expected multiple data buffers, got {}",
+ array.data_buffers().len()
+ );
+ for i in 0..100 {
+ assert_eq!(array.value(i), value);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org