You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/22 15:08:12 UTC
(datafusion) branch main updated: perf: Optimize `translate` to use new bulk-NULL string builders (#22171)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 4a41173ba3 perf: Optimize `translate` to use new bulk-NULL string builders (#22171)
4a41173ba3 is described below
commit 4a41173ba3df9b5d47638599c819a1e6e46ad92b
Author: Neil Conway <ne...@gmail.com>
AuthorDate: Fri May 22 10:24:24 2026 -0400
perf: Optimize `translate` to use new bulk-NULL string builders (#22171)
## Which issue does this PR close?
- Closes #22170.
## Rationale for this change
This PR refactors and optimizes the `translate` UDF. In particular, we
switch to using the new bulk-NULL string builders, avoiding per-row NULL
computation, and avoid an intermediate string copy by using
`append_with` / `append_byte_map`.
Benchmarks (Arm64):
Scalar (scalar_from_to)
- size=1024, str_len=8: 13.6 µs → 8.5 µs (-37.5%)
- size=1024, str_len=32: 26.2 µs → 20.6 µs (-21.4%)
- size=1024, str_len=128: 87.9 µs → 68.9 µs (-21.6%)
- size=1024, str_len=1024: 572.9 µs → 531.1 µs (-7.3%)
- size=4096, str_len=8: 51.6 µs → 31.4 µs (-39.1%)
- size=4096, str_len=32: 103.1 µs → 79.8 µs (-22.6%)
- size=4096, str_len=128: 341.1 µs → 273.4 µs (-19.8%)
- size=4096, str_len=1024: 2.3 ms → 2.1 ms (-8.7%)
Array — ASCII (array_from_to)
- size=1024, str_len=8: 50.6 µs → 21.2 µs (-58.1%)
- size=1024, str_len=32: 106.5 µs → 26.7 µs (-74.9%)
- size=1024, str_len=128: 265.4 µs → 59.9 µs (-77.4%)
- size=1024, str_len=1024: 1760.8 µs → 797.1 µs (-54.7%)
- size=4096, str_len=8: 211.4 µs → 84.3 µs (-60.1%)
- size=4096, str_len=32: 435.2 µs → 120.6 µs (-72.3%)
- size=4096, str_len=128: 1079.0 µs → 487.6 µs (-54.8%)
- size=4096, str_len=1024: 7.2 ms → 3.2 ms (-55.6%)
Array — non-ASCII (array_from_to_non_ascii)
- size=1024, str_len=8: 71.2 µs → 68.6 µs (-3.7%)
- size=1024, str_len=32: 228.8 µs → 236.9 µs (+3.5%)
- size=1024, str_len=128: 880.5 µs → 881.4 µs (+0.1%)
- size=1024, str_len=1024: 6.7 ms → 6.7 ms (+0.6%)
- size=4096, str_len=8: 375.5 µs → 376.6 µs (+0.3%)
- size=4096, str_len=32: 1041.2 µs → 1079.6 µs (+3.7%)
- size=4096, str_len=128: 3.5 ms → 3.6 ms (+2.9%)
- size=4096, str_len=1024: 27.0 ms → 26.8 ms (-0.7%)
## What changes are included in this PR?
* Switch from using the Rust StringBuilders to our new bulk-NULL string
builders
* Switch from per-row NULL checks to computing the NULL bitmaps with
`NullBuffer::union_many`
* Use `append_with` and `append_byte_map` rather than `append_value`,
which avoids an intermediate scratch buffer
* Refactor lookup table code to use a single `TranslationTable` enum
* Add a benchmark for the "varying `from`/`to`, Unicode strings" case
* Add a unit test
## Are these changes tested?
Yes; new test added.
## Are there any user-facing changes?
No.
---
datafusion/functions/benches/translate.rs | 59 +++-
datafusion/functions/src/unicode/translate.rs | 406 ++++++++++++++++----------
2 files changed, 316 insertions(+), 149 deletions(-)
diff --git a/datafusion/functions/benches/translate.rs b/datafusion/functions/benches/translate.rs
index d0568ba0f5..adde7b4bd7 100644
--- a/datafusion/functions/benches/translate.rs
+++ b/datafusion/functions/benches/translate.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::OffsetSizeTrait;
+use arrow::array::{GenericStringArray, OffsetSizeTrait};
use arrow::datatypes::{DataType, Field};
use arrow::util::bench_util::create_string_array_with_len;
use criterion::{Criterion, SamplingMode, criterion_group, criterion_main};
@@ -23,10 +23,37 @@ use datafusion_common::config::ConfigOptions;
use datafusion_common::{DataFusionError, ScalarValue};
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
use datafusion_functions::unicode;
+use rand::SeedableRng;
+use rand::prelude::IndexedRandom;
+use rand::rngs::StdRng;
use std::hint::black_box;
use std::sync::Arc;
use std::time::Duration;
+// Mix of 2-byte (Greek) and 3-byte (CJK/Hangul) UTF-8 to exercise
+// variable-width char paths in translate.
+const NON_ASCII_ALPHABET: &[char] = &[
+ 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η', 'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'ο', 'π', 'ρ',
+ 'σ', 'τ', 'υ', 'φ', 'χ', 'ψ', 'ω', '日', '本', '語', '中', '文', '한', '국', '어',
+];
+
+fn create_non_ascii_string_array<O: OffsetSizeTrait>(
+ size: usize,
+ char_count: usize,
+ seed: u64,
+) -> GenericStringArray<O> {
+ let mut rng = StdRng::seed_from_u64(seed);
+ (0..size)
+ .map(|_| {
+ Some(
+ (0..char_count)
+ .map(|_| *NON_ASCII_ALPHABET.choose(&mut rng).unwrap())
+ .collect::<String>(),
+ )
+ })
+ .collect()
+}
+
fn create_args_array_from_to<O: OffsetSizeTrait>(
size: usize,
str_len: usize,
@@ -42,6 +69,25 @@ fn create_args_array_from_to<O: OffsetSizeTrait>(
]
}
+fn create_args_array_from_to_non_ascii<O: OffsetSizeTrait>(
+ size: usize,
+ str_len: usize,
+) -> Vec<ColumnarValue> {
+ let string_array = Arc::new(create_non_ascii_string_array::<O>(
+ size,
+ str_len,
+ 0xA110_AAAA,
+ ));
+ let from_array = Arc::new(create_non_ascii_string_array::<O>(size, 3, 0xA110_BBBB));
+ let to_array = Arc::new(create_non_ascii_string_array::<O>(size, 2, 0xA110_CCCC));
+
+ vec![
+ ColumnarValue::Array(string_array),
+ ColumnarValue::Array(from_array),
+ ColumnarValue::Array(to_array),
+ ]
+}
+
fn create_args_scalar_from_to<O: OffsetSizeTrait>(
size: usize,
str_len: usize,
@@ -91,6 +137,17 @@ fn criterion_benchmark(c: &mut Criterion) {
})
});
+ let args = create_args_array_from_to_non_ascii::<i32>(size, str_len);
+ group.bench_function(
+ format!("array_from_to_non_ascii [str_len={str_len}]"),
+ |b| {
+ b.iter(|| {
+ let args_cloned = args.clone();
+ black_box(invoke_translate_with_args(args_cloned, size))
+ })
+ },
+ );
+
let args = create_args_scalar_from_to::<i32>(size, str_len);
group.bench_function(format!("scalar_from_to [str_len={str_len}]"), |b| {
b.iter(|| {
diff --git a/datafusion/functions/src/unicode/translate.rs b/datafusion/functions/src/unicode/translate.rs
index 29dc660b86..85e83897f4 100644
--- a/datafusion/functions/src/unicode/translate.rs
+++ b/datafusion/functions/src/unicode/translate.rs
@@ -15,13 +15,16 @@
// specific language governing permissions and limitations
// under the License.
-use arrow::array::{
- ArrayAccessor, ArrayIter, ArrayRef, AsArray, LargeStringBuilder, StringBuilder,
- StringLikeArrayBuilder, StringViewBuilder,
-};
+use arrow::array::{Array, ArrayRef, AsArray, GenericStringArray, StringArrayType};
+use arrow::buffer::NullBuffer;
use arrow::datatypes::DataType;
use datafusion_common::HashMap;
+use super::common::try_as_scalar_str;
+use crate::strings::{
+ BulkNullStringArrayBuilder, GenericStringArrayBuilder, StringViewArrayBuilder,
+ StringWriter,
+};
use crate::utils::make_scalar_function;
use datafusion_common::{Result, exec_err};
use datafusion_expr::TypeSignature::Exact;
@@ -96,14 +99,7 @@ impl ScalarUDFImpl for TranslateFunc {
try_as_scalar_str(&args.args[1]),
try_as_scalar_str(&args.args[2]),
) {
- let to_chars: Vec<char> = to_str.chars().collect();
-
- let mut from_map: HashMap<char, usize> = HashMap::new();
- for (index, c) in from_str.chars().enumerate() {
- from_map.entry(c).or_insert(index);
- }
-
- let ascii_table = build_ascii_translate_table(from_str, to_str);
+ let table = build_translate_table(from_str, to_str);
let string_array = args.args[0].to_array_of_size(args.number_rows)?;
let len = string_array.len();
@@ -111,38 +107,24 @@ impl ScalarUDFImpl for TranslateFunc {
let result = match string_array.data_type() {
DataType::Utf8View => {
let arr = string_array.as_string_view();
- let builder = StringViewBuilder::with_capacity(len);
- translate_with_map(
- arr,
- &from_map,
- &to_chars,
- ascii_table.as_ref(),
- builder,
- )
+ let builder = StringViewArrayBuilder::with_capacity(len);
+ translate_with_table(&arr, &table, builder)
}
DataType::Utf8 => {
let arr = string_array.as_string::<i32>();
- let builder =
- StringBuilder::with_capacity(len, arr.value_data().len());
- translate_with_map(
- arr,
- &from_map,
- &to_chars,
- ascii_table.as_ref(),
- builder,
- )
+ let builder = GenericStringArrayBuilder::<i32>::with_capacity(
+ len,
+ arr.value_data().len(),
+ );
+ translate_with_table(&arr, &table, builder)
}
DataType::LargeUtf8 => {
let arr = string_array.as_string::<i64>();
- let builder =
- LargeStringBuilder::with_capacity(len, arr.value_data().len());
- translate_with_map(
- arr,
- &from_map,
- &to_chars,
- ascii_table.as_ref(),
- builder,
- )
+ let builder = GenericStringArrayBuilder::<i64>::with_capacity(
+ len,
+ arr.value_data().len(),
+ );
+ translate_with_table(&arr, &table, builder)
}
other => {
return exec_err!(
@@ -162,8 +144,6 @@ impl ScalarUDFImpl for TranslateFunc {
}
}
-use super::common::try_as_scalar_str;
-
fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {
let len = args[0].len();
match args[0].data_type() {
@@ -171,24 +151,28 @@ fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {
let string_array = args[0].as_string_view();
let from_array = args[1].as_string::<i32>();
let to_array = args[2].as_string::<i32>();
- let builder = StringViewBuilder::with_capacity(len);
- translate(string_array, from_array, to_array, builder)
+ let builder = StringViewArrayBuilder::with_capacity(len);
+ translate(&string_array, from_array, to_array, builder)
}
DataType::Utf8 => {
let string_array = args[0].as_string::<i32>();
let from_array = args[1].as_string::<i32>();
let to_array = args[2].as_string::<i32>();
- let builder =
- StringBuilder::with_capacity(len, string_array.value_data().len());
- translate(string_array, from_array, to_array, builder)
+ let builder = GenericStringArrayBuilder::<i32>::with_capacity(
+ len,
+ string_array.value_data().len(),
+ );
+ translate(&string_array, from_array, to_array, builder)
}
DataType::LargeUtf8 => {
let string_array = args[0].as_string::<i64>();
let from_array = args[1].as_string::<i32>();
let to_array = args[2].as_string::<i32>();
- let builder =
- LargeStringBuilder::with_capacity(len, string_array.value_data().len());
- translate(string_array, from_array, to_array, builder)
+ let builder = GenericStringArrayBuilder::<i64>::with_capacity(
+ len,
+ string_array.value_data().len(),
+ );
+ translate(&string_array, from_array, to_array, builder)
}
other => {
exec_err!("Unsupported data type {other:?} for function translate")
@@ -196,69 +180,89 @@ fn invoke_translate(args: &[ArrayRef]) -> Result<ArrayRef> {
}
}
-/// Replaces each character in string that matches a character in the from set with the corresponding character in the to set. If from is longer than to, occurrences of the extra characters in from are deleted.
+/// Replaces each character in string that matches a character in the from set
+/// with the corresponding character in the to set. If from is longer than to,
+/// occurrences of the extra characters in from are deleted.
+///
/// translate('12345', '143', 'ax') = 'a2x5'
-fn translate<'a, V, B, O>(
- string_array: V,
- from_array: B,
- to_array: B,
+fn translate<'a, S, O>(
+ string_array: &S,
+ from_array: &GenericStringArray<i32>,
+ to_array: &GenericStringArray<i32>,
mut builder: O,
) -> Result<ArrayRef>
where
- V: ArrayAccessor<Item = &'a str>,
- B: ArrayAccessor<Item = &'a str>,
- O: StringLikeArrayBuilder,
+ S: StringArrayType<'a>,
+ O: BulkNullStringArrayBuilder,
{
- let string_array_iter = ArrayIter::new(string_array);
- let from_array_iter = ArrayIter::new(from_array);
- let to_array_iter = ArrayIter::new(to_array);
-
- let mut from_map: HashMap<char, usize> = HashMap::new();
- let mut to_chars: Vec<char> = Vec::new();
- let mut result_buf = String::new();
-
- for ((string, from), to) in string_array_iter.zip(from_array_iter).zip(to_array_iter)
- {
- match (string, from, to) {
- (Some(string), Some(from), Some(to)) => {
- from_map.clear();
- to_chars.clear();
- result_buf.clear();
-
- for (index, c) in from.chars().enumerate() {
- from_map.entry(c).or_insert(index);
- }
+ let mut from_map: HashMap<char, Option<char>> = HashMap::new();
+ let len = string_array.len();
+ let nulls = NullBuffer::union_many([
+ string_array.nulls(),
+ from_array.nulls(),
+ to_array.nulls(),
+ ]);
+
+ if let Some(nulls_ref) = nulls.as_ref() {
+ for i in 0..len {
+ if nulls_ref.is_null(i) {
+ builder.append_placeholder();
+ continue;
+ }
- to_chars.extend(to.chars());
+ // SAFETY: union of input nulls is non-null at i, so each input is too.
+ let string = unsafe { string_array.value_unchecked(i) };
+ let from = unsafe { from_array.value_unchecked(i) };
+ let to = unsafe { to_array.value_unchecked(i) };
+ append_translated_row(&mut builder, string, from, to, &mut from_map);
+ }
+ } else {
+ for i in 0..len {
+ // SAFETY: i < len, and no input has a null buffer.
+ let string = unsafe { string_array.value_unchecked(i) };
+ let from = unsafe { from_array.value_unchecked(i) };
+ let to = unsafe { to_array.value_unchecked(i) };
+ append_translated_row(&mut builder, string, from, to, &mut from_map);
+ }
+ }
- translate_char_by_char(string, &from_map, &to_chars, &mut result_buf);
+ builder.finish(nulls)
+}
- builder.append_value(&result_buf);
- }
- _ => builder.append_null(),
- }
+#[inline]
+fn append_translated_row<B: BulkNullStringArrayBuilder>(
+ builder: &mut B,
+ string: &str,
+ from: &str,
+ to: &str,
+ from_map: &mut HashMap<char, Option<char>>,
+) {
+ if let Some(ascii_table) = build_ascii_translate_table(from, to) {
+ append_translated_ascii(builder, string, &ascii_table);
+ return;
+ }
+
+ from_map.clear();
+ let mut to_iter = to.chars();
+ for c in from.chars() {
+ let replacement = to_iter.next();
+ from_map.entry(c).or_insert(replacement);
}
- Ok(builder.finish())
+ builder.append_with(|w| write_translated_chars(w, string, from_map));
}
-/// Translate `input` character-by-character using `from_map` and `to_chars`,
-/// appending the result to `buf`.
#[inline]
-fn translate_char_by_char(
+fn write_translated_chars<W: StringWriter>(
+ w: &mut W,
input: &str,
- from_map: &HashMap<char, usize>,
- to_chars: &[char],
- buf: &mut String,
+ from_map: &HashMap<char, Option<char>>,
) {
for c in input.chars() {
match from_map.get(&c) {
- Some(n) => {
- if let Some(&replacement) = to_chars.get(*n) {
- buf.push(replacement);
- }
- }
- None => buf.push(c),
+ Some(Some(r)) => w.write_char(*r),
+ Some(None) => {} // delete: `from` had no corresponding `to` char
+ None => w.write_char(c),
}
}
}
@@ -268,86 +272,170 @@ fn translate_char_by_char(
/// value > 127 works since valid ASCII is 0–127.
const ASCII_DELETE: u8 = 0xFF;
-/// If `from` and `to` are both ASCII, build a fixed-size lookup table for
-/// translation. Each entry maps an input byte to its replacement byte, or to
-/// [`ASCII_DELETE`] if the character should be removed. Returns `None` if
-/// either string contains non-ASCII characters.
-fn build_ascii_translate_table(from: &str, to: &str) -> Option<[u8; 128]> {
+/// Lookup table for ASCII-only translation. Entries 0..128 map input bytes to
+/// replacement bytes, or `ASCII_DELETE` if the character should be deleted.
+/// Entries 128..256 map to themselves so non-ASCII bytes pass through
+/// unchanged.
+#[derive(Debug)]
+struct AsciiTranslateTable {
+ map: [u8; 256],
+ has_delete: bool,
+}
+
+/// We use a byte-indexed table when both `from` and `to` strings are ASCII,
+/// otherwise a char-indexed map where `None` means delete.
+#[expect(
+ clippy::large_enum_variant,
+ reason = "one instance per call, passed by reference"
+)]
+enum TranslateTable {
+ Byte(AsciiTranslateTable),
+ Char(HashMap<char, Option<char>>),
+}
+
+#[inline]
+fn build_translate_table(from: &str, to: &str) -> TranslateTable {
+ if let Some(ascii) = build_ascii_translate_table(from, to) {
+ return TranslateTable::Byte(ascii);
+ }
+ let mut from_map: HashMap<char, Option<char>> = HashMap::with_capacity(from.len());
+ let mut to_iter = to.chars();
+ for c in from.chars() {
+ let replacement = to_iter.next();
+ from_map.entry(c).or_insert(replacement);
+ }
+ TranslateTable::Char(from_map)
+}
+
+/// Returns `None` if either string contains non-ASCII characters.
+fn build_ascii_translate_table(from: &str, to: &str) -> Option<AsciiTranslateTable> {
if !from.is_ascii() || !to.is_ascii() {
return None;
}
- let mut table = [0u8; 128];
- for i in 0..128u8 {
- table[i as usize] = i;
- }
+
let to_bytes = to.as_bytes();
+ let mut map = std::array::from_fn::<u8, 256, _>(|i| i as u8);
let mut seen = [false; 128];
+ let mut has_delete = false;
+
for (i, from_byte) in from.bytes().enumerate() {
let idx = from_byte as usize;
if !seen[idx] {
seen[idx] = true;
if i < to_bytes.len() {
- table[idx] = to_bytes[i];
+ map[idx] = to_bytes[i];
} else {
- table[idx] = ASCII_DELETE;
+ map[idx] = ASCII_DELETE;
+ has_delete = true;
}
}
}
- Some(table)
+
+ Some(AsciiTranslateTable { map, has_delete })
+}
+
+#[inline]
+fn append_translated_ascii<B: BulkNullStringArrayBuilder>(
+ builder: &mut B,
+ input: &str,
+ table: &AsciiTranslateTable,
+) {
+ // Fast path: equal-length byte-to-byte map when no deletions.
+ if !table.has_delete {
+ // SAFETY: ASCII source bytes map to ASCII replacements; non-ASCII
+ // bytes 128..256 map to themselves, so multi-byte UTF-8 sequences
+ // pass through unchanged. Output length equals input length and
+ // remains valid UTF-8.
+ unsafe {
+ builder.append_byte_map(input.as_bytes(), |b| table.map[b as usize]);
+ }
+ } else {
+ builder.append_with(|w| write_translated_ascii(w, input, table));
+ }
}
-/// Optimized translate for constant `from` and `to` arguments: uses a pre-built
-/// translation map instead of rebuilding it for every row. When an ASCII byte
-/// lookup table is provided, ASCII input rows use the lookup table; non-ASCII
-/// inputs fall back to the char-based map.
-fn translate_with_map<'a, V, O>(
- string_array: V,
- from_map: &HashMap<char, usize>,
- to_chars: &[char],
- ascii_table: Option<&[u8; 128]>,
+#[inline]
+fn write_translated_ascii<W: StringWriter>(
+ w: &mut W,
+ input: &str,
+ table: &AsciiTranslateTable,
+) {
+ let bytes = input.as_bytes();
+ let mut copy_start = 0;
+
+ for (i, &b) in bytes.iter().enumerate() {
+ let mapped = table.map[b as usize];
+ if mapped == b {
+ continue;
+ }
+
+ if copy_start < i {
+ w.write_str(&input[copy_start..i]);
+ }
+ if mapped != ASCII_DELETE {
+ w.write_char(mapped as char);
+ }
+ copy_start = i + 1;
+ }
+
+ if copy_start < input.len() {
+ w.write_str(&input[copy_start..]);
+ }
+}
+
+fn translate_with_table<'a, S, O>(
+ string_array: &S,
+ table: &TranslateTable,
mut builder: O,
) -> Result<ArrayRef>
where
- V: ArrayAccessor<Item = &'a str>,
- O: StringLikeArrayBuilder,
+ S: StringArrayType<'a>,
+ O: BulkNullStringArrayBuilder,
{
- let mut result_buf = String::new();
- let mut ascii_buf: Vec<u8> = Vec::new();
-
- for string in ArrayIter::new(string_array) {
- match string {
- Some(s) => {
- // Fast path: byte-level table lookup for ASCII strings
- if let Some(table) = ascii_table
- && s.is_ascii()
- {
- ascii_buf.clear();
- for &b in s.as_bytes() {
- let mapped = table[b as usize];
- if mapped != ASCII_DELETE {
- ascii_buf.push(mapped);
- }
- }
- // SAFETY: all bytes are ASCII, hence valid UTF-8.
- builder.append_value(unsafe {
- std::str::from_utf8_unchecked(&ascii_buf)
- });
- } else {
- result_buf.clear();
- translate_char_by_char(s, from_map, to_chars, &mut result_buf);
- builder.append_value(&result_buf);
- }
+ let len = string_array.len();
+ let nulls = string_array.nulls().cloned();
+
+ if let Some(nulls_ref) = nulls.as_ref() {
+ for i in 0..len {
+ if nulls_ref.is_null(i) {
+ builder.append_placeholder();
+ continue;
}
- None => builder.append_null(),
+
+ // SAFETY: input null buffer is non-null at i.
+ let s = unsafe { string_array.value_unchecked(i) };
+ apply_translate_table(&mut builder, s, table);
+ }
+ } else {
+ for i in 0..len {
+ // SAFETY: no null buffer means every index is valid.
+ let s = unsafe { string_array.value_unchecked(i) };
+ apply_translate_table(&mut builder, s, table);
}
}
- Ok(builder.finish())
+ builder.finish(nulls)
+}
+
+#[inline]
+fn apply_translate_table<B: BulkNullStringArrayBuilder>(
+ builder: &mut B,
+ input: &str,
+ table: &TranslateTable,
+) {
+ match table {
+ TranslateTable::Byte(t) => append_translated_ascii(builder, input, t),
+ TranslateTable::Char(m) => {
+ builder.append_with(|w| write_translated_chars(w, input, m))
+ }
+ }
}
#[cfg(test)]
mod tests {
- use arrow::array::{Array, StringArray, StringViewArray};
+ use std::sync::Arc;
+
+ use arrow::array::{Array, ArrayRef, StringArray, StringViewArray};
use arrow::datatypes::DataType::{Utf8, Utf8View};
use datafusion_common::{Result, ScalarValue};
@@ -430,8 +518,7 @@ mod tests {
Utf8,
StringArray
);
- // Non-ASCII input with ASCII scalar from/to: exercises the
- // char-based fallback within translate_with_map.
+ // Non-ASCII input with ASCII scalar from/to.
test_function!(
TranslateFunc::new(),
vec![
@@ -502,4 +589,27 @@ mod tests {
Ok(())
}
+
+ #[test]
+ fn test_array_args_with_nulls() -> Result<()> {
+ let string_array = Arc::new(StringArray::from(vec![
+ Some("café!"),
+ Some("abc"),
+ Some("abc"),
+ ])) as ArrayRef;
+ let from_array =
+ Arc::new(StringArray::from(vec![Some("!"), Some("a"), None])) as ArrayRef;
+ let to_array =
+ Arc::new(StringArray::from(vec![Some(""), Some("x"), Some("y")])) as ArrayRef;
+
+ let result = super::invoke_translate(&[string_array, from_array, to_array])?;
+ let result = result.as_any().downcast_ref::<StringArray>().unwrap();
+
+ assert_eq!(result.len(), 3);
+ assert_eq!(result.value(0), "café");
+ assert_eq!(result.value(1), "xbc");
+ assert!(result.is_null(2));
+
+ Ok(())
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org