You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/15 05:51:43 UTC
(datafusion) branch gh-readonly-queue/main/pr-23330-fc931625210c601c65f649b0bebdbc9e1cdda377 created (now 7f6cc60e55)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23330-fc931625210c601c65f649b0bebdbc9e1cdda377
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 7f6cc60e55 Preserve string slice function return types (#23330)
This branch includes the following new commits:
new 7f6cc60e55 Preserve string slice function return types (#23330)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: Preserve string slice function return types (#23330)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23330-fc931625210c601c65f649b0bebdbc9e1cdda377
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 7f6cc60e55075615899bf317588ba7f6e495977b
Author: xudong.w <wx...@gmail.com>
AuthorDate: Wed Jul 15 13:51:22 2026 +0800
Preserve string slice function return types (#23330)
## Which issue does this PR close?
- Closes #23329.
## Rationale for this change
`left`, `right`, and `substr` currently always return `Utf8View`, even
when their first argument is `Utf8` or `LargeUtf8`. This makes them
inconsistent with nearby string-producing functions such as `lower`,
`upper`, `reverse`, `translate`, and `substr_index`, which preserve the
first argument's string type.
The unconditional `Utf8View` return type also means callers that
intentionally avoid view types cannot keep `Utf8` flowing through these
functions without explicit casts back to `Utf8`, even when they
explicitly opt out of view types with
`datafusion.sql_parser.map_string_types_to_utf8view=false` and
`datafusion.execution.parquet.schema_force_view_types=false`.
This PR revisits the performance tradeoff from #21441 / #21442 and makes
these functions follow the same input-driven return-type contract as the
other string functions.
## What changes are included in this PR?
- Change `left`, `right`, and `substr` return type inference to use the
first argument's string type.
- Return `StringArray` / `LargeStringArray` for `Utf8` / `LargeUtf8`
inputs.
- Keep the optimized `StringViewArray` path for `Utf8View` inputs.
- Update unit tests and sqllogictests to cover `Utf8`, `LargeUtf8`, and
`Utf8View` return types.
## Are these changes tested?
Yes.
## Are there any user-facing changes?
Yes. `left`, `right`, and `substr` now preserve the first argument's
string type:
- `Utf8 -> Utf8`
- `LargeUtf8 -> LargeUtf8`
- `Utf8View -> Utf8View`
This may remove implicit downstream casts for users that expect `Utf8`,
but it also means `Utf8` / `LargeUtf8` inputs no longer use the previous
zero-copy `Utf8View` result representation.
---
datafusion/functions/src/unicode/common.rs | 77 +------
datafusion/functions/src/unicode/left.rs | 67 +++---
datafusion/functions/src/unicode/right.rs | 67 +++---
datafusion/functions/src/unicode/substr.rs | 244 +++++++--------------
.../test_files/string/string_query.slt.part | 25 ++-
5 files changed, 189 insertions(+), 291 deletions(-)
diff --git a/datafusion/functions/src/unicode/common.rs b/datafusion/functions/src/unicode/common.rs
index 092f2b8003..5b7262e29d 100644
--- a/datafusion/functions/src/unicode/common.rs
+++ b/datafusion/functions/src/unicode/common.rs
@@ -151,79 +151,20 @@ pub(crate) fn general_left_right<F: LeftRightSlicer>(
}
}
-/// Returns true if all offsets in the array fit in i32, meaning the values
-/// buffer can be referenced by StringView's offset field.
-fn values_fit_in_i32<T: OffsetSizeTrait>(string_array: &GenericStringArray<T>) -> bool {
- string_array
- .offsets()
- .last()
- .map(|offset| offset.as_usize() <= i32::MAX as usize)
- .unwrap_or(true)
-}
-
/// `left`/`right` for Utf8/LargeUtf8 input.
-///
-/// When offsets fit in i32, produces a zero-copy `StringViewArray` with views
-/// pointing into the input values buffer. Otherwise falls back to building a
-/// `StringViewArray` by copying.
fn general_left_right_array<T: OffsetSizeTrait, F: LeftRightSlicer>(
string_array: &GenericStringArray<T>,
n_array: &Int64Array,
) -> Result<ArrayRef> {
- if !values_fit_in_i32(string_array) {
- let result = string_array
- .iter()
- .zip(n_array.iter())
- .map(|(string, n)| match (string, n) {
- (Some(string), Some(n)) => Some(&string[F::slice(string, n)]),
- _ => None,
- })
- .collect::<StringViewArray>();
- return Ok(Arc::new(result) as ArrayRef);
- }
-
- let len = string_array.len();
- let offsets = string_array.value_offsets();
- let nulls = NullBuffer::union(string_array.nulls(), n_array.nulls());
-
- let mut views_buf = Vec::with_capacity(len);
- let mut has_out_of_line = false;
-
- for (i, offset) in offsets.iter().enumerate().take(len) {
- if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
- views_buf.push(0);
- continue;
- }
-
- // SAFETY: we just checked validity above
- let string = unsafe { string_array.value_unchecked(i) };
- let n = n_array.value(i);
- let range = F::slice(string, n);
- let result_bytes = &string.as_bytes()[range.clone()];
- if result_bytes.len() > 12 {
- has_out_of_line = true;
- }
-
- let buf_offset = offset.as_usize() as u32 + range.start as u32;
- views_buf.push(make_view(result_bytes, 0, buf_offset));
- }
-
- let views = ScalarBuffer::from(views_buf);
- let data_buffers = if has_out_of_line {
- vec![string_array.values().clone()]
- } else {
- vec![]
- };
-
- // SAFETY:
- // - Each view is produced by `make_view` with correct bytes and offset
- // - Out-of-line views reference buffer index 0, which is the original
- // values buffer included in data_buffers when has_out_of_line is true
- // - values_fit_in_i32 guarantees all offsets fit in i32
- unsafe {
- let array = StringViewArray::new_unchecked(views, data_buffers, nulls);
- Ok(Arc::new(array) as ArrayRef)
- }
+ let result = string_array
+ .iter()
+ .zip(n_array.iter())
+ .map(|(string, n)| match (string, n) {
+ (Some(string), Some(n)) => Some(&string[F::slice(string, n)]),
+ _ => None,
+ })
+ .collect::<GenericStringArray<T>>();
+ Ok(Arc::new(result) as ArrayRef)
}
/// `general_left_right` for StringViewArray input.
diff --git a/datafusion/functions/src/unicode/left.rs b/datafusion/functions/src/unicode/left.rs
index 423ab4d5dc..0788e69d92 100644
--- a/datafusion/functions/src/unicode/left.rs
+++ b/datafusion/functions/src/unicode/left.rs
@@ -79,8 +79,8 @@ impl ScalarUDFImpl for LeftFunc {
&self.signature
}
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Utf8View)
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
}
/// Returns first n characters in the string, or when n is negative, returns all but last |n| characters.
@@ -108,8 +108,8 @@ impl ScalarUDFImpl for LeftFunc {
#[cfg(test)]
mod tests {
- use arrow::array::{Array, StringViewArray};
- use arrow::datatypes::DataType::Utf8View;
+ use arrow::array::{Array, LargeStringArray, StringArray, StringViewArray};
+ use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
@@ -127,8 +127,19 @@ mod tests {
],
Ok(Some("ab")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
+ );
+ test_function!(
+ LeftFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("abcde".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::from(2i64)),
+ ],
+ Ok(Some("ab")),
+ &str,
+ LargeUtf8,
+ LargeStringArray
);
test_function!(
LeftFunc::new(),
@@ -138,8 +149,8 @@ mod tests {
],
Ok(Some("abcde")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -149,8 +160,8 @@ mod tests {
],
Ok(Some("abc")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -160,8 +171,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -171,8 +182,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -182,8 +193,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -193,8 +204,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -204,8 +215,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -215,8 +226,8 @@ mod tests {
],
Ok(Some("joséé")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
LeftFunc::new(),
@@ -226,8 +237,8 @@ mod tests {
],
Ok(Some("joséé")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
#[cfg(not(feature = "unicode_expressions"))]
test_function!(
@@ -240,8 +251,8 @@ mod tests {
"function left requires compilation with feature flag: unicode_expressions."
),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
// StringView cases
@@ -307,8 +318,8 @@ mod tests {
],
Ok(Some(expected.as_str())),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
}
diff --git a/datafusion/functions/src/unicode/right.rs b/datafusion/functions/src/unicode/right.rs
index 0ed170fef7..21fb0690a1 100644
--- a/datafusion/functions/src/unicode/right.rs
+++ b/datafusion/functions/src/unicode/right.rs
@@ -79,8 +79,8 @@ impl ScalarUDFImpl for RightFunc {
&self.signature
}
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Utf8View)
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
}
/// Returns right n characters in the string, or when n is negative, returns all but first |n| characters.
@@ -108,8 +108,8 @@ impl ScalarUDFImpl for RightFunc {
#[cfg(test)]
mod tests {
- use arrow::array::{Array, StringViewArray};
- use arrow::datatypes::DataType::Utf8View;
+ use arrow::array::{Array, LargeStringArray, StringArray, StringViewArray};
+ use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
@@ -127,8 +127,19 @@ mod tests {
],
Ok(Some("de")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
+ );
+ test_function!(
+ RightFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some("abcde".to_string()))),
+ ColumnarValue::Scalar(ScalarValue::from(2i64)),
+ ],
+ Ok(Some("de")),
+ &str,
+ LargeUtf8,
+ LargeStringArray
);
test_function!(
RightFunc::new(),
@@ -138,8 +149,8 @@ mod tests {
],
Ok(Some("abcde")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -149,8 +160,8 @@ mod tests {
],
Ok(Some("cde")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -160,8 +171,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -171,8 +182,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -182,8 +193,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -193,8 +204,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -204,8 +215,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -215,8 +226,8 @@ mod tests {
],
Ok(Some("érend")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
RightFunc::new(),
@@ -226,8 +237,8 @@ mod tests {
],
Ok(Some("éérend")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
#[cfg(not(feature = "unicode_expressions"))]
test_function!(
@@ -240,8 +251,8 @@ mod tests {
"function right requires compilation with feature flag: unicode_expressions."
),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
// StringView cases
@@ -304,8 +315,8 @@ mod tests {
],
Ok(Some(expected.as_str())),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
}
diff --git a/datafusion/functions/src/unicode/substr.rs b/datafusion/functions/src/unicode/substr.rs
index 903c03857e..0cae215224 100644
--- a/datafusion/functions/src/unicode/substr.rs
+++ b/datafusion/functions/src/unicode/substr.rs
@@ -17,11 +17,11 @@
use std::sync::Arc;
-use crate::strings::{StringViewArrayBuilder, append_view};
+use crate::strings::append_view;
use crate::utils::make_scalar_function;
use arrow::array::{
Array, ArrayRef, AsArray, GenericStringArray, Int64Array, OffsetSizeTrait,
- StringArrayType, StringViewArray, make_view,
+ StringArrayType, StringViewArray,
};
use arrow::buffer::{NullBuffer, ScalarBuffer};
use arrow::datatypes::DataType;
@@ -111,9 +111,8 @@ impl ScalarUDFImpl for SubstrFunc {
&self.signature
}
- // `SubstrFunc` always generates `Utf8View` output for its efficiency.
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- Ok(DataType::Utf8View)
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
@@ -319,131 +318,37 @@ fn string_view_substr(
}
}
-fn values_fit_in_i32<T: OffsetSizeTrait>(string_array: &GenericStringArray<T>) -> bool {
- // The Arrow spec defines StringView offset fields as signed 32-bit
- // integers, so the maximum representable offset is i32::MAX.
- string_array
- .offsets()
- .last()
- .map(|offset| offset.as_usize() <= i32::MAX as usize)
- .unwrap_or(true)
-}
-
-#[inline]
-fn append_view_from_buffer(
- views_buf: &mut Vec<u128>,
- substr: &str,
- byte_offset: usize,
-) -> bool {
- let byte_offset =
- u32::try_from(byte_offset).expect("validated string buffer offset fits in i32");
- let view = make_view(substr.as_bytes(), 0, byte_offset);
- views_buf.push(view);
- substr.len() > 12
-}
-
-#[expect(clippy::needless_range_loop)]
fn generic_string_substr<T: OffsetSizeTrait>(
string_array: &GenericStringArray<T>,
args: &[ArrayRef],
) -> Result<ArrayRef> {
- // We'd like to return a StringViewArray that points into the input string
- // array's values buffer. Since the Arrow spec defines StringView offsets
- // as i32, we can't use this approach when the values buffer is >2GB, so
- // fallback to copying.
- if !values_fit_in_i32(string_array) {
- return generic_string_substr_copy(string_array, args);
- }
-
let start_array = as_int64_array(&args[0])?;
let count_array_opt = args.get(1).map(|a| as_int64_array(a)).transpose()?;
let is_ascii = enable_ascii_fast_path(&string_array, start_array, count_array_opt);
- let offsets = string_array.value_offsets();
- let mut views_buf = Vec::with_capacity(string_array.len());
- let mut has_out_of_line = false;
-
- // Combine null bitmaps from all inputs in bulk.
let nulls = NullBuffer::union_many([
string_array.nulls(),
start_array.nulls(),
count_array_opt.and_then(|a| a.nulls()),
]);
- for i in 0..string_array.len() {
- if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
- views_buf.push(0);
- continue;
- }
-
- let string = string_array.value(i);
- let source_offset = offsets[i].as_usize();
- let start = start_array.value(i);
- let count = count_array_opt.map(|a| a.value(i));
-
- let (byte_start, byte_end) = get_true_start_end(string, start, count, is_ascii)?;
- has_out_of_line |= append_view_from_buffer(
- &mut views_buf,
- &string[byte_start..byte_end],
- source_offset + byte_start,
- );
- }
+ let result = (0..string_array.len())
+ .map(|i| {
+ if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
+ return Ok(None);
+ }
- let views_buf = ScalarBuffer::from(views_buf);
+ let string = string_array.value(i);
+ let start = start_array.value(i);
+ let count = count_array_opt.map(|a| a.value(i));
- // If all result strings are stored inline, we don't need to retain the
- // input string array.
- let data_buffers = if has_out_of_line {
- vec![string_array.values().clone()]
- } else {
- vec![]
- };
+ let (byte_start, byte_end) =
+ get_true_start_end(string, start, count, is_ascii)?;
+ Ok(Some(&string[byte_start..byte_end]))
+ })
+ .collect::<Result<GenericStringArray<T>>>()?;
- // Safety:
- // (1) The blocks of the given views are all provided
- // (2) Each referenced range in the source values buffer is within bounds
- unsafe {
- let array = StringViewArray::new_unchecked(views_buf, data_buffers, nulls);
- Ok(Arc::new(array) as ArrayRef)
- }
-}
-
-// Fallback for `generic_string_substr` if we can't use zerocopy because the
-// input string array is too large.
-fn generic_string_substr_copy<T: OffsetSizeTrait>(
- string_array: &GenericStringArray<T>,
- args: &[ArrayRef],
-) -> Result<ArrayRef> {
- let start_array = as_int64_array(&args[0])?;
- let count_array_opt = args.get(1).map(|a| as_int64_array(a)).transpose()?;
-
- let is_ascii = enable_ascii_fast_path(&string_array, start_array, count_array_opt);
-
- // Combine null bitmaps from all inputs in bulk.
- let nulls = NullBuffer::union_many([
- string_array.nulls(),
- start_array.nulls(),
- count_array_opt.and_then(|a| a.nulls()),
- ]);
-
- let len = string_array.len();
- let mut result_builder = StringViewArrayBuilder::with_capacity(len);
-
- for i in 0..len {
- if nulls.as_ref().is_some_and(|n| n.is_null(i)) {
- result_builder.append_placeholder();
- continue;
- }
-
- let string = string_array.value(i);
- let start = start_array.value(i);
- let count = count_array_opt.map(|a| a.value(i));
-
- let (byte_start, byte_end) = get_true_start_end(string, start, count, is_ascii)?;
- result_builder.append_value(&string[byte_start..byte_end]);
- }
-
- Ok(Arc::new(result_builder.finish(nulls)?) as ArrayRef)
+ Ok(Arc::new(result) as ArrayRef)
}
#[cfg(test)]
@@ -451,9 +356,10 @@ mod tests {
use std::sync::Arc;
use arrow::array::{
- Array, ArrayRef, AsArray, Int64Array, StringArray, StringViewArray,
+ Array, ArrayRef, AsArray, Int64Array, LargeStringArray, StringArray,
+ StringViewArray,
};
- use arrow::datatypes::DataType::Utf8View;
+ use arrow::datatypes::DataType::{LargeUtf8, Utf8, Utf8View};
use datafusion_common::{Result, ScalarValue, exec_err};
use datafusion_expr::{ColumnarValue, ScalarUDFImpl};
@@ -563,8 +469,21 @@ mod tests {
],
Ok(Some("alphabet")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
+ );
+ test_function!(
+ SubstrFunc::new(),
+ vec![
+ ColumnarValue::Scalar(ScalarValue::LargeUtf8(Some(
+ "alphabet".to_string()
+ ))),
+ ColumnarValue::Scalar(ScalarValue::from(0i64)),
+ ],
+ Ok(Some("alphabet")),
+ &str,
+ LargeUtf8,
+ LargeStringArray
);
test_function!(
SubstrFunc::new(),
@@ -574,8 +493,8 @@ mod tests {
],
Ok(Some("ésoj")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -585,8 +504,8 @@ mod tests {
],
Ok(Some("joséésoj")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -596,8 +515,8 @@ mod tests {
],
Ok(Some("alphabet")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -607,8 +526,8 @@ mod tests {
],
Ok(Some("lphabet")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -618,8 +537,8 @@ mod tests {
],
Ok(Some("phabet")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -629,8 +548,8 @@ mod tests {
],
Ok(Some("alphabet")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -640,8 +559,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -651,8 +570,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -663,8 +582,8 @@ mod tests {
],
Ok(Some("ph")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -675,8 +594,8 @@ mod tests {
],
Ok(Some("phabet")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -687,8 +606,8 @@ mod tests {
],
Ok(Some("alph")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
// starting from 5 (10 + -5)
test_function!(
@@ -700,8 +619,8 @@ mod tests {
],
Ok(Some("alph")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
// starting from -1 (4 + -5)
test_function!(
@@ -713,8 +632,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
// starting from 0 (5 + -5)
test_function!(
@@ -726,8 +645,8 @@ mod tests {
],
Ok(Some("")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -738,8 +657,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -750,8 +669,8 @@ mod tests {
],
Ok(None),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -762,8 +681,8 @@ mod tests {
],
exec_err!("negative count not allowed: -1"),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -774,8 +693,8 @@ mod tests {
],
Ok(Some("és")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
#[cfg(not(feature = "unicode_expressions"))]
test_function!(
@@ -788,8 +707,8 @@ mod tests {
"function substr requires compilation with feature flag: unicode_expressions."
),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -799,8 +718,8 @@ mod tests {
],
exec_err!("start position overflow: -9223372036854775808"),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -811,8 +730,8 @@ mod tests {
],
exec_err!("start position overflow: -9223372036854775808"),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
test_function!(
SubstrFunc::new(),
@@ -823,8 +742,8 @@ mod tests {
],
Ok(Some("arge count")),
&str,
- Utf8View,
- StringViewArray
+ Utf8,
+ StringArray
);
Ok(())
@@ -832,7 +751,6 @@ mod tests {
#[test]
fn test_sliced_string_array_array_args() -> Result<()> {
- // Use strings longer than 12 bytes so the result views are out-of-line.
let string_array = Arc::new(StringArray::from(vec![
"skipped_prefix_value",
"alphabet_long_string",
@@ -843,7 +761,7 @@ mod tests {
let count_array = Arc::new(Int64Array::from(vec![15, 14])) as ArrayRef;
let result = super::substr(&[string_array, start_array, count_array])?;
- let result = result.as_string_view();
+ let result = result.as_string::<i32>();
assert_eq!(result.value(0), "phabet_long_str");
assert_eq!(result.value(1), "ésojanother_lo");
diff --git a/datafusion/sqllogictest/test_files/string/string_query.slt.part b/datafusion/sqllogictest/test_files/string/string_query.slt.part
index dac4dd06db..9fcacbaa54 100644
--- a/datafusion/sqllogictest/test_files/string/string_query.slt.part
+++ b/datafusion/sqllogictest/test_files/string/string_query.slt.part
@@ -1860,11 +1860,28 @@ SELECT left(ascii_1, 0), right(ascii_1, 0) FROM test_basic_operator
NULL NULL
NULL NULL
-# left and right return Utf8View
-query TT
-SELECT arrow_typeof(left(ascii_1, 3)), arrow_typeof(right(ascii_1, 3)) FROM test_basic_operator LIMIT 1
+# left and right preserve the input string type
+query TTTTTT
+SELECT
+ arrow_typeof(left(arrow_cast(ascii_1, 'Utf8'), 3)),
+ arrow_typeof(right(arrow_cast(ascii_1, 'Utf8'), 3)),
+ arrow_typeof(left(arrow_cast(ascii_1, 'LargeUtf8'), 3)),
+ arrow_typeof(right(arrow_cast(ascii_1, 'LargeUtf8'), 3)),
+ arrow_typeof(left(arrow_cast(ascii_1, 'Utf8View'), 3)),
+ arrow_typeof(right(arrow_cast(ascii_1, 'Utf8View'), 3))
+FROM test_basic_operator LIMIT 1
+----
+Utf8 Utf8 LargeUtf8 LargeUtf8 Utf8View Utf8View
+
+# substr preserves the input string type
+query TTT
+SELECT
+ arrow_typeof(substr(arrow_cast(ascii_1, 'Utf8'), 1, 3)),
+ arrow_typeof(substr(arrow_cast(ascii_1, 'LargeUtf8'), 1, 3)),
+ arrow_typeof(substr(arrow_cast(ascii_1, 'Utf8View'), 1, 3))
+FROM test_basic_operator LIMIT 1
----
-Utf8View Utf8View
+Utf8 LargeUtf8 Utf8View
# --------------------------------------
# Test repeat() against array inputs with various null patterns. The scalar
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org