You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/10 15:42:39 UTC
(datafusion) 01/01: feat: support binary types in octet_length (#24867)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24867-5747e87e95f3a224b9bf98a8c157c9e069a7d24c
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 5e4224779669ace7839c60d1c014a98802f11f6b
Author: Eddie A Tejeda <ed...@users.noreply.github.com>
AuthorDate: Thu Sep 10 15:42:14 2026 +0000
feat: support binary types in octet_length (#24867)
## Which issue does this PR close?
- Closes #24765.
## Rationale for this change
`octet_length` should count the raw bytes of binary data without
expensive encoding workaround.
## What changes are included in this PR?
- Adds support for `Binary`, `LargeBinary`, and `BinaryView`.
- Returns `Int64` for `LargeBinary` and `Int32` for the other types.
- Updates the error message to say “strings or binary.”
## What is the testing strategy for this PR?
Unit and SQL logic tests cover binary arrays and scalars, non-UTF-8
data, nulls, dictionary encoding, and return types.
## Are there any user-facing changes?
Yes. `octet_length` now counts the raw bytes of binary values. Existing
string behavior is unchanged.
---------
Co-authored-by: Eddie A Tejeda <66...@users.noreply.github.com>
---
datafusion/expr-common/src/signature.rs | 3 +
datafusion/functions/src/string/octet_length.rs | 168 +++++++++++++++++++++--
datafusion/functions/src/utils.rs | 6 +-
datafusion/sqllogictest/test_files/functions.slt | 65 +++++++++
docs/source/user-guide/expressions.md | 2 +-
docs/source/user-guide/sql/scalar_functions.md | 4 +-
6 files changed, 236 insertions(+), 12 deletions(-)
diff --git a/datafusion/expr-common/src/signature.rs b/datafusion/expr-common/src/signature.rs
index 2bde7c89a8..d88cc791e9 100644
--- a/datafusion/expr-common/src/signature.rs
+++ b/datafusion/expr-common/src/signature.rs
@@ -508,6 +508,9 @@ impl TypeSignatureClass {
TypeSignatureClass::Binary if native_type.is_binary() => {
Ok(origin_type.to_owned())
}
+ // Binary has an unambiguous default, unlike Timestamp; cast NULL rather
+ // than letting the generic arm below pass DataType::Null downstream.
+ TypeSignatureClass::Binary if native_type.is_null() => Ok(DataType::Binary),
TypeSignatureClass::Decimal if native_type.is_decimal() => {
Ok(origin_type.to_owned())
}
diff --git a/datafusion/functions/src/string/octet_length.rs b/datafusion/functions/src/string/octet_length.rs
index 02df262ee2..90ea81cb70 100644
--- a/datafusion/functions/src/string/octet_length.rs
+++ b/datafusion/functions/src/string/octet_length.rs
@@ -24,13 +24,13 @@ use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,
- ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
+ ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_macros::user_doc;
#[user_doc(
doc_section(label = "String Functions"),
- description = "Returns the length of a string in bytes.",
+ description = "Returns the length of a string or binary in bytes.",
syntax_example = "octet_length(str)",
sql_example = r#"```sql
> select octet_length('Ångström');
@@ -40,7 +40,10 @@ use datafusion_macros::user_doc;
| 10 |
+--------------------------------+
```"#,
- standard_argument(name = "str", prefix = "String"),
+ argument(
+ name = "str",
+ description = "String or binary expression to operate on. Can be a constant, column, or function, and any combination of operators."
+ ),
related_udf(name = "bit_length"),
related_udf(name = "length")
)]
@@ -58,10 +61,22 @@ impl Default for OctetLengthFunc {
impl OctetLengthFunc {
pub fn new() -> Self {
Self {
- signature: Signature::coercible(
+ signature: Signature::one_of(
vec![
- Coercion::new_exact(TypeSignatureClass::Native(logical_string()))
- .with_encoding_preservation(EncodingPreservation::dictionary()),
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Native(logical_string()))
+ .with_encoding_preservation(
+ EncodingPreservation::dictionary(),
+ ),
+ ]),
+ // `TypeSignatureClass::Binary` also admits FixedSizeBinary,
+ // which `Native(logical_binary())` would reject.
+ TypeSignature::Coercible(vec![
+ Coercion::new_exact(TypeSignatureClass::Binary)
+ .with_encoding_preservation(
+ EncodingPreservation::dictionary(),
+ ),
+ ]),
],
Volatility::Immutable,
),
@@ -107,6 +122,16 @@ fn octet_length_scalar(value: &ScalarValue) -> ScalarValue {
ScalarValue::Utf8View(v) => {
ScalarValue::Int32(v.as_ref().map(|x| x.len() as i32))
}
+ ScalarValue::Binary(v) => ScalarValue::Int32(v.as_ref().map(|x| x.len() as i32)),
+ ScalarValue::LargeBinary(v) => {
+ ScalarValue::Int64(v.as_ref().map(|x| x.len() as i64))
+ }
+ ScalarValue::BinaryView(v) => {
+ ScalarValue::Int32(v.as_ref().map(|x| x.len() as i32))
+ }
+ ScalarValue::FixedSizeBinary(_, v) => {
+ ScalarValue::Int32(v.as_ref().map(|x| x.len() as i32))
+ }
ScalarValue::Dictionary(key_type, value) => ScalarValue::Dictionary(
key_type.clone(),
Box::new(octet_length_scalar(value)),
@@ -119,8 +144,11 @@ fn octet_length_scalar(value: &ScalarValue) -> ScalarValue {
mod tests {
use std::sync::Arc;
- use arrow::array::{Array, Int32Array, StringArray};
- use arrow::datatypes::DataType::Int32;
+ use arrow::array::{
+ Array, BinaryArray, BinaryViewArray, FixedSizeBinaryArray, Int32Array,
+ Int64Array, LargeBinaryArray, StringArray,
+ };
+ use arrow::datatypes::DataType::{Int32, Int64};
use datafusion_common::ScalarValue;
use datafusion_common::{Result, exec_err};
@@ -232,6 +260,130 @@ mod tests {
Int32Array
);
+ // Binary inputs: byte length, no string coercion.
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Array(Arc::new(BinaryArray::from(vec![
+ &b"chars"[..],
+ &b"chars2"[..],
+ ])))],
+ Ok(Some(5)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Binary(Some(
+ b"chars".to_vec()
+ )))],
+ Ok(Some(5)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ // Arbitrary non-UTF-8 bytes: the case CAST(col AS VARCHAR) cannot serve.
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Binary(Some(vec![
+ 0xff, 0xfe, 0x00, 0x80
+ ])))],
+ Ok(Some(4)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Binary(Some(vec![])))],
+ Ok(Some(0)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::Binary(None))],
+ Ok(None),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Array(Arc::new(BinaryViewArray::from(vec![
+ &b"chars"[..],
+ &b"chars2"[..],
+ ])))],
+ Ok(Some(5)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::BinaryView(Some(
+ b"chars".to_vec()
+ )))],
+ Ok(Some(5)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ // FixedSizeBinary reports its fixed width per non-null row.
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Array(Arc::new(
+ FixedSizeBinaryArray::try_from_iter(
+ vec![&b"abc"[..], &b"def"[..]].into_iter()
+ )?
+ ))],
+ Ok(Some(3)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::FixedSizeBinary(
+ 5,
+ Some(b"chars".to_vec())
+ ))],
+ Ok(Some(5)),
+ i32,
+ Int32,
+ Int32Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::FixedSizeBinary(5, None))],
+ Ok(None),
+ i32,
+ Int32,
+ Int32Array
+ );
+ // LargeBinary widens the return type to Int64, mirroring LargeUtf8.
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Array(Arc::new(LargeBinaryArray::from(
+ vec![&b"chars"[..], &b"chars2"[..]]
+ )))],
+ Ok(Some(5)),
+ i64,
+ Int64,
+ Int64Array
+ );
+ test_function!(
+ OctetLengthFunc::new(),
+ vec![ColumnarValue::Scalar(ScalarValue::LargeBinary(Some(
+ b"chars".to_vec()
+ )))],
+ Ok(Some(5)),
+ i64,
+ Int64,
+ Int64Array
+ );
+
Ok(())
}
}
diff --git a/datafusion/functions/src/utils.rs b/datafusion/functions/src/utils.rs
index 52ce0bd72c..8e1dcb3b91 100644
--- a/datafusion/functions/src/utils.rs
+++ b/datafusion/functions/src/utils.rs
@@ -33,7 +33,8 @@ use std::sync::Arc;
///
/// If the input type is `Utf8` or `Binary` the return type is `$utf8Type`,
///
-/// If the input type is `Utf8View` the return type is $utf8Type,
+/// If the input type is `Utf8View`, `BinaryView` or `FixedSizeBinary` the
+/// return type is `$utf8Type`,
macro_rules! get_optimal_return_type {
($FUNC:ident, $largeUtf8Type:expr, $utf8Type:expr) => {
pub(crate) fn $FUNC(arg_type: &DataType, name: &str) -> Result<DataType> {
@@ -44,10 +45,13 @@ macro_rules! get_optimal_return_type {
DataType::Utf8 | DataType::Binary => $utf8Type,
// Utf8View max offset size is u32::MAX, the same as UTF8
DataType::Utf8View | DataType::BinaryView => $utf8Type,
+ // FixedSizeBinary sizes are declared as i32
+ DataType::FixedSizeBinary(_) => $utf8Type,
DataType::Null => DataType::Null,
DataType::Dictionary(_, value_type) => match **value_type {
DataType::LargeUtf8 | DataType::LargeBinary => $largeUtf8Type,
DataType::Utf8 | DataType::Binary => $utf8Type,
+ DataType::FixedSizeBinary(_) => $utf8Type,
DataType::Null => DataType::Null,
_ => {
return datafusion_common::exec_err!(
diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt
index 008be05852..486892d8ad 100644
--- a/datafusion/sqllogictest/test_files/functions.slt
+++ b/datafusion/sqllogictest/test_files/functions.slt
@@ -918,6 +918,71 @@ ORDER BY id
2 2 Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32))
NULL NULL Dictionary(Int32, Int32) Dictionary(Int32, Dictionary(UInt32, Int32))
+# octet_length over binary types: byte semantics, no string coercion
+query I
+SELECT octet_length(arrow_cast('foo', 'Binary'))
+----
+3
+
+query IT
+SELECT octet_length(arrow_cast('josé', 'Binary')),
+ arrow_typeof(octet_length(arrow_cast('josé', 'Binary')))
+----
+5 Int32
+
+query IT
+SELECT octet_length(arrow_cast('foo', 'BinaryView')),
+ arrow_typeof(octet_length(arrow_cast('foo', 'BinaryView')))
+----
+3 Int32
+
+# LargeBinary widens the return type to Int64, mirroring LargeUtf8
+query IT
+SELECT octet_length(arrow_cast('foo', 'LargeBinary')),
+ arrow_typeof(octet_length(arrow_cast('foo', 'LargeBinary')))
+----
+3 Int64
+
+query I
+SELECT octet_length(arrow_cast(NULL, 'Binary'))
+----
+NULL
+
+# A bare NULL coerces through the string branch and keeps a typed Int32 result
+query IT
+SELECT octet_length(NULL), arrow_typeof(octet_length(NULL))
+----
+NULL Int32
+
+# A dictionary over Null coerces to a string dictionary rather than reaching
+# the arrow length kernel, which rejects Null values
+query ?T
+SELECT octet_length(arrow_cast(NULL, 'Dictionary(Int32, Null)')),
+ arrow_typeof(octet_length(arrow_cast(NULL, 'Dictionary(Int32, Null)')))
+----
+NULL Dictionary(Int32, Int32)
+
+# Dictionary encoding is preserved over binary values too
+query ?T
+SELECT octet_length(arrow_cast(arrow_cast('foo', 'Binary'), 'Dictionary(Int32, Binary)')),
+ arrow_typeof(octet_length(arrow_cast(arrow_cast('foo', 'Binary'), 'Dictionary(Int32, Binary)')))
+----
+3 Dictionary(Int32, Int32)
+
+# FixedSizeBinary reports its fixed width
+query IT
+SELECT octet_length(arrow_cast(arrow_cast('foo', 'Binary'), 'FixedSizeBinary(3)')),
+ arrow_typeof(octet_length(arrow_cast(arrow_cast('foo', 'Binary'), 'FixedSizeBinary(3)')))
+----
+3 Int32
+
+# Dictionary encoding is preserved over FixedSizeBinary as well
+query ?T
+SELECT octet_length(arrow_cast(arrow_cast(arrow_cast('foo', 'Binary'), 'FixedSizeBinary(3)'), 'Dictionary(Int32, FixedSizeBinary(3))')),
+ arrow_typeof(octet_length(arrow_cast(arrow_cast(arrow_cast('foo', 'Binary'), 'FixedSizeBinary(3)'), 'Dictionary(Int32, FixedSizeBinary(3))')))
+----
+3 Dictionary(Int32, Int32)
+
query ??TT
SELECT character_length(dict_col), character_length(nested_dict_col),
arrow_typeof(character_length(dict_col)),
diff --git a/docs/source/user-guide/expressions.md b/docs/source/user-guide/expressions.md
index 3fbc11e0c9..55f64135dd 100644
--- a/docs/source/user-guide/expressions.md
+++ b/docs/source/user-guide/expressions.md
@@ -191,7 +191,7 @@ select log(-1), log(0), sqrt(-1);
| lpad(text, length, [, fill]) | Extends the string to length (`length`) by prepending the characters (`fill`) (a space by default). Example: `lpad('bb', 5, 'a') → aaabb` |
| ltrim(text, text) | Removes all specified characters (`characters`) from the beginning of the string (`text`). Example: `ltrim('aabchelloccb', 'abc') -> helloccb` |
| md5(text) | Computes the MD5 hash of the argument (`text`). |
-| octet_length(text) | Returns number of bytes in the string (`text`). |
+| octet_length(text) | Returns number of bytes in the string or binary (`text`). |
| repeat(text, number) | Repeats the string the specified number of times. Example: `repeat('1', 4) -> 1111` |
| replace(string, from, to) | Replaces a specified string (`from`) with another specified string (`to`) in the string (`string`). Example: `replace('Hello', 'replace', 'el') -> Hola` |
| reverse(text) | Reverses the order of the characters in the string (`text`). Example: `reverse('hello') -> olleh` |
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index bdc8efb1bd..6726b7e145 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -1663,7 +1663,7 @@ trim(LEADING trim_str FROM str)
### `octet_length`
-Returns the length of a string in bytes.
+Returns the length of a string or binary in bytes.
```sql
octet_length(str)
@@ -1671,7 +1671,7 @@ octet_length(str)
#### Arguments
-- **str**: String expression to operate on. Can be a constant, column, or function, and any combination of operators.
+- **str**: String or binary expression to operate on. Can be a constant, column, or function, and any combination of operators.
#### Example
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org