You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/07/11 06:35:14 UTC
(datafusion) branch gh-readonly-queue/main/pr-23456-3e058f09c91a18c8f36b71a656231603b33888c2 created (now 5b60fcb110)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-23456-3e058f09c91a18c8f36b71a656231603b33888c2
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 5b60fcb110 perf: optimize encode in datafusion-functions (#23456)
This branch includes the following new commits:
new 5b60fcb110 perf: optimize encode in datafusion-functions (#23456)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: perf: optimize encode in datafusion-functions (#23456)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23456-3e058f09c91a18c8f36b71a656231603b33888c2
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 5b60fcb1107b4d9b4dffa11ff10ed8083b5e3a66
Author: Andy Grove <ag...@apache.org>
AuthorDate: Sat Jul 11 00:34:51 2026 -0600
perf: optimize encode in datafusion-functions (#23456)
## Which issue does this PR close?
N/A
## Rationale for this change
Rewrote the hex path of encode() to write hex directly into one
pre-sized buffer via hex::encode_to_slice, eliminating a per-element
String allocation and copy per row.
## What changes are included in this PR?
Rewrote the hex path of encode() to write hex directly into one
pre-sized buffer via hex::encode_to_slice, eliminating a per-element
String allocation and copy per row.
## Are these changes tested?
Existing tests
Benchmark:
- hex_encode_1024: 80.934% faster (base 25305ns -> cand 4824ns)
- hex_encode_4096: 81.297% faster (base 99386ns -> cand 18587ns)
- hex_encode_8192: 80.24% faster (base 200031ns -> cand 39525ns)
## Are there any user-facing changes?
No
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
---
datafusion/functions/benches/encoding.rs | 25 +++++++++++++++++
datafusion/functions/src/encoding/inner.rs | 45 ++++++++++++++++++++++++++----
2 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/datafusion/functions/benches/encoding.rs b/datafusion/functions/benches/encoding.rs
index 0b8f0c5c51..451baff518 100644
--- a/datafusion/functions/benches/encoding.rs
+++ b/datafusion/functions/benches/encoding.rs
@@ -27,10 +27,35 @@ use std::sync::Arc;
fn criterion_benchmark(c: &mut Criterion) {
let decode = encoding::decode();
+ let encode = encoding::encode();
let config_options = Arc::new(ConfigOptions::default());
for size in [1024, 4096, 8192] {
let bin_array = Arc::new(create_binary_array::<i32>(size, 0.2));
+
+ c.bench_function(&format!("hex_encode/{size}"), |b| {
+ let method = ColumnarValue::Scalar("hex".into());
+ let arg_fields = vec![
+ Field::new("a", bin_array.data_type().to_owned(), true).into(),
+ Field::new("b", method.data_type().to_owned(), true).into(),
+ ];
+ let args = vec![ColumnarValue::Array(bin_array.clone()), method];
+ let return_field = Field::new("f", DataType::Utf8, true).into();
+
+ b.iter(|| {
+ black_box(
+ encode
+ .invoke_with_args(ScalarFunctionArgs {
+ args: args.clone(),
+ arg_fields: arg_fields.clone(),
+ number_rows: size,
+ return_field: Arc::clone(&return_field),
+ config_options: Arc::clone(&config_options),
+ })
+ .unwrap(),
+ )
+ })
+ });
c.bench_function(&format!("base64_decode/{size}"), |b| {
let method = ColumnarValue::Scalar("base64".into());
let encoded = encoding::encode()
diff --git a/datafusion/functions/src/encoding/inner.rs b/datafusion/functions/src/encoding/inner.rs
index 877acbb529..027ec8e5e5 100644
--- a/datafusion/functions/src/encoding/inner.rs
+++ b/datafusion/functions/src/encoding/inner.rs
@@ -410,11 +410,7 @@ impl Encoding {
.collect();
Ok(Arc::new(array))
}
- Self::Hex => {
- let array: GenericStringArray<OutputOffset> =
- array.iter().map(|x| x.map(hex::encode)).collect();
- Ok(Arc::new(array))
- }
+ Self::Hex => hex_encode_array::<_, OutputOffset>(array),
}
}
@@ -459,6 +455,45 @@ impl Encoding {
}
}
+/// Hex-encode a binary array into a string array, writing the lowercase hex
+/// digits directly into a single pre-sized value buffer. Each input byte maps
+/// to exactly two hex characters, so the output size is known up front and no
+/// per-element `String` is allocated.
+fn hex_encode_array<'a, InputBinaryArray, OutputOffset>(
+ array: &InputBinaryArray,
+) -> Result<ArrayRef>
+where
+ InputBinaryArray: BinaryArrayType<'a>,
+ OutputOffset: OffsetSizeTrait,
+{
+ let total_input_bytes: usize = array.iter().flatten().map(|v| v.len()).sum();
+
+ let mut values = vec![0u8; total_input_bytes * 2];
+ let mut offsets = Vec::<OutputOffset>::with_capacity(array.len() + 1);
+ offsets.push(OutputOffset::zero());
+
+ let mut pos = 0usize;
+ for v in array.iter() {
+ if let Some(v) = v {
+ let out_len = v.len() * 2;
+ // The slice is sized to exactly `2 * v.len()`, which is the only
+ // condition under which `encode_to_slice` can fail, so this cannot
+ // error.
+ hex::encode_to_slice(v, &mut values[pos..pos + out_len])
+ .map_err(|e| exec_datafusion_err!("Failed to encode to hex: {e}"))?;
+ pos += out_len;
+ }
+ offsets.push(OutputOffset::usize_as(pos));
+ }
+
+ let array = GenericStringArray::<OutputOffset>::try_new(
+ OffsetBuffer::new(offsets.into()),
+ Buffer::from_vec(values),
+ array.nulls().cloned(),
+ )?;
+ Ok(Arc::new(array))
+}
+
fn delegated_decode<'a, DecodeFunction, InputBinaryArray, OutputOffset>(
decode: DecodeFunction,
input: &InputBinaryArray,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org