You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/14 23:22:25 UTC
(datafusion) 01/01: perf: compare nested ScalarValues by type before converting to ArrayData (#26066)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-26066-c280a4b47be1f02f3994dc461d6b6335581cda18
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit cda4dd65c968103891bb1ba7352811124a6dd7f9
Author: Alexy Khrabrov <de...@gmail.com>
AuthorDate: Wed Oct 14 23:01:41 2026 +0000
perf: compare nested ScalarValues by type before converting to ArrayData (#26066)
## Which issue does this PR close?
- Closes #26065.
## Rationale for this change
Physical planning slows down, faster than the query grows, when
projections carry many typed NULL struct literals: a `UNION ALL` of 35
branches with 35 struct columns of 20 fields takes 862 ms in
`create_physical_plan` on `main`. `EquivalenceGroup::add_constant`
compares each new constant with every existing class, and
`ScalarValue::eq` compares nested values with arrow's array equality,
which converts both arrays to `ArrayData` before it compares anything,
even when the two values have different types.
## What changes are included in this PR?
`ScalarValue::eq` compares the nested variants (`List`, `LargeList`,
`FixedSizeList`, `ListView`, `LargeListView`, `Struct`, `Map`) through a
helper that returns early for the same `Arc`, and compares lengths and
data types before arrow's array equality. Results are unchanged; only
values of the same type and length reach the `ArrayData` comparison.
With the reproducer from the issue (release build, Apple M1 Max),
`create_physical_plan`:
| struct columns x fields | main | this PR |
|---|---:|---:|
| 10 x 20 | 31 ms | 17 ms |
| 20 x 20 | 179 ms | 47 ms |
| 35 x 20 | 862 ms | 147 ms |
| 35 x 40 | 1,766 ms | 253 ms |
## What is the testing strategy for this PR?
`test_nested_scalar_eq` covers typed NULL structs of the same and of
different types, struct values that are equal and unequal, and lists of
different element types. The scalar tests in `datafusion-common` pass.
## Are there any user-facing changes?
No.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
Co-authored-by: Adrian Garcia Badaracco <17...@users.noreply.github.com>
---
datafusion/common/src/scalar/mod.rs | 70 +++++++++++++++++++++++++++++++++----
1 file changed, 63 insertions(+), 7 deletions(-)
diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs
index 280aeaec16..9884e5b0b5 100644
--- a/datafusion/common/src/scalar/mod.rs
+++ b/datafusion/common/src/scalar/mod.rs
@@ -481,6 +481,19 @@ impl Hash for Fl<f16> {
}
}
+/// Equality of the arrays inside nested scalars.
+///
+/// Arrow's array equality converts both arrays to `ArrayData` before it
+/// compares anything, which is costly for wide nested types. Values of
+/// different lengths or types are unequal without that, and an array is
+/// equal to itself.
+fn nested_eq<T: Array + PartialEq>(a: &Arc<T>, b: &Arc<T>) -> bool {
+ Arc::ptr_eq(a, b)
+ || (a.len() == b.len()
+ && a.data_type() == b.data_type()
+ && a.as_ref() == b.as_ref())
+}
+
// manual implementation of `PartialEq`
impl PartialEq for ScalarValue {
fn eq(&self, other: &Self) -> bool {
@@ -552,19 +565,19 @@ impl PartialEq for ScalarValue {
(FixedSizeBinary(_, _), _) => false,
(LargeBinary(v1), LargeBinary(v2)) => v1.eq(v2),
(LargeBinary(_), _) => false,
- (FixedSizeList(v1), FixedSizeList(v2)) => v1.eq(v2),
+ (FixedSizeList(v1), FixedSizeList(v2)) => nested_eq(v1, v2),
(FixedSizeList(_), _) => false,
- (List(v1), List(v2)) => v1.eq(v2),
+ (List(v1), List(v2)) => nested_eq(v1, v2),
(List(_), _) => false,
- (LargeList(v1), LargeList(v2)) => v1.eq(v2),
+ (LargeList(v1), LargeList(v2)) => nested_eq(v1, v2),
(LargeList(_), _) => false,
- (ListView(v1), ListView(v2)) => v1.eq(v2),
+ (ListView(v1), ListView(v2)) => nested_eq(v1, v2),
(ListView(_), _) => false,
- (LargeListView(v1), LargeListView(v2)) => v1.eq(v2),
+ (LargeListView(v1), LargeListView(v2)) => nested_eq(v1, v2),
(LargeListView(_), _) => false,
- (Struct(v1), Struct(v2)) => v1.eq(v2),
+ (Struct(v1), Struct(v2)) => nested_eq(v1, v2),
(Struct(_), _) => false,
- (Map(v1), Map(v2)) => v1.eq(v2),
+ (Map(v1), Map(v2)) => nested_eq(v1, v2),
(Map(_), _) => false,
(Date32(v1), Date32(v2)) => v1.eq(v2),
(Date32(_), _) => false,
@@ -9929,6 +9942,49 @@ mod tests {
}
}
+ #[test]
+ fn test_nested_scalar_eq() {
+ let struct_type = |name: &str| {
+ DataType::Struct(Fields::from(vec![
+ Field::new(format!("{name}_a"), DataType::Int32, true),
+ Field::new(format!("{name}_b"), DataType::Utf8, true),
+ ]))
+ };
+ let null_x = ScalarValue::try_from(&struct_type("x")).unwrap();
+ let null_x_again = ScalarValue::try_from(&struct_type("x")).unwrap();
+ let null_y = ScalarValue::try_from(&struct_type("y")).unwrap();
+ assert_eq!(null_x, null_x.clone());
+ assert_eq!(null_x, null_x_again);
+ assert_ne!(null_x, null_y);
+
+ let value = |a: i32| {
+ ScalarValue::Struct(Arc::new(StructArray::from(vec![
+ (
+ Arc::new(Field::new("x_a", DataType::Int32, true)),
+ Arc::new(Int32Array::from(vec![Some(a)])) as ArrayRef,
+ ),
+ (
+ Arc::new(Field::new("x_b", DataType::Utf8, true)),
+ Arc::new(StringArray::from(vec![Some("b")])) as ArrayRef,
+ ),
+ ])))
+ };
+ assert_eq!(value(1), value(1));
+ assert_ne!(value(1), value(2));
+ assert_ne!(value(1), null_x);
+
+ let list_i32 = ScalarValue::List(ScalarValue::new_list_nullable(
+ &[ScalarValue::Int32(Some(1))],
+ &DataType::Int32,
+ ));
+ let list_i64 = ScalarValue::List(ScalarValue::new_list_nullable(
+ &[ScalarValue::Int64(Some(1))],
+ &DataType::Int64,
+ ));
+ assert_eq!(list_i32, list_i32.clone());
+ assert_ne!(list_i32, list_i64);
+ }
+
#[test]
fn test_struct_nulls() {
let fields_b = Fields::from(vec![
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org