You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by fi...@apache.org on 2025/07/18 09:27:38 UTC
(datafusion) branch main updated: Implement equals for stateful functions (#16781)
This is an automated email from the ASF dual-hosted git repository.
findepi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new afd82351ca Implement equals for stateful functions (#16781)
afd82351ca is described below
commit afd82351ca09e3320be847211ad0f246edfc0eb3
Author: Piotr Findeisen <pi...@gmail.com>
AuthorDate: Fri Jul 18 11:27:34 2025 +0200
Implement equals for stateful functions (#16781)
* Implement equals for stateful functions
Default implementation of `ScalarUDFImpl::equals`,
`AggregateUDFImpl::equals` and `WindowUDFImpl::equals` is correct for
stateless functions and those which only state is the `Signature`, which
is most of the functions.
This implements `equals` and `hash_value` for functions that have state
other than `Signature` object.
This fixes correctness issues which could occur when such stateful
functions are used together in one query.
* downgrade for MSRV
* Improve doc
* Update default UDF:: equals to compare aliases too
* Update default UDF:: equals to compare type too (‼️)
* remove now-obsoleted UDF equals/hash customizations
remove these which compare signature, aliases, as the default handles
these now
* remove equals impl which compares name, signature -- default does that
* cleanup imports
---
datafusion-examples/examples/function_factory.rs | 33 ++++++
.../tests/user_defined/user_defined_aggregates.rs | 27 +++++
.../user_defined/user_defined_scalar_functions.rs | 117 ++++++++++++++++++++-
.../user_defined/user_defined_window_functions.rs | 55 ++++++++++
datafusion/doc/src/lib.rs | 4 +-
datafusion/expr/src/async_udf.rs | 17 +++
datafusion/expr/src/expr_fn.rs | 104 ++++++++++++++++++
datafusion/expr/src/udaf.rs | 19 +++-
datafusion/expr/src/udf.rs | 20 +++-
datafusion/expr/src/udwf.rs | 19 +++-
datafusion/ffi/src/udaf/mod.rs | 32 +++++-
datafusion/ffi/src/udf/mod.rs | 33 ++++++
datafusion/ffi/src/udwf/mod.rs | 36 ++++++-
.../src/approx_percentile_cont_with_weight.rs | 25 +++++
.../functions-aggregate/src/bit_and_or_xor.rs | 37 ++++++-
datafusion/functions-aggregate/src/first_last.rs | 49 +++++++++
datafusion/functions-aggregate/src/regr.rs | 29 +++++
datafusion/functions-aggregate/src/stddev.rs | 18 ++++
datafusion/functions-aggregate/src/string_agg.rs | 24 +++++
datafusion/functions-window/src/lead_lag.rs | 20 +++-
datafusion/functions-window/src/nth_value.rs | 20 +++-
datafusion/functions-window/src/rank.rs | 31 +++++-
.../src/simplify_expressions/expr_simplifier.rs | 32 ++++++
datafusion/proto/tests/cases/mod.rs | 63 +++++++++++
datafusion/sql/tests/sql_integration.rs | 29 +++++
25 files changed, 862 insertions(+), 31 deletions(-)
diff --git a/datafusion-examples/examples/function_factory.rs b/datafusion-examples/examples/function_factory.rs
index 21da359633..425b0861ff 100644
--- a/datafusion-examples/examples/function_factory.rs
+++ b/datafusion-examples/examples/function_factory.rs
@@ -28,6 +28,7 @@ use datafusion::logical_expr::{
ColumnarValue, CreateFunction, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
Signature, Volatility,
};
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::result::Result as RResult;
use std::sync::Arc;
@@ -153,6 +154,38 @@ impl ScalarUDFImpl for ScalarFunctionWrapper {
fn output_ordering(&self, _input: &[ExprProperties]) -> Result<SortProperties> {
Ok(SortProperties::Unordered)
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ expr,
+ signature,
+ return_type,
+ } = self;
+ name == &other.name
+ && expr == &other.expr
+ && signature == &other.signature
+ && return_type == &other.return_type
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ expr,
+ signature,
+ return_type,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ expr.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ hasher.finish()
+ }
}
impl ScalarFunctionWrapper {
diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs
index aa5a72c0fb..7f1a12e9cd 100644
--- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs
+++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs
@@ -957,6 +957,33 @@ impl AggregateUDFImpl for MetadataBasedAggregateUdf {
curr_sum: 0,
}))
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ metadata,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && metadata == &other.metadata
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ metadata: _, // unhashable
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Debug)]
diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
index 90e49e504c..c5f9bdeb69 100644
--- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
+++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs
@@ -217,6 +217,34 @@ impl ScalarUDFImpl for Simple0ArgsScalarUDF {
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(100))))
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ return_type,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && return_type == &other.return_type
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ return_type,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[tokio::test]
@@ -557,6 +585,34 @@ impl ScalarUDFImpl for AddIndexToStringVolatileScalarUDF {
};
Ok(ColumnarValue::Array(Arc::new(StringArray::from(answer))))
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ return_type,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && return_type == &other.return_type
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ return_type,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[tokio::test]
@@ -974,6 +1030,38 @@ impl ScalarUDFImpl for ScalarFunctionWrapper {
Ok(ExprSimplifyResult::Simplified(replacement))
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ expr,
+ signature,
+ return_type,
+ } = self;
+ name == &other.name
+ && expr == &other.expr
+ && signature == &other.signature
+ && return_type == &other.return_type
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ expr,
+ signature,
+ return_type,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ expr.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ hasher.finish()
+ }
}
impl ScalarFunctionWrapper {
@@ -1450,7 +1538,30 @@ impl ScalarUDFImpl for MetadataBasedUdf {
}
fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
- self.name == other.name()
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ metadata,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && metadata == &other.metadata
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ metadata: _, // unhashable
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ hasher.finish()
}
}
@@ -1669,10 +1780,6 @@ impl ScalarUDFImpl for ExtensionBasedUdf {
}
}
}
-
- fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
- self.name == other.name()
- }
}
struct MyUserExtentionType {}
diff --git a/datafusion/core/tests/user_defined/user_defined_window_functions.rs b/datafusion/core/tests/user_defined/user_defined_window_functions.rs
index bcd2c3945e..3251847d03 100644
--- a/datafusion/core/tests/user_defined/user_defined_window_functions.rs
+++ b/datafusion/core/tests/user_defined/user_defined_window_functions.rs
@@ -40,6 +40,7 @@ use datafusion_physical_expr::{
PhysicalExpr,
};
use std::collections::HashMap;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::{
any::Any,
ops::Range,
@@ -568,6 +569,33 @@ impl OddCounter {
fn field(&self, field_args: WindowUDFFieldArgs) -> Result<FieldRef> {
Ok(Field::new(field_args.name(), DataType::Int64, true).into())
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ test_state,
+ aliases,
+ } = self;
+ signature == &other.signature
+ && Arc::ptr_eq(test_state, &other.test_state)
+ && aliases == &other.aliases
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ test_state,
+ aliases,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ signature.hash(&mut hasher);
+ Arc::as_ptr(test_state).hash(&mut hasher);
+ aliases.hash(&mut hasher);
+ hasher.finish()
+ }
}
ctx.register_udwf(WindowUDF::from(SimpleWindowUDF::new(test_state)))
@@ -815,6 +843,33 @@ impl WindowUDFImpl for MetadataBasedWindowUdf {
.with_metadata(self.metadata.clone())
.into())
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ metadata,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && metadata == &other.metadata
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ metadata: _, // unhashable
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Debug)]
diff --git a/datafusion/doc/src/lib.rs b/datafusion/doc/src/lib.rs
index f9b916c2b3..ca74c3b06d 100644
--- a/datafusion/doc/src/lib.rs
+++ b/datafusion/doc/src/lib.rs
@@ -39,7 +39,7 @@
/// thus all text should be in English.
///
/// [SQL function documentation]: https://datafusion.apache.org/user-guide/sql/index.html
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Hash)]
pub struct Documentation {
/// The section in the documentation where the UDF will be documented
pub doc_section: DocSection,
@@ -158,7 +158,7 @@ impl Documentation {
}
}
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Hash)]
pub struct DocSection {
/// True to include this doc section in the public
/// documentation, false otherwise
diff --git a/datafusion/expr/src/async_udf.rs b/datafusion/expr/src/async_udf.rs
index d900c16345..24ed124bb2 100644
--- a/datafusion/expr/src/async_udf.rs
+++ b/datafusion/expr/src/async_udf.rs
@@ -26,6 +26,7 @@ use datafusion_expr_common::columnar_value::ColumnarValue;
use datafusion_expr_common::signature::Signature;
use std::any::Any;
use std::fmt::{Debug, Display};
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::sync::Arc;
/// A scalar UDF that can invoke using async methods
@@ -111,6 +112,22 @@ impl ScalarUDFImpl for AsyncScalarUDF {
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
internal_err!("async functions should not be called directly")
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { inner } = self;
+ // TODO when MSRV >= 1.86.0, switch to `inner.equals(other.inner.as_ref())` leveraging trait upcasting
+ Arc::ptr_eq(inner, &other.inner)
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { inner } = self;
+ let mut hasher = DefaultHasher::new();
+ Arc::as_ptr(inner).hash(&mut hasher);
+ hasher.finish()
+ }
}
impl Display for AsyncScalarUDF {
diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs
index 001dfce59c..c0351a9dca 100644
--- a/datafusion/expr/src/expr_fn.rs
+++ b/datafusion/expr/src/expr_fn.rs
@@ -44,6 +44,7 @@ use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
use sqlparser::ast::NullTreatment;
use std::any::Any;
use std::fmt::Debug;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::ops::Not;
use std::sync::Arc;
@@ -474,6 +475,38 @@ impl ScalarUDFImpl for SimpleScalarUDF {
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
(self.fun)(&args.args)
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ return_type,
+ fun,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && return_type == &other.return_type
+ && Arc::ptr_eq(fun, &other.fun)
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ return_type,
+ fun,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ Arc::as_ptr(fun).hash(&mut hasher);
+ hasher.finish()
+ }
}
/// Creates a new UDAF with a specific signature, state type and return type.
@@ -594,6 +627,42 @@ impl AggregateUDFImpl for SimpleAggregateUDF {
fn state_fields(&self, _args: StateFieldsArgs) -> Result<Vec<FieldRef>> {
Ok(self.state_fields.clone())
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ return_type,
+ accumulator,
+ state_fields,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && return_type == &other.return_type
+ && Arc::ptr_eq(accumulator, &other.accumulator)
+ && state_fields == &other.state_fields
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ return_type,
+ accumulator,
+ state_fields,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ Arc::as_ptr(accumulator).hash(&mut hasher);
+ state_fields.hash(&mut hasher);
+ hasher.finish()
+ }
}
/// Creates a new UDWF with a specific signature, state type and return type.
@@ -686,6 +755,41 @@ impl WindowUDFImpl for SimpleWindowUDF {
true,
)))
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ return_type,
+ partition_evaluator_factory,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && return_type == &other.return_type
+ && Arc::ptr_eq(
+ partition_evaluator_factory,
+ &other.partition_evaluator_factory,
+ )
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ return_type,
+ partition_evaluator_factory,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ Arc::as_ptr(partition_evaluator_factory).hash(&mut hasher);
+ hasher.finish()
+ }
}
pub fn interval_year_month_lit(value: &str) -> Expr {
diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs
index c6ccddc70c..b6c8eb627c 100644
--- a/datafusion/expr/src/udaf.rs
+++ b/datafusion/expr/src/udaf.rs
@@ -899,26 +899,35 @@ pub trait AggregateUDFImpl: Debug + Send + Sync {
/// Return true if this aggregate UDF is equal to the other.
///
/// Allows customizing the equality of aggregate UDFs.
+ /// *Must* be implemented explicitly if the UDF type has internal state.
/// Must be consistent with [`Self::hash_value`] and follow the same rules as [`Eq`]:
///
/// - reflexive: `a.equals(a)`;
/// - symmetric: `a.equals(b)` implies `b.equals(a)`;
/// - transitive: `a.equals(b)` and `b.equals(c)` implies `a.equals(c)`.
///
- /// By default, compares [`Self::name`] and [`Self::signature`].
+ /// By default, compares type, [`Self::name`], [`Self::aliases`] and [`Self::signature`].
fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
- self.name() == other.name() && self.signature() == other.signature()
+ self.as_any().type_id() == other.as_any().type_id()
+ && self.name() == other.name()
+ && self.aliases() == other.aliases()
+ && self.signature() == other.signature()
}
/// Returns a hash value for this aggregate UDF.
///
- /// Allows customizing the hash code of aggregate UDFs. Similarly to [`Hash`] and [`Eq`],
- /// if [`Self::equals`] returns true for two UDFs, their `hash_value`s must be the same.
+ /// Allows customizing the hash code of aggregate UDFs.
+ /// *Must* be implemented explicitly whenever [`Self::equals`] is implemented.
///
- /// By default, hashes [`Self::name`] and [`Self::signature`].
+ /// Similarly to [`Hash`] and [`Eq`], if [`Self::equals`] returns true for two UDFs,
+ /// their `hash_value`s must be the same.
+ ///
+ /// By default, it is consistent with default implementation of [`Self::equals`].
fn hash_value(&self) -> u64 {
let hasher = &mut DefaultHasher::new();
+ self.as_any().type_id().hash(hasher);
self.name().hash(hasher);
+ self.aliases().hash(hasher);
self.signature().hash(hasher);
hasher.finish()
}
diff --git a/datafusion/expr/src/udf.rs b/datafusion/expr/src/udf.rs
index 81865a836d..3a94981ae4 100644
--- a/datafusion/expr/src/udf.rs
+++ b/datafusion/expr/src/udf.rs
@@ -697,26 +697,35 @@ pub trait ScalarUDFImpl: Debug + Send + Sync {
/// Return true if this scalar UDF is equal to the other.
///
/// Allows customizing the equality of scalar UDFs.
+ /// *Must* be implemented explicitly if the UDF type has internal state.
/// Must be consistent with [`Self::hash_value`] and follow the same rules as [`Eq`]:
///
/// - reflexive: `a.equals(a)`;
/// - symmetric: `a.equals(b)` implies `b.equals(a)`;
/// - transitive: `a.equals(b)` and `b.equals(c)` implies `a.equals(c)`.
///
- /// By default, compares [`Self::name`] and [`Self::signature`].
+ /// By default, compares type, [`Self::name`], [`Self::aliases`] and [`Self::signature`].
fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
- self.name() == other.name() && self.signature() == other.signature()
+ self.as_any().type_id() == other.as_any().type_id()
+ && self.name() == other.name()
+ && self.aliases() == other.aliases()
+ && self.signature() == other.signature()
}
/// Returns a hash value for this scalar UDF.
///
- /// Allows customizing the hash code of scalar UDFs. Similarly to [`Hash`] and [`Eq`],
- /// if [`Self::equals`] returns true for two UDFs, their `hash_value`s must be the same.
+ /// Allows customizing the hash code of scalar UDFs.
+ /// *Must* be implemented explicitly whenever [`Self::equals`] is implemented.
///
- /// By default, hashes [`Self::name`] and [`Self::signature`].
+ /// Similarly to [`Hash`] and [`Eq`], if [`Self::equals`] returns true for two UDFs,
+ /// their `hash_value`s must be the same.
+ ///
+ /// By default, it is consistent with default implementation of [`Self::equals`].
fn hash_value(&self) -> u64 {
let hasher = &mut DefaultHasher::new();
+ self.as_any().type_id().hash(hasher);
self.name().hash(hasher);
+ self.aliases().hash(hasher);
self.signature().hash(hasher);
hasher.finish()
}
@@ -832,6 +841,7 @@ impl ScalarUDFImpl for AliasedScalarUDFImpl {
fn hash_value(&self) -> u64 {
let hasher = &mut DefaultHasher::new();
+ std::any::type_name::<Self>().hash(hasher);
self.inner.hash_value().hash(hasher);
self.aliases.hash(hasher);
hasher.finish()
diff --git a/datafusion/expr/src/udwf.rs b/datafusion/expr/src/udwf.rs
index f5ab78e0c1..b19a083bef 100644
--- a/datafusion/expr/src/udwf.rs
+++ b/datafusion/expr/src/udwf.rs
@@ -362,26 +362,35 @@ pub trait WindowUDFImpl: Debug + Send + Sync {
/// Return true if this window UDF is equal to the other.
///
/// Allows customizing the equality of window UDFs.
+ /// *Must* be implemented explicitly if the UDF type has internal state.
/// Must be consistent with [`Self::hash_value`] and follow the same rules as [`Eq`]:
///
/// - reflexive: `a.equals(a)`;
/// - symmetric: `a.equals(b)` implies `b.equals(a)`;
/// - transitive: `a.equals(b)` and `b.equals(c)` implies `a.equals(c)`.
///
- /// By default, compares [`Self::name`] and [`Self::signature`].
+ /// By default, compares type, [`Self::name`], [`Self::aliases`] and [`Self::signature`].
fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
- self.name() == other.name() && self.signature() == other.signature()
+ self.as_any().type_id() == other.as_any().type_id()
+ && self.name() == other.name()
+ && self.aliases() == other.aliases()
+ && self.signature() == other.signature()
}
/// Returns a hash value for this window UDF.
///
- /// Allows customizing the hash code of window UDFs. Similarly to [`Hash`] and [`Eq`],
- /// if [`Self::equals`] returns true for two UDFs, their `hash_value`s must be the same.
+ /// Allows customizing the hash code of window UDFs.
+ /// *Must* be implemented explicitly whenever [`Self::equals`] is implemented.
///
- /// By default, hashes [`Self::name`] and [`Self::signature`].
+ /// Similarly to [`Hash`] and [`Eq`], if [`Self::equals`] returns true for two UDFs,
+ /// their `hash_value`s must be the same.
+ ///
+ /// By default, it is consistent with default implementation of [`Self::equals`].
fn hash_value(&self) -> u64 {
let hasher = &mut DefaultHasher::new();
+ self.as_any().type_id().hash(hasher);
self.name().hash(hasher);
+ self.aliases().hash(hasher);
self.signature().hash(hasher);
hasher.finish()
}
diff --git a/datafusion/ffi/src/udaf/mod.rs b/datafusion/ffi/src/udaf/mod.rs
index eb7a408ab1..63d44110a6 100644
--- a/datafusion/ffi/src/udaf/mod.rs
+++ b/datafusion/ffi/src/udaf/mod.rs
@@ -15,8 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use std::{ffi::c_void, sync::Arc};
-
use abi_stable::{
std_types::{ROption, RResult, RStr, RString, RVec},
StableAbi,
@@ -41,6 +39,8 @@ use datafusion::{
};
use datafusion_proto_common::from_proto::parse_proto_fields_to_fields;
use groups_accumulator::{FFI_GroupsAccumulator, ForeignGroupsAccumulator};
+use std::hash::{DefaultHasher, Hash, Hasher};
+use std::{ffi::c_void, sync::Arc};
use crate::util::{rvec_wrapped_to_vec_fieldref, vec_fieldref_to_rvec_wrapped};
use crate::{
@@ -553,6 +553,34 @@ impl AggregateUDFImpl for ForeignAggregateUDF {
Ok(rvec_wrapped_to_vec_datatype(&result_types)?)
}
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ aliases,
+ udaf,
+ } = self;
+ signature == &other.signature
+ && aliases == &other.aliases
+ && std::ptr::eq(udaf, &other.udaf)
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ aliases,
+ udaf,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ aliases.hash(&mut hasher);
+ std::ptr::hash(udaf, &mut hasher);
+ hasher.finish()
+ }
}
#[repr(C)]
diff --git a/datafusion/ffi/src/udf/mod.rs b/datafusion/ffi/src/udf/mod.rs
index 303acc783b..09cd0df128 100644
--- a/datafusion/ffi/src/udf/mod.rs
+++ b/datafusion/ffi/src/udf/mod.rs
@@ -46,6 +46,7 @@ use datafusion::{
use return_type_args::{
FFI_ReturnFieldArgs, ForeignReturnFieldArgs, ForeignReturnFieldArgsOwned,
};
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::{ffi::c_void, sync::Arc};
pub mod return_type_args;
@@ -407,6 +408,38 @@ impl ScalarUDFImpl for ForeignScalarUDF {
Ok(rvec_wrapped_to_vec_datatype(&result_types)?)
}
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ aliases,
+ udf,
+ signature,
+ } = self;
+ name == &other.name
+ && aliases == &other.aliases
+ && std::ptr::eq(udf, &other.udf)
+ && signature == &other.signature
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ aliases,
+ udf,
+ signature,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ aliases.hash(&mut hasher);
+ std::ptr::hash(udf, &mut hasher);
+ signature.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[cfg(test)]
diff --git a/datafusion/ffi/src/udwf/mod.rs b/datafusion/ffi/src/udwf/mod.rs
index 504bf7a411..b0b769219d 100644
--- a/datafusion/ffi/src/udwf/mod.rs
+++ b/datafusion/ffi/src/udwf/mod.rs
@@ -15,8 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-use std::{ffi::c_void, sync::Arc};
-
use abi_stable::{
std_types::{ROption, RResult, RString, RVec},
StableAbi,
@@ -42,6 +40,8 @@ use partition_evaluator::{FFI_PartitionEvaluator, ForeignPartitionEvaluator};
use partition_evaluator_args::{
FFI_PartitionEvaluatorArgs, ForeignPartitionEvaluatorArgs,
};
+use std::hash::{DefaultHasher, Hash, Hasher};
+use std::{ffi::c_void, sync::Arc};
mod partition_evaluator;
mod partition_evaluator_args;
mod range;
@@ -334,6 +334,38 @@ impl WindowUDFImpl for ForeignWindowUDF {
let options: Option<&FFI_SortOptions> = self.udf.sort_options.as_ref().into();
options.map(|s| s.into())
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ aliases,
+ udf,
+ signature,
+ } = self;
+ name == &other.name
+ && aliases == &other.aliases
+ && std::ptr::eq(udf, &other.udf)
+ && signature == &other.signature
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ aliases,
+ udf,
+ signature,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ aliases.hash(&mut hasher);
+ std::ptr::hash(udf, &mut hasher);
+ signature.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[repr(C)]
diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs b/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs
index 5180d45889..ab847e8388 100644
--- a/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs
+++ b/datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs
@@ -17,6 +17,7 @@
use std::any::Any;
use std::fmt::{Debug, Formatter};
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem::size_of_val;
use std::sync::Arc;
@@ -186,6 +187,30 @@ impl AggregateUDFImpl for ApproxPercentileContWithWeight {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ approx_percentile_cont,
+ } = self;
+ signature == &other.signature
+ && approx_percentile_cont.equals(&other.approx_percentile_cont)
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ approx_percentile_cont,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ hasher.write_u64(approx_percentile_cont.hash_value());
+ hasher.finish()
+ }
}
#[derive(Debug)]
diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rs
index 4512162ba5..8ca5d992a7 100644
--- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs
+++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs
@@ -20,6 +20,7 @@
use std::any::Any;
use std::collections::HashSet;
use std::fmt::{Display, Formatter};
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem::{size_of, size_of_val};
use ahash::RandomState;
@@ -196,7 +197,7 @@ make_bitwise_udaf_expr_and_func!(
);
/// The different types of bitwise operations that can be performed.
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
enum BitwiseOperationType {
And,
Or,
@@ -312,6 +313,38 @@ impl AggregateUDFImpl for BitwiseOperation {
fn documentation(&self) -> Option<&Documentation> {
Some(self.documentation)
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ operation,
+ func_name,
+ documentation,
+ } = self;
+ signature == &other.signature
+ && operation == &other.operation
+ && func_name == &other.func_name
+ && documentation == &other.documentation
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ operation,
+ func_name,
+ documentation,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ operation.hash(&mut hasher);
+ func_name.hash(&mut hasher);
+ documentation.hash(&mut hasher);
+ hasher.finish()
+ }
}
struct BitAndAccumulator<T: ArrowNumericType> {
@@ -478,7 +511,7 @@ impl<T: ArrowNumericType> Default for DistinctBitXorAccumulator<T> {
impl<T: ArrowNumericType> Accumulator for DistinctBitXorAccumulator<T>
where
- T::Native: std::ops::BitXor<Output = T::Native> + std::hash::Hash + Eq,
+ T::Native: std::ops::BitXor<Output = T::Native> + Hash + Eq,
{
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
if values.is_empty() {
diff --git a/datafusion/functions-aggregate/src/first_last.rs b/datafusion/functions-aggregate/src/first_last.rs
index 9b5b821317..0856237d08 100644
--- a/datafusion/functions-aggregate/src/first_last.rs
+++ b/datafusion/functions-aggregate/src/first_last.rs
@@ -19,6 +19,7 @@
use std::any::Any;
use std::fmt::Debug;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem::size_of_val;
use std::sync::Arc;
@@ -292,6 +293,30 @@ impl AggregateUDFImpl for FirstValue {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ is_input_pre_ordered,
+ } = self;
+ signature == &other.signature
+ && is_input_pre_ordered == &other.is_input_pre_ordered
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ is_input_pre_ordered,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ is_input_pre_ordered.hash(&mut hasher);
+ hasher.finish()
+ }
}
// TODO: rename to PrimitiveGroupsAccumulator
@@ -1212,6 +1237,30 @@ impl AggregateUDFImpl for LastValue {
}
}
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ is_input_pre_ordered,
+ } = self;
+ signature == &other.signature
+ && is_input_pre_ordered == &other.is_input_pre_ordered
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ is_input_pre_ordered,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ is_input_pre_ordered.hash(&mut hasher);
+ hasher.finish()
+ }
}
/// This accumulator is used when there is no ordering specified for the
diff --git a/datafusion/functions-aggregate/src/regr.rs b/datafusion/functions-aggregate/src/regr.rs
index 0f84aa1323..4600f6570f 100644
--- a/datafusion/functions-aggregate/src/regr.rs
+++ b/datafusion/functions-aggregate/src/regr.rs
@@ -38,6 +38,7 @@ use datafusion_expr::{
};
use std::any::Any;
use std::fmt::Debug;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem::size_of_val;
use std::sync::{Arc, LazyLock};
@@ -320,6 +321,34 @@ impl AggregateUDFImpl for Regr {
fn documentation(&self) -> Option<&Documentation> {
self.regr_type.documentation()
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ regr_type,
+ func_name,
+ } = self;
+ signature == &other.signature
+ && regr_type == &other.regr_type
+ && func_name == &other.func_name
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ regr_type,
+ func_name,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ regr_type.hash(&mut hasher);
+ func_name.hash(&mut hasher);
+ hasher.finish()
+ }
}
/// `RegrAccumulator` is used to compute linear regression aggregate functions
diff --git a/datafusion/functions-aggregate/src/stddev.rs b/datafusion/functions-aggregate/src/stddev.rs
index bf6d21a808..2f9f1cac84 100644
--- a/datafusion/functions-aggregate/src/stddev.rs
+++ b/datafusion/functions-aggregate/src/stddev.rs
@@ -19,6 +19,7 @@
use std::any::Any;
use std::fmt::{Debug, Formatter};
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem::align_of_val;
use std::sync::Arc;
@@ -153,6 +154,23 @@ impl AggregateUDFImpl for Stddev {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { signature, alias } = self;
+ signature == &other.signature && alias == &other.alias
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { signature, alias } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ alias.hash(&mut hasher);
+ hasher.finish()
+ }
}
make_udaf_expr_and_func!(
diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs
index 09199e19cf..56c5ee1aaa 100644
--- a/datafusion/functions-aggregate/src/string_agg.rs
+++ b/datafusion/functions-aggregate/src/string_agg.rs
@@ -18,6 +18,7 @@
//! [`StringAgg`] accumulator for the `string_agg` function
use std::any::Any;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::mem::size_of_val;
use crate::array_agg::ArrayAgg;
@@ -180,6 +181,29 @@ impl AggregateUDFImpl for StringAgg {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ array_agg,
+ } = self;
+ signature == &other.signature && array_agg.equals(&other.array_agg)
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ array_agg,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ hasher.write_u64(array_agg.hash_value());
+ hasher.finish()
+ }
}
#[derive(Debug)]
diff --git a/datafusion/functions-window/src/lead_lag.rs b/datafusion/functions-window/src/lead_lag.rs
index e2a755371e..05bbd1e114 100644
--- a/datafusion/functions-window/src/lead_lag.rs
+++ b/datafusion/functions-window/src/lead_lag.rs
@@ -35,6 +35,7 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr;
use std::any::Any;
use std::cmp::min;
use std::collections::VecDeque;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::ops::{Neg, Range};
use std::sync::{Arc, LazyLock};
@@ -93,7 +94,7 @@ pub fn lead(
lead_udwf().call(vec![arg, shift_offset_lit, default_lit])
}
-#[derive(Debug)]
+#[derive(Debug, PartialEq, Eq, Hash)]
enum WindowShiftKind {
Lag,
Lead,
@@ -298,6 +299,23 @@ impl WindowUDFImpl for WindowShift {
WindowShiftKind::Lead => Some(get_lead_doc()),
}
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { signature, kind } = self;
+ signature == &other.signature && kind == &other.kind
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { signature, kind } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ kind.hash(&mut hasher);
+ hasher.finish()
+ }
}
/// When `lead`/`lag` is evaluated on a `NULL` expression we attempt to
diff --git a/datafusion/functions-window/src/nth_value.rs b/datafusion/functions-window/src/nth_value.rs
index 0b83e1ff9f..4c7dd995ec 100644
--- a/datafusion/functions-window/src/nth_value.rs
+++ b/datafusion/functions-window/src/nth_value.rs
@@ -35,6 +35,7 @@ use field::WindowUDFFieldArgs;
use std::any::Any;
use std::cmp::Ordering;
use std::fmt::Debug;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::ops::Range;
use std::sync::LazyLock;
@@ -76,7 +77,7 @@ pub fn nth_value(arg: datafusion_expr::Expr, n: i64) -> datafusion_expr::Expr {
}
/// Tag to differentiate special use cases of the NTH_VALUE built-in window function.
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub enum NthValueKind {
First,
Last,
@@ -335,6 +336,23 @@ impl WindowUDFImpl for NthValue {
NthValueKind::Nth => Some(get_nth_value_doc()),
}
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { signature, kind } = self;
+ signature == &other.signature && kind == &other.kind
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { signature, kind } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ kind.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Debug, Clone)]
diff --git a/datafusion/functions-window/src/rank.rs b/datafusion/functions-window/src/rank.rs
index 969a957cdd..5099b25e1d 100644
--- a/datafusion/functions-window/src/rank.rs
+++ b/datafusion/functions-window/src/rank.rs
@@ -36,6 +36,7 @@ use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
use field::WindowUDFFieldArgs;
use std::any::Any;
use std::fmt::Debug;
+use std::hash::{DefaultHasher, Hash, Hasher};
use std::iter;
use std::ops::Range;
use std::sync::{Arc, LazyLock};
@@ -95,7 +96,7 @@ impl Rank {
}
}
-#[derive(Debug, Copy, Clone)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub enum RankType {
Basic,
Dense,
@@ -242,6 +243,34 @@ impl WindowUDFImpl for Rank {
RankType::Percent => Some(get_percent_rank_doc()),
}
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ rank_type,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && rank_type == &other.rank_type
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ rank_type,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ rank_type.hash(&mut hasher);
+ hasher.finish()
+ }
}
/// State for the RANK(rank) built-in window function.
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index 13e27612ae..9a3a8bcd23 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -2181,6 +2181,7 @@ mod tests {
};
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
+ use std::hash::{DefaultHasher, Hash, Hasher};
use std::{
collections::HashMap,
ops::{BitAnd, BitOr, BitXor},
@@ -4404,6 +4405,21 @@ mod tests {
None
}
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { simplify } = self;
+ simplify == &other.simplify
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { simplify } = self;
+ let mut hasher = DefaultHasher::new();
+ simplify.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[test]
@@ -4474,6 +4490,22 @@ mod tests {
fn field(&self, _field_args: WindowUDFFieldArgs) -> Result<FieldRef> {
unimplemented!("not needed for tests")
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { simplify } = self;
+ simplify == &other.simplify
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { simplify } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ simplify.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Debug)]
struct VolatileUdf {
diff --git a/datafusion/proto/tests/cases/mod.rs b/datafusion/proto/tests/cases/mod.rs
index 4c7da2768e..6158b727df 100644
--- a/datafusion/proto/tests/cases/mod.rs
+++ b/datafusion/proto/tests/cases/mod.rs
@@ -27,6 +27,7 @@ use datafusion_functions_window_common::field::WindowUDFFieldArgs;
use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
use std::any::Any;
use std::fmt::Debug;
+use std::hash::{DefaultHasher, Hash, Hasher};
mod roundtrip_logical_plan;
mod roundtrip_physical_plan;
@@ -80,6 +81,34 @@ impl ScalarUDFImpl for MyRegexUdf {
fn aliases(&self) -> &[String] {
&self.aliases
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ signature,
+ pattern,
+ aliases,
+ } = self;
+ signature == &other.signature
+ && pattern == &other.pattern
+ && aliases == &other.aliases
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ signature,
+ pattern,
+ aliases,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ pattern.hash(&mut hasher);
+ aliases.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -123,6 +152,23 @@ impl AggregateUDFImpl for MyAggregateUDF {
) -> datafusion_common::Result<Box<dyn Accumulator>> {
unimplemented!()
}
+
+ fn equals(&self, other: &dyn AggregateUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { signature, result } = self;
+ signature == &other.signature && result == &other.result
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { signature, result } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ result.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Clone, PartialEq, ::prost::Message)]
@@ -172,6 +218,23 @@ impl WindowUDFImpl for CustomUDWF {
) -> datafusion_common::Result<FieldRef> {
Ok(Field::new(field_args.name(), DataType::UInt64, false).into())
}
+
+ fn equals(&self, other: &dyn WindowUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self { signature, payload } = self;
+ signature == &other.signature && payload == &other.payload
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self { signature, payload } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ signature.hash(&mut hasher);
+ payload.hash(&mut hasher);
+ hasher.finish()
+ }
}
#[derive(Debug)]
diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs
index 306fbbdaab..1e857b6a07 100644
--- a/datafusion/sql/tests/sql_integration.rs
+++ b/datafusion/sql/tests/sql_integration.rs
@@ -16,6 +16,7 @@
// under the License.
use std::any::Any;
+use std::hash::{DefaultHasher, Hash, Hasher};
#[cfg(test)]
use std::sync::Arc;
use std::vec;
@@ -3348,6 +3349,34 @@ impl ScalarUDFImpl for DummyUDF {
fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result<ColumnarValue> {
panic!("dummy - not implemented")
}
+
+ fn equals(&self, other: &dyn ScalarUDFImpl) -> bool {
+ let Some(other) = other.as_any().downcast_ref::<Self>() else {
+ return false;
+ };
+ let Self {
+ name,
+ signature,
+ return_type,
+ } = self;
+ name == &other.name
+ && signature == &other.signature
+ && return_type == &other.return_type
+ }
+
+ fn hash_value(&self) -> u64 {
+ let Self {
+ name,
+ signature,
+ return_type,
+ } = self;
+ let mut hasher = DefaultHasher::new();
+ std::any::type_name::<Self>().hash(&mut hasher);
+ name.hash(&mut hasher);
+ signature.hash(&mut hasher);
+ return_type.hash(&mut hasher);
+ hasher.finish()
+ }
}
fn parse_decimals_parser_options() -> ParserOptions {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org