You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/14 14:48:59 UTC
(datafusion) 01/01: feat: add `strictly_order_preserving` for more expr (#23922)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-23922-85d4cbb0a9a885f2fd22bfdac468844446ad70d6
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 2653bc67589e210d2d8b492b75feab6c5cf77809
Author: Raz Luvaton <16...@users.noreply.github.com>
AuthorDate: Mon Sep 14 14:48:34 2026 +0000
feat: add `strictly_order_preserving` for more expr (#23922)
## Which issue does this PR close?
- Closes #23920
## Rationale for this change
just more uses of the new property and comment on some non strictly
order preserving for why not
## What changes are included in this PR?
marked if something is order preserving or not, for those who are not
I've added a comment why not, since some expressions might be confusing
why not
## Are these changes tested?
no
## Are there any user-facing changes?
not api ones
---
datafusion/functions-nested/src/make_array.rs | 6 ++++++
datafusion/functions-nested/src/repeat.rs | 7 +++++++
datafusion/functions/src/math/factorial.rs | 11 ++++++++++
datafusion/functions/src/math/floor.rs | 5 +++++
datafusion/functions/src/string/repeat.rs | 6 ++++++
datafusion/sqllogictest/test_files/order.slt | 30 +++++++++++++++++++++++++++
6 files changed, 65 insertions(+)
diff --git a/datafusion/functions-nested/src/make_array.rs b/datafusion/functions-nested/src/make_array.rs
index ba746cd9cf..a63217cd5c 100644
--- a/datafusion/functions-nested/src/make_array.rs
+++ b/datafusion/functions-nested/src/make_array.rs
@@ -37,6 +37,7 @@ use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
+use datafusion_expr_common::sort_properties::ExprProperties;
use datafusion_macros::user_doc;
use itertools::Itertools as _;
@@ -113,6 +114,11 @@ impl ScalarUDFImpl for MakeArray {
&self.aliases
}
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // Not strictly order preserving since null input does not correspond to null output
+ Ok(false)
+ }
+
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
if arg_types.is_empty() {
Ok(vec![])
diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs
index b275ae661e..dc99d78036 100644
--- a/datafusion/functions-nested/src/repeat.rs
+++ b/datafusion/functions-nested/src/repeat.rs
@@ -37,6 +37,7 @@ use datafusion_expr::{
Volatility,
};
use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
+use datafusion_expr_common::sort_properties::ExprProperties;
use datafusion_macros::user_doc;
use std::mem::size_of;
use std::sync::Arc;
@@ -137,6 +138,12 @@ impl ScalarUDFImpl for ArrayRepeat {
make_scalar_function(array_repeat_inner)(&args.args)
}
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // this is not strictly order preserving
+ // since array_repeat(NULL, 2) = [NULL, NULL] and not NULL
+ Ok(false)
+ }
+
fn aliases(&self) -> &[String] {
&self.aliases
}
diff --git a/datafusion/functions/src/math/factorial.rs b/datafusion/functions/src/math/factorial.rs
index f4e9b60dd3..d2a41a2dfb 100644
--- a/datafusion/functions/src/math/factorial.rs
+++ b/datafusion/functions/src/math/factorial.rs
@@ -28,6 +28,7 @@ use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
+use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties};
use datafusion_macros::user_doc;
#[user_doc(
@@ -116,6 +117,16 @@ impl ScalarUDFImpl for FactorialFunc {
}
}
+ fn output_ordering(&self, inputs: &[ExprProperties]) -> Result<SortProperties> {
+ // Keep the same ordering as the input
+ Ok(inputs[0].sort_properties)
+ }
+
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // factorial(0) == factorial(1) == 1 so ordering is not preserved
+ Ok(false)
+ }
+
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
diff --git a/datafusion/functions/src/math/floor.rs b/datafusion/functions/src/math/floor.rs
index 24a93fa7e5..9c01238d4a 100644
--- a/datafusion/functions/src/math/floor.rs
+++ b/datafusion/functions/src/math/floor.rs
@@ -261,6 +261,11 @@ impl ScalarUDFImpl for FloorFunc {
Ok(input[0].sort_properties)
}
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // floor allows distinct inputs to collapse into equal outputs
+ Ok(false)
+ }
+
fn evaluate_bounds(&self, inputs: &[&Interval]) -> Result<Interval> {
let data_type = decimal_floor_ceil_return_type(&inputs[0].data_type());
Interval::make_unbounded(&data_type)
diff --git a/datafusion/functions/src/string/repeat.rs b/datafusion/functions/src/string/repeat.rs
index a53f1e2e4f..cf2b8fd25c 100644
--- a/datafusion/functions/src/string/repeat.rs
+++ b/datafusion/functions/src/string/repeat.rs
@@ -32,6 +32,7 @@ use datafusion_common::{
use datafusion_expr::{ColumnarValue, Documentation, Volatility};
use datafusion_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature};
use datafusion_expr_common::signature::{Coercion, TypeSignatureClass};
+use datafusion_expr_common::sort_properties::ExprProperties;
use datafusion_macros::user_doc;
#[user_doc(
@@ -160,6 +161,11 @@ impl ScalarUDFImpl for RepeatFunc {
fn documentation(&self) -> Option<&Documentation> {
self.doc()
}
+
+ fn strictly_order_preserving(&self, _inputs: &[ExprProperties]) -> Result<bool> {
+ // repeat('bc', 2) = 'bcbc' > repeat('bca', 2) = 'bcabca', so ordering is not preserved
+ Ok(false)
+ }
}
/// Computes repeat for a single string value with max size check
diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt
index 1a02f5f59a..d080139bc7 100644
--- a/datafusion/sqllogictest/test_files/order.slt
+++ b/datafusion/sqllogictest/test_files/order.slt
@@ -799,6 +799,36 @@ logical_plan
03)----TableScan: multiple_ordered_table projection=[a]
physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[floor(CAST(a@1 AS Float64)) as a_], file_type=csv, has_header=true
+# A merely monotone (`preserves_lex_ordering`, but not strictly order-preserving) function such as factorial()
+# does NOT justify the suffix keys:
+# factorial(0) == factorial(1) which means that it collapses distinct inputs into one output value, and `b`
+# is not sorted within such a run, so a SortExec must remain.
+query TT
+EXPLAIN SELECT factorial(a) AS a_, b
+FROM multiple_ordered_table
+ORDER BY a_, b;
+----
+logical_plan
+01)Sort: a_ ASC NULLS LAST, multiple_ordered_table.b ASC NULLS LAST
+02)--Projection: factorial(CAST(multiple_ordered_table.a AS Int64)) AS a_, multiple_ordered_table.b
+03)----TableScan: multiple_ordered_table projection=[a, b]
+physical_plan
+01)SortExec: expr=[a_@0 ASC NULLS LAST, b@1 ASC NULLS LAST], preserve_partitioning=[false]
+02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[factorial(CAST(a@1 AS Int64)) as a_, b], file_type=csv, has_header=true
+
+# Monotonicity alone is still enough when the expression is the *only* sort
+# key, so here the SortExec is removed even though factorial() is not strict:
+query TT
+EXPLAIN SELECT factorial(a) AS a_
+FROM multiple_ordered_table
+ORDER BY a_;
+----
+logical_plan
+01)Sort: a_ ASC NULLS LAST
+02)--Projection: factorial(CAST(multiple_ordered_table.a AS Int64)) AS a_
+03)----TableScan: multiple_ordered_table projection=[a]
+physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[factorial(CAST(a@1 AS Int64)) as a_], file_type=csv, has_header=true
+
# The SLT runner sets `target_partitions` to 4 instead of using the default, so
# reset it explicitly.
statement ok
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org