You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/14 13:08:11 UTC
(datafusion) branch gh-readonly-queue/main/pr-25243-7b00b63e048b7e0de6ca42b82e0824d77862d702 created (now 85d4cbb0a9)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25243-7b00b63e048b7e0de6ca42b82e0824d77862d702
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at 85d4cbb0a9 Support ORDER BY ALL for projected expressions (#25243)
This branch includes the following new commits:
new 85d4cbb0a9 Support ORDER BY ALL for projected expressions (#25243)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: Support ORDER BY ALL for projected expressions (#25243)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25243-7b00b63e048b7e0de6ca42b82e0824d77862d702
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 85d4cbb0a9a885f2fd22bfdac468844446ad70d6
Author: Artem Osipov <59...@users.noreply.github.com>
AuthorDate: Mon Sep 14 13:07:46 2026 +0000
Support ORDER BY ALL for projected expressions (#25243)
## Which issue does this PR close?
Closes a planner gap where `ORDER BY ALL` only accepted raw column
projections and was silently ignored for set-operation outputs.
## Rationale for this change
`ORDER BY ALL` means sorting by every output column in select-list
order. Expanding it to the equivalent 1-based ordinal keys supports
computed expressions, aliases, aggregate outputs, wildcard-expanded
projections, and set-operation outputs through one planner path.
The ordinal expansion also sorts already-projected columns rather than
evaluating computed expressions again. Physical execution remains on
DataFusion's existing vectorized `SortExec`; this adds no row-wise
conversion or custom physical operator.
## What changes are included?
- expand `OrderByKind::All` to ordinal `OrderByExpr`s from the output
width
- provide output width for non-`SELECT` set expressions instead of
dropping `ORDER BY ALL`
- add focused execution coverage for computed expressions, null-order
options, set operations, and aggregate output
## Are these changes tested?
- `cargo +1.95.0 test -p datafusion-sqllogictest --test sqllogictests --
order_by_all.slt --test-threads 1`
- `cargo +1.95.0 test -p datafusion-sql --test sql_integration --
--test-threads 8` (591 passed)
- `cargo +1.95.0 clippy -p datafusion-sql --all-targets -- -D warnings`
- `cargo +1.95.0 fmt --all -- --check`
Snowflake dialect parsing support is proposed independently in
apache/datafusion-sqlparser-rs#2502; this planner change is generic to
every dialect that already emits `OrderByKind::All`.
---
datafusion/sql/src/query.rs | 62 +++++++---------
.../sqllogictest/test_files/order_by_all.slt | 86 ++++++++++++++++++++++
2 files changed, 111 insertions(+), 37 deletions(-)
diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs
index e2b9e4d2d5..d23d1a008f 100644
--- a/datafusion/sql/src/query.rs
+++ b/datafusion/sql/src/query.rs
@@ -27,11 +27,10 @@ use datafusion_expr::{
CreateMemoryTable, DdlStatement, Distinct, Expr, LogicalPlan, LogicalPlanBuilder,
};
use sqlparser::ast::{
- Expr as SQLExpr, ExprWithAliasAndOrderBy, Ident, LimitClause, Offset, OffsetRows,
- OrderBy, OrderByExpr, OrderByKind, PipeOperator, Query, SelectInto, SetExpr,
- SetOperator, SetQuantifier, TableAlias,
+ Expr as SQLExpr, ExprWithAliasAndOrderBy, LimitClause, Offset, OffsetRows, OrderBy,
+ OrderByExpr, OrderByKind, PipeOperator, Query, SelectInto, SetExpr, SetOperator,
+ SetQuantifier, TableAlias, Value,
};
-use sqlparser::tokenizer::Span;
impl<S: ContextProvider> SqlToRel<'_, S> {
/// Generate a logical plan from an SQL query/subquery
@@ -83,7 +82,8 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
let plan = crate::stack::maybe_grow(|| {
self.set_expr_to_plan(other, planner_context)
})?;
- let oby_exprs = to_order_by_exprs(order_by)?;
+ let oby_exprs =
+ to_order_by_exprs(order_by, plan.schema().fields().len())?;
let order_by_rex = self.order_by_to_sort_expr(
oby_exprs,
plan.schema(),
@@ -379,47 +379,35 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
}
/// Returns the order by expressions from the query.
-fn to_order_by_exprs(order_by: Option<OrderBy>) -> Result<Vec<OrderByExpr>> {
- to_order_by_exprs_with_select(order_by, None)
-}
-
-/// Returns the order by expressions from the query with the select expressions.
-pub(crate) fn to_order_by_exprs_with_select(
+fn to_order_by_exprs(
order_by: Option<OrderBy>,
- select_exprs: Option<&Vec<Expr>>,
+ order_by_all_column_count: usize,
) -> Result<Vec<OrderByExpr>> {
let Some(OrderBy { kind, interpolate }) = order_by else {
- // If no order by, return an empty array.
return Ok(vec![]);
};
if let Some(_interpolate) = interpolate {
return not_impl_err!("ORDER BY INTERPOLATE is not supported");
}
match kind {
- OrderByKind::All(order_by_options) => {
- let Some(exprs) = select_exprs else {
- return Ok(vec![]);
- };
- let order_by_exprs = exprs
- .iter()
- .map(|select_expr| match select_expr {
- Expr::Column(column) => Ok(OrderByExpr {
- expr: SQLExpr::Identifier(Ident {
- value: column.name.clone(),
- quote_style: None,
- span: Span::empty(),
- }),
- options: order_by_options,
- with_fill: None,
- }),
- // TODO: Support other types of expressions
- _ => not_impl_err!(
- "ORDER BY ALL is not supported for non-column expressions"
- ),
- })
- .collect::<Result<Vec<_>>>()?;
- Ok(order_by_exprs)
- }
+ OrderByKind::All(options) => Ok((1..=order_by_all_column_count)
+ .map(|position| OrderByExpr {
+ expr: SQLExpr::value(Value::Number(position.to_string(), false)),
+ options,
+ with_fill: None,
+ })
+ .collect()),
OrderByKind::Expressions(order_by_exprs) => Ok(order_by_exprs),
}
}
+
+/// Returns the order by expressions from the query with the select expressions.
+pub(crate) fn to_order_by_exprs_with_select(
+ order_by: Option<OrderBy>,
+ select_exprs: Option<&Vec<Expr>>,
+) -> Result<Vec<OrderByExpr>> {
+ let Some(select_exprs) = select_exprs else {
+ return to_order_by_exprs(order_by, 0);
+ };
+ to_order_by_exprs(order_by, select_exprs.len())
+}
diff --git a/datafusion/sqllogictest/test_files/order_by_all.slt b/datafusion/sqllogictest/test_files/order_by_all.slt
new file mode 100644
index 0000000000..88d9360380
--- /dev/null
+++ b/datafusion/sqllogictest/test_files/order_by_all.slt
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ORDER BY ALL expands computed expressions and aliases in select-list order.
+statement ok
+set datafusion.sql_parser.dialect = 'DuckDB';
+
+query II
+SELECT column1 + 1 AS computed, length(column2) AS text_length
+FROM (VALUES (2, 'x'), (1, 'zzz'), (1, 'a')) AS t(column1, column2)
+ORDER BY ALL;
+----
+2 1
+2 3
+3 1
+
+# Direction and null ordering apply to every expanded sort key.
+query IT
+SELECT column1 + 1 AS computed, column2 AS label
+FROM (VALUES (2, 'b'), (NULL, 'n'), (1, 'a')) AS t(column1, column2)
+ORDER BY ALL DESC NULLS FIRST;
+----
+NULL n
+3 b
+2 a
+
+# ORDER BY ALL also applies to the output columns of a set operation.
+query IT
+SELECT 2 AS x, 'b' AS y
+UNION ALL SELECT 1, 'z'
+UNION ALL SELECT 1, 'a'
+ORDER BY ALL;
+----
+1 a
+1 z
+2 b
+
+# Aggregate outputs are sorted by their projected positions too.
+query II
+SELECT column1 AS value, count(*) AS occurrences
+FROM (VALUES (2), (1), (1)) AS t(column1)
+GROUP BY column1
+ORDER BY ALL DESC;
+----
+2 1
+1 2
+
+# Swapped aliases must sort by output positions, not resolve back to input names.
+query II
+SELECT b AS a, a AS b
+FROM (VALUES (2, 1), (1, 2), (1, 1)) AS t(a, b)
+ORDER BY ALL;
+----
+1 1
+1 2
+2 1
+
+# Duplicate output names from a self-join remain unambiguous through ordinals.
+query II
+WITH t(id) AS (VALUES (2), (1))
+SELECT l.id, r.id
+FROM t AS l
+CROSS JOIN t AS r
+ORDER BY ALL;
+----
+1 1
+1 2
+2 1
+2 2
+
+statement ok
+set datafusion.sql_parser.dialect = 'Generic';
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org