You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/05/09 16:57:00 UTC
(spark) branch master updated: [SPARK-52046][SQL] Prettify OuterReference on AggregateExpression names when using toPrettySQL
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new aa89883746b0 [SPARK-52046][SQL] Prettify OuterReference on AggregateExpression names when using toPrettySQL
aa89883746b0 is described below
commit aa89883746b06ab7d12fe450e9596520c75f9718
Author: mihailoale-db <mi...@databricks.com>
AuthorDate: Sat May 10 00:56:44 2025 +0800
[SPARK-52046][SQL] Prettify OuterReference on AggregateExpression names when using toPrettySQL
### What changes were proposed in this pull request?
In this PR I propose that we remove backticks, qualifiers from `OuterReference` over `AggregateExpression` when creating name. In other words, name for `column` which is an outer reference would be `outer(min(column))` instead of `outer('min(col)')`.
We also introduce a flag to guard the behavior as it changes the schema of some specific `LATERAL JOIN` queries.
### Why are the changes needed?
To ease development of single-pass analyzer and improve names in fixed-point implementation.
### Does this PR introduce _any_ user-facing change?
Schema changes in a way that `OuterReference` over `AggregateExpression` name doesn't contain backticks and qualifiers.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50836 from mihailoale-db/prettifyouterrefname.
Authored-by: mihailoale-db <mi...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../apache/spark/sql/catalyst/util/package.scala | 37 ++++++++++++++++++----
.../org/apache/spark/sql/internal/SQLConf.scala | 8 +++++
2 files changed, 38 insertions(+), 7 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index 47e13762ab88..523579810f1a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -27,11 +27,12 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
import org.apache.spark.sql.connector.catalog.MetadataColumn
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{MetadataBuilder, NumericType, StringType, StructType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.{SparkErrorUtils, Utils}
-package object util extends Logging {
+package object util extends Logging with SQLConfHelper {
/** Silences output to stderr or stdout for the duration of f */
def quietly[A](f: => A): A = {
@@ -103,13 +104,27 @@ package object util extends Logging {
private def usePrettyExpression(e: Expression, stripOuterReference: Boolean = true): Expression =
e transform {
case aggregateExpression: AggregateExpression
- if stripOuterReference && SubExprUtils.containsOuter(aggregateExpression) =>
+ if stripOuterReference && conf.getConf(
+ SQLConf.PRETTY_ALIAS_NAME_FOR_CORRELATED_AGGREGATE_FUNCTION
+ ) && SubExprUtils.containsOuter(aggregateExpression) =>
val strippedAggregateExpression = SubExprUtils.stripOuterReference(aggregateExpression)
OuterReference(
new PrettyAttribute(
Alias(
strippedAggregateExpression,
- toPrettySQL(strippedAggregateExpression)
+ toPrettySQL(strippedAggregateExpression, stripOuterReference)
+ )().toAttribute
+ )
+ )
+ case _ @ OuterReference(aggregateExpression: AggregateExpression)
+ if conf.getConf(
+ SQLConf.PRETTY_ALIAS_NAME_FOR_CORRELATED_AGGREGATE_FUNCTION
+ ) =>
+ OuterReference(
+ new PrettyAttribute(
+ Alias(
+ aggregateExpression,
+ toPrettySQL(aggregateExpression, stripOuterReference)
)().toAttribute
)
)
@@ -119,16 +134,24 @@ package object util extends Logging {
case Literal(null, dataType) => PrettyAttribute("NULL", dataType)
case e: GetStructField =>
val name = e.name.getOrElse(e.childSchema(e.ordinal).name)
- PrettyAttribute(usePrettyExpression(e.child).sql + "." + name, e.dataType)
+ PrettyAttribute(
+ usePrettyExpression(e.child, stripOuterReference).sql + "." + name,
+ e.dataType
+ )
case e: GetArrayStructFields =>
- PrettyAttribute(s"${usePrettyExpression(e.child)}.${e.field.name}", e.dataType)
+ PrettyAttribute(
+ s"${usePrettyExpression(e.child, stripOuterReference)}.${e.field.name}",
+ e.dataType
+ )
case r: InheritAnalysisRules =>
PrettyAttribute(
- r.makeSQLString(r.parameters.map(parameter => toPrettySQL(parameter))),
+ r.makeSQLString(
+ r.parameters.map(parameter => toPrettySQL(parameter, stripOuterReference))
+ ),
r.dataType
)
case c: Cast if c.getTagValue(Cast.USER_SPECIFIED_CAST).isEmpty =>
- PrettyAttribute(usePrettyExpression(c.child).sql, c.dataType)
+ PrettyAttribute(usePrettyExpression(c.child, stripOuterReference).sql, c.dataType)
case p: PythonFuncExpression => PrettyPythonUDF(p.name, p.dataType, p.children)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 6679337fd3c5..688b996ec9f5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -5818,6 +5818,14 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
+ val PRETTY_ALIAS_NAME_FOR_CORRELATED_AGGREGATE_FUNCTION =
+ buildConf("spark.sql.prettyAliasNameForCorrelatedAggFunc.enabled")
+ .internal()
+ .doc("When true, use prettified name for correlated aggregate functions.")
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(true)
+
/**
* Holds information about keys that have been deprecated.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org