You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/10/13 21:49:38 UTC
(spark) branch branch-4.x updated: [SPARK-60193][SQL] Single-pass resolver code health improvements
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 09f113d6252b [SPARK-60193][SQL] Single-pass resolver code health improvements
09f113d6252b is described below
commit 09f113d6252b3222f1c0e2896f39af1c9a53991c
Author: Mihailo Aleksic <mi...@databricks.com>
AuthorDate: Tue Oct 13 21:49:16 2026 +0000
[SPARK-60193][SQL] Single-pass resolver code health improvements
### What changes were proposed in this pull request?
In this PR I propose to introduce some single-pass resolver code health improvements.
### Why are the changes needed?
To improve the code health of the single-pass resolver.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #59399 from mihailoale-db/single-pass-resolver-remove-dbr-leftovers.
Authored-by: Mihailo Aleksic <mi...@databricks.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
(cherry picked from commit 2dad66372a4ee946a96e1249ec1e2cb79342f35b)
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../analysis/resolver/AggregateResolver.scala | 13 ++---
.../analysis/resolver/AttributeScopeStack.scala | 5 +-
.../ExplicitlyUnsupportedResolverFeature.scala | 14 ++---
.../analysis/resolver/ExpressionIdAssigner.scala | 44 ++++++---------
.../analysis/resolver/ExpressionResolver.scala | 5 +-
.../sql/catalyst/analysis/resolver/NameScope.scala | 25 ++-------
.../catalyst/analysis/resolver/PlanRewriter.scala | 2 +-
.../resolver/ProducesUnresolvedSubtree.scala | 8 +--
.../analysis/resolver/ProjectResolver.scala | 1 -
.../catalyst/analysis/resolver/ResolverGuard.scala | 23 +-------
.../catalyst/analysis/resolver/SortResolver.scala | 4 +-
.../resolver/SubqueryExpressionResolver.scala | 65 ++--------------------
.../catalyst/analysis/resolver/SubqueryScope.scala | 9 ++-
.../catalyst/analysis/resolver/ViewResolver.scala | 17 +-----
14 files changed, 56 insertions(+), 179 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala
index da9083f99fd2..e81a36e601ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala
@@ -58,22 +58,21 @@ class AggregateResolver(
/**
* Resolve [[Aggregate]] operator.
*
- * 1. Push [[WindowResolutionContext]] for potential window resolution.
- * 2. Resolve the child (inline table).
- * 3. Clear [[NameScope.availableAliases]]. Those are only relevant for the immediate aggregate
+ * 1. Resolve the child (inline table).
+ * 2. Clear [[NameScope.availableAliases]]. Those are only relevant for the immediate aggregate
* expressions for output prioritization to work correctly in
* [[NameScope.tryResolveMultipartNameByOutput]].
- * 4. Resolve aggregate expressions using [[ExpressionResolver.resolveAggregateExpressions]] and
+ * 3. Resolve aggregate expressions using [[ExpressionResolver.resolveAggregateExpressions]] and
* set [[OperatorResolutionContextStack.ordinalReplacementExpressions]] for grouping
* expressions resolution.
- * 5. If there's just one [[UnresolvedAttribute]] with a single-part name "ALL", expand it using
+ * 4. If there's just one [[UnresolvedAttribute]] with a single-part name "ALL", expand it using
* aggregate expressions which don't contain aggregate functions. There should not exist a
* column with that name in the lower operator's output, otherwise it takes precedence.
- * 6. Resolve grouping expressions using [[ExpressionResolver.resolveGroupingExpressions]]. This
+ * 5. Resolve grouping expressions using [[ExpressionResolver.resolveGroupingExpressions]]. This
* includes alias references to aggregate expressions, which is done in
* [[NameScope.resolveMultipartName]] and replacing [[UnresolvedOrdinals]] with corresponding
* expressions from aggregate list, done in [[OrdinalResolver]].
- * 7. Remove all the unnecessary [[Alias]]es from the grouping (all the aliases) and aggregate
+ * 6. Remove all the unnecessary [[Alias]]es from the grouping (all the aliases) and aggregate
* (keep the outermost one) expressions. This is needed to stay compatible with the
* fixed-point implementation. For example:
*
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala
index ca6bda1373ee..da6e618024ac 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala
@@ -57,9 +57,8 @@ class AttributeScopeStack {
/**
* Check if the `attribute` is present in this stack. We check the current scope by default. If
- * `isOuterReference` is true, we check the outer scopes too. If
- * `spark.sql.optimizer.supportNestedCorrelatedSubqueries.enabled` is true, we check all the
- * outer scopes, otherwise we check the first outer scope only.
+ * `isOuterReference` is true, we check the first outer scope only, because only one level of
+ * correlation is supported.
*/
def contains(attribute: Attribute, isOuterReference: Boolean = false): Boolean = {
if (!isOuterReference) {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala
index fad95a9555fb..8ef63cb5ef58 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala
@@ -20,15 +20,11 @@ package org.apache.spark.sql.catalyst.analysis.resolver
/**
* This is an addon to [[ResolverGuard]] functionality for features that cannot be determined by
* only looking at the unresolved plan. [[Resolver]] will throw this control-flow exception
- * when it encounters some explicitly unsupported feature. Later behavior depends on the value of
- * [[ANALYZER_SINGLE_PASS_RESOLVER_EXPOSE_EXPLICITLY_UNSUPPORTED_FEATURE_IN_DUAL_RUNS]] flag:
- * - If it is true: It will later be caught by [[HybridAnalyzer]] to abort single-pass
- * analysis without comparing single-pass and fixed-point results. The motivation for this
- * feature is the same as for the [[ResolverGuard]] - we want to have an explicit allowlist of
- * unimplemented features that we are aware of, and `UNSUPPORTED_SINGLE_PASS_ANALYZER_FEATURE`
- * will signal us the rest of the gaps.
- * - If it is false: It will be thrown by the [[HybridAnalyzer]] in order to get better sense
- * of coverage.
+ * when it encounters some explicitly unsupported feature. It will later be caught by
+ * [[HybridAnalyzer]] to abort single-pass analysis without comparing single-pass and fixed-point
+ * results. The motivation for this feature is the same as for the [[ResolverGuard]] - we want to
+ * have an explicit allowlist of unimplemented features that we are aware of, and
+ * `UNSUPPORTED_SINGLE_PASS_ANALYZER_FEATURE` will signal us the rest of the gaps.
*
* For example, [[UnresolvedRelation]] can be intermediately resolved by [[ResolveRelations]] as
* [[UnresolvedCatalogRelation]] or a [[View]] (among all others). Say that for now the views
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala
index c8f24b3e2003..f42fa3439953 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala
@@ -223,11 +223,8 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
* plan. In this case we call [[pushMapping]] with `isSubqueryRoot = true` to pass the current
* mapping as outer mapping to the subquery branches. Any subquery branch may reference
* outer attributes, so if `isSubqueryRoot` is `false`, we pass the previous `outerMapping` to
- * lower branches. For every subquery level, if
- * `spark.sql.optimizer.supportNestedCorrelatedSubqueries.enabled` is set to `false`, only
- * one level of correlation is supported and the current `mapping` becomes `outerMapping`
- * for the next level. Otherwise, the current `mapping` is merged with existing
- * `outerMapping` as the new `outerMapping` for the next level.
+ * lower branches. For every subquery level, the current `mapping` becomes the new
+ * `outerMapping` for the next level.
* - Continue remapping expressions until we reach the root of the operator tree.
*/
class ExpressionIdAssigner {
@@ -244,10 +241,8 @@ class ExpressionIdAssigner {
* @param isSubqueryRoot whether the new branch is related to a subquery root.
* If the new branch is not related to a subquery root, we propagate `outerMapping` itself,
* because any nested subquery operator may reference outer attributes.
- * If the new branch is related to a subquery root and
- * `spark.sql.optimizer.supportNestedCorrelatedSubqueries` is turned off, we propagate current
- * `mapping` as the new `outerMapping` for the next level. If the flag is turned on, we
- * merge current `mapping` with existing `outerMapping` as new `outerMapping`.
+ * If the new branch is related to a subquery root, we propagate current `mapping` as the new
+ * `outerMapping` for the next level.
*/
def pushMapping(isSubqueryRoot: Boolean = false): Unit = {
val currentStackEntry = mappingStack.peek()
@@ -270,10 +265,11 @@ class ExpressionIdAssigner {
* @param collectChildMapping whether to collect a child mapping into the current stack entry.
* This is used in multi-child operators to automatically propagate mapped expression IDs
* upwards using [[createMappingFromChildMappings]].
- * @param oldOutputSet a special parameter used when AssignNewExprIds is calling [[popMapping]]
- * for optimized plans. oldOutputSet contains the original output `ExprId`s for the current
- * child plan. If it is not empty, [[popMapping]] filters out Ids not related to the original
- * outputSet and only maintains the mapping from original oldId to new Id and new Id to itself.
+ * @param oldOutputSet a special parameter used when [[popMapping]] is called while reassigning
+ * expression IDs in optimized plans. oldOutputSet contains the original output `ExprId`s for
+ * the current child plan. If it is not empty, [[popMapping]] filters out Ids not related to the
+ * original outputSet and only maintains the mapping from original oldId to new Id and new Id to
+ * itself.
* This is useful because certain optimization plans may contain duplicated `ExprId`s covered
* with aliases from different branches. E.g:
* {{{
@@ -536,21 +532,16 @@ class ExpressionIdAssigner {
* df.groupBy($"a", $"b".as("sum_d")).agg(Map.empty[String, String])
* }}}
*
- * `alwaysUpdateAttributeReferences` and `alwaysMaintainAttributeReferences` are used in
- * `AssignNewExprIds` when reassigning expression ids for optimized plans.
+ * `alwaysUpdateAttributeReferences` and `alwaysMaintainAttributeReferences` are used when
+ * reassigning expression ids for optimized plans.
* Setting `alwaysUpdateAttributeReferences` to true ensures that a new expression id is assigned
* to the input expression when it is not in the current mapping, regardless of whether it is in
* `globalExpressionIds` or not. This is usually used for new attributes generated from special
* operators like `Generate`, `Expand`, `UnionLoop`, etc.
- * For example, when optimization rules duplicate plans and call `AssignNewExprIds` to reassign
- * the expression ids in the duplicate plan, it expects all the attributes in the duplicate plan
- * to get new expression ids except for AggregateExpressions used in Partial Aggregates.
+ * For example, when optimization rules duplicate plans and reassign the expression ids in the
+ * duplicate plan, they expect all the attributes in the duplicate plan to get new expression ids
+ * except for AggregateExpressions used in Partial Aggregates.
* {{{
- * caller:
- * // Some optimization rules which do self joins
- * val duplicate = AssignNewExprId(plan)
- * val res = Join(plan, duplicate)
- *
* plan:
* Project [out#1, null AS NULL#2, 'column' AS column#3]
* +- Generate explode(array(1, 2)), false, [out#1]
@@ -570,8 +561,7 @@ class ExpressionIdAssigner {
* expression id, regardless of whether it is in `globalExpressionIds` or not. This is used to
* keep the intermediate results of [[Partial]]/[[PartialMerge]] mode [[AggregateExpression]]s
* unchanged. Analyzer usecase does not need this feature as [[Partial]]/[[PartialMerge]] mode
- * [[AggregateExpression]] only appear after the optimization rule
- * [[AggregatePushdownThroughJoins]].
+ * [[AggregateExpression]] only appear after optimization.
*/
def mapExpression[NamedExpressionType <: NamedExpression](
originalExpression: NamedExpressionType,
@@ -778,8 +768,8 @@ class ExpressionIdAssigner {
* expression, or return a corresponding new instance of the same attribute, that was previously
* reallocated and is present in the current [[mappingStack]] entry or the outer mapping.
*
- * This function is used in [[AssignNewExprIds]] when the originalAttribute can be either from
- * the current mapping or from the outer mapping.
+ * This function is used when reassigning expression IDs in optimized plans, where the
+ * originalAttribute can be either from the current mapping or from the outer mapping.
*/
def mapExpressionOrOuterReference(
originalAttribute: AttributeReference): AttributeReference = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala
index c763d8323438..9aea89e27043 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala
@@ -1284,9 +1284,8 @@ class ExpressionResolver(
}
/**
- * Returns true if LateralColumnAlias resolution is enabled, current operator is not created
- * because of a generated column and current expression is a grouping one (grouping expressions)
- * can't reference an LCA.
+ * Returns true if LateralColumnAlias resolution is enabled and current expression is not a
+ * grouping one (grouping expressions can't reference an LCA).
*/
private def canLaterallyReferenceColumn: Boolean = {
traversals.current.lcaEnabled &&
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala
index ad232adfd273..78e615c8f74b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala
@@ -1199,9 +1199,7 @@ class NameScopeStack(
/**
* Resolve multipart name into a [[NameTarget]]. The order is as follows:
* - Try current scope. See [[NameScope.resolveMultipartName]] for more details.
- * - Try the closest outer scope and support only one level of correlation
- * if `spark.sql.optimizer.supportNestedCorrelatedSubqueries.enabled`
- * is false. If the flag is true, Try from the closest outer scope to the root scope.
+ * - Try the closest outer scope. Only one level of correlation is supported.
* - Try to resolve `multipartName` as a session (temporary or scripting) variable. See
* [[VariableResolution.resolveMultipartName]] for more details.
*
@@ -1222,10 +1220,7 @@ class NameScopeStack(
* SELECT col1, (SELECT col1 FROM VALUES (2)) AS b FROM VALUES (1)
* }}}
*
- * Correlations beyond one level are supported only when
- * `spark.sql.optimizer.supportNestedCorrelatedSubqueries.enabled` is set to true.
- *
- * If the flag is set to false, we only support one level of correlation:
+ * Correlations beyond one level are not supported:
* {{{
* -- 3 levels, fails with `UNRESOLVED_COLUMN`.
* SELECT (
@@ -1235,16 +1230,6 @@ class NameScopeStack(
* ) FROM VALUES (1) AS t1;
* }}}
*
- * If the flag is set to true, we support correlations beyond one level:
- * {{{
- * -- 3 levels, returns [1, 2, 3].
- * SELECT (
- * SELECT (
- * SELECT t1.col1 FROM VALUES (3) AS t3
- * ) FROM VALUES (2) AS t2
- * ) FROM VALUES (1) AS t1;
- * }}}
- *
* Correlated references are accessible from lower subquery operators:
* {{{
* -- Returns [1, 1]
@@ -1438,10 +1423,8 @@ class NameScopeStack(
}
/**
- * If `spark.sql.optimizer.supportNestedCorrelatedSubqueries.enabled` is set to false, find
- * the nearest outer scope and return it if we are in a subquery. Otherwise, returns all outer
- * scopes for the subquery entry point. The first scope is the nearest outer scope and the last
- * scope is the root scope.
+ * Find the nearest outer scope and return it if we are in a subquery. Only one level of
+ * correlation is supported, so at most one scope is returned.
*/
private def outerScopes: mutable.ArrayBuffer[NameScope] = {
val supportNestedCorrelations = false
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala
index fcb0badf4ce8..f6283a75179b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
/**
* Utility wrapper on top of [[RuleExecutor]], used to apply post-resolution rules on single-pass
- * resolution result. [[SinglePassRewriter]] transforms the plan and the subqueries inside.
+ * resolution result. [[PlanRewriter]] transforms the plan and the subqueries inside.
*/
class PlanRewriter(
planRewriteRules: Seq[Rule[LogicalPlan]],
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala
index 5c5db64c1e93..30c479203ea2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala
@@ -31,10 +31,10 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
* should return identity instead of trying to resolve it.
*
* Note: [[Literal]] children are excluded from tagging to drastically reduce the chance of a data
- * race (see ES-1725904). Unlike other expression nodes, literals are not reallocated during
- * resolution (resolved and unresolved literals are physically the same object), so mutating their
- * tags can cause a data race when the same literal is used concurrently across threads. Since
- * literals are leaf nodes, the boundary tag is unnecessary for them anyway.
+ * race. Unlike other expression nodes, literals are not reallocated during resolution (resolved
+ * and unresolved literals are physically the same object), so mutating their tags can cause a
+ * data race when the same literal is used concurrently across threads. Since literals are leaf
+ * nodes, the boundary tag is unnecessary for them anyway.
*/
trait ProducesUnresolvedSubtree extends ResolvesExpressionChildren {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala
index 4a8b95cffec9..8130d18e3212 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala
@@ -37,7 +37,6 @@ class ProjectResolver(operatorResolver: Resolver, expressionResolver: Expression
/**
* [[Project]] introduces a new scope to resolve its subtree and project list expressions.
- * It also introduces a new [[WindowResolutionContext]] for potential window resolution.
* During the resolution we determine whether the output operator will be [[Aggregate]] or
* [[Project]] (based on the `hasAggregateExpressionsOutsideWindow` flag) and delegate the
* operator building process to an appropriate method in the following way:
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala
index 52826ab3b679..b6c04baca8b8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala
@@ -96,8 +96,6 @@ class ResolverGuard(
* their children. For unimplemented ones, return Some("reason").
*/
private def checkOperator(operator: LogicalPlan): Option[String] = {
- checkOperatorSecondPassAnalysis(operator)
-
operator match {
case unresolvedWith: UnresolvedWith =>
checkUnresolvedWith(unresolvedWith)
@@ -171,9 +169,6 @@ class ResolverGuard(
def unapply(operator: LogicalPlan): Option[String] = checkOperator(operator)
}
- private def checkOperatorSecondPassAnalysis(operator: LogicalPlan): Unit = {
- }
-
/**
* Method used to check if expressions are supported by the new analyzer.
* For LeafNode types, we return None or Some("reason"). For other ones, check their children.
@@ -666,8 +661,6 @@ class ResolverGuard(
} else if (conf.getConf(SQLConf.LEGACY_CTE_PRECEDENCE_POLICY) !=
LegacyBehaviorPolicy.CORRECTED) {
Some("legacyCTEPrecedencePolicy")
- } else if (conf.getConfString("pipelines.id", null) != null) {
- Some("dlt")
} else if (!conf.getConf(SQLConf.PRIORITIZE_ORDINAL_RESOLUTION_IN_SORT)) {
Some("prioritizeOrdinalResolutionInSortDisabled")
} else if (conf.getConf(SQLConf.PERSISTENT_CATALOG_FIRST)) {
@@ -762,8 +755,6 @@ object ResolverGuard {
private val UNSUPPORTED_FUNCTION_NAMES = {
val map = new IdentifierMap[Unit]()
// User info functions are not supported.
- // [[InlineUserInfoExpressions]] cannot be invoked as a post-hoc rule as it produces
- // inconsistent aliases based on the table type (because of the rule ordering in fixed-point).
map += ("current_user", ())
map += ("session_user", ())
map += ("user", ())
@@ -771,24 +762,12 @@ object ResolverGuard {
map += ("session_window", ())
map += ("window", ())
map += ("window_time", ())
- // Functions that are not resolved properly.
// Functions that produce wrong schemas/plans because of alias assignment.
map += ("from_json", ())
}
/**
- * Experimental functions that are supported.
- */
- private val SUPPORTED_EXPERIMENTAL_FUNCTION_NAMES = {
- val map = new IdentifierMap[Unit]()
- map += ("collate", ())
- map += ("schema_of_json", ())
- map += ("schema_of_xml", ())
- }
-
- /**
- * Higher order functions that are supported but guarded under the
- * `ANALYZER_SINGLE_PASS_RESOLVER_ENABLE_HIGHER_ORDER_FUNCTIONS_RESOLUTION` flag.
+ * Higher order functions that are not yet supported by the single-pass resolver.
*/
private val HIGHER_ORDER_FUNCTIONS = {
val map = new IdentifierMap[Unit]()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala
index 1219d16e4c83..19198b7fc30e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala
@@ -61,8 +61,8 @@ class SortResolver(operatorResolver: Resolver, expressionResolver: ExpressionRes
* resolution.
* 2. Resolve order expressions using [[ExpressionResolver.resolveExpressionTreeInOperator]].
* 3. In case order expressions contain only one element, `UnresolvedAttribute(ALL)`, which
- * can't be resolved from current (nor from hidden output) - this is done using
- * [[ResolveAsAllKeyword]], skip previous step and resolve it as an `ALL` keyword (by copying
+ * can't be resolved from current (nor from hidden output) - this is checked using
+ * [[canOrderByAll]], skip previous step and resolve it as an `ALL` keyword (by copying
* child's output and transforming it into attributes).
* 4. In case there are attributes referenced in the order expressions are resolved using
* the hidden output (or in case we order by [[AggregateExpression]]s which are not present
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala
index e83dcb4165fc..574fca29d818 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala
@@ -159,12 +159,11 @@ class SubqueryExpressionResolver(expressionResolver: ExpressionResolver, resolve
* under the subquery root.
* - Fresh [[SubqueryRegistry]] scope to isolate the subquery expression plan resolution.
* - Fresh [[OperatorResolutionContext]] in order to preserve the context of the outer operator.
- * Collect outer expressions because they may be required for [[Window]] resolution. For
- * further context look at [[CollectsWindowSourceExpressions]].
+ * Collect outer references from the resolved subquery plan to use them as the subquery
+ * expression's `outerAttrs`.
*/
private def resolveSubqueryExpressionPlan(
unresolvedSubqueryPlan: LogicalPlan): ResolvedSubqueryExpressionPlan = {
- val parentSubqueryScope = subqueryRegistry.currentScope
expressionIdAssigner.pushMapping(isSubqueryRoot = true)
scopes.pushScope(isSubqueryRoot = true)
cteRegistry.pushScope(isRoot = true)
@@ -178,7 +177,7 @@ class SubqueryExpressionResolver(expressionResolver: ExpressionResolver, resolve
val resolvedSubqueryExpressionPlan = try {
val resolvedPlan = resolver.resolve(unresolvedSubqueryPlan)
- val outerReferences = processOuterReferences(resolvedPlan, parentSubqueryScope)
+ val outerReferences = processOuterReferences(resolvedPlan)
ResolvedSubqueryExpressionPlan(
plan = resolvedPlan,
@@ -201,62 +200,10 @@ class SubqueryExpressionResolver(expressionResolver: ExpressionResolver, resolve
}
/**
- * Return `outerAttrs` for the resolved subquery plan.
- * If `spark.sql.optimizer.supportNestedCorrelatedSubqueries.enabled` is set to false, simply
- * collect the outer references within the subquery plan.
- *
- * Otherwise,
- * 1. Get the outer references within the resolved subquery plan and also get the outer
- * attributes from any nested subqueries.
- * 2. For each outer reference, using the `outerReferenceScopeLevel` map from current subquery
- * scope to check if it has only one level of correlation or beyond one level of correlation.
- * If it is beyond one level of correlation, wrap it with [[OuterScopeReference]] and add it to
- * the parent subquery scope's `outerReferenceScopeLevel` map with level - 1. E.g: For the
- * following query:
- *
- * {{{
- * SELECT *
- * FROM t1
- * WHERE t1.a = (
- * SELECT MAX(t2.a)
- * FROM t2
- * WHERE t2.a = (
- * SELECT MAX(t3.a)
- * FROM t3
- * WHERE t3.b > t2.b AND t3.c > t1.c
- * ) AND t2.b > t1.b
- * }}}
- *
- * After resolving:
- *
- * {{{
- * Aggregate (MAX(t3.a))
- * Filter (t3.b > OuterReference(t2.b) AND t3.c > OuterReference(t1.c))
- * Scan t3
- * }}}
- *
- * The innermost subquery has two outer references: `t2.b(outerScopeReferenceLevel = 1)` and
- * `t1.c(outerScopeReferenceLevel = 2)`. The innermost subquery expression is resolved as
- * `scalar-subquery(outerAttrs = [t2.b, OuterScopeReference(t1.c)])` and
- * `(t1.c.exprId -> outerScopeReferenceLevel = 1)` is added to the `outerReferenceScopeLevel`
- * map of the parent subquery scope.
- * After resolving the parent subquery plan:
- *
- * {{{
- * Aggregate (MAX(t2.a))
- * Filter (t2.a = scalar-subquery(outerAttrs = [t2.b, OuterScopeReference(t1.c)]) AND
- * t2.b > OuterReference(t1.b))
- * Scan t2
- * }}}
- *
- * The collected outer references are `t1.b` and the `OuterScopeReference(t1.c)` from the
- * innermost subquery. Both of them have `outerScopeReferenceLevel = 1`, they are all resolved
- * as one level outer references. The subquery expression is then resolved as
- * `scalar-subquery(outerAttrs = [t1.b, t1.c])`.
+ * Return `outerAttrs` for the resolved subquery plan by collecting the outer references within
+ * the subquery plan. Only one level of correlation is supported.
*/
- private def processOuterReferences(
- resolvedPlan: LogicalPlan,
- parentSubqueryScope: SubqueryScope): Seq[Expression] = {
+ private def processOuterReferences(resolvedPlan: LogicalPlan): Seq[Expression] = {
SubExprUtils.getOuterReferences(resolvedPlan)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala
index 6f83dd0fb6b2..aa0714ba0345 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala
@@ -70,12 +70,11 @@ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, LogicalPl
* WHERE t3.b > t2.b AND t3.c > t1.c
* ) AND t2.b > t1.b
* }}}
- * We get the [[outerScopeLevel]] map for the outer references during attributes resolution
+ * We record the outer scope level for the outer references during attributes resolution
* which indicates how many subquery levels we need to go up to find the attribute reference.
- * We need the outerScopeLevel map when resolving the subquery expressions, where we collect
- * the outer references and wrap the ones cannot be resolved in the current subquery scope or
- * its parent scopes with [[OuterScopeReference]]. More details can be found in
- * `SubqueryExpressionResolver.processOuterReferences`.
+ * Only one level of correlation is currently supported, so the query above is not supported
+ * and the subquery expression's `outerAttrs` are simply the outer references collected from
+ * the resolved subquery plan.
*
* Notes:
* - Spark only supports outer aggregates in the subqueries inside `HAVING`;
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
index 8aa5262c4b59..d82f04ec1537 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
@@ -79,10 +79,6 @@ class ViewResolver(
/**
* Resolve the `unresolvedView` and its underlying plan. This method uses parent [[Resolver]] to
* resolve the view child. [[View]] resolution consists of the following steps:
- * - Check if the view has any row-level security policies or column masking rules applied. If
- * so, throw [[ExplicitlyUnsupportedResolverFeature]].
- * - Check if the view is a materialized view. If so, throw
- * [[ExplicitlyUnsupportedResolverFeature]].
* - Check if the single-pass resolver fully supports the view plan using the [[ResolverGuard]].
* Throw [[ExplicitlyUnsupportedResolverFeature]] if the view plan is not supported.
* - Set the [[ViewResolutionContext]] for the view plan resolution.
@@ -90,16 +86,10 @@ class ViewResolver(
* view.
* - Resolve the view plan using parent [[Resolver]].
* - Sets the views child as analyzed in order to avoid recursing into it using rewrite rules.
- * - Create a new [[CatalogTable]] description for the resolved view based on the original
- * [[UnresolvedRelation.options]], original [[CatalogTable]] description and used
- * [[ViewResolutionContext]].
- * - Return the resolved [[View]] with the resolved child and a new [[CatalogTable]]
- * description.
+ * - Return the resolved [[View]] with the resolved child and the original
+ * [[UnresolvedRelation.options]].
*/
override def resolve(unresolvedView: View): View = {
-
- checkMaterializedView(unresolvedView)
-
val (resolvedChild, usedViewResolutionContext) = withViewResolutionContext(unresolvedView) {
SQLConf.withExistingConf(
View.effectiveSQLConf(
@@ -163,9 +153,6 @@ class ViewResolver(
}
}
}
- private def checkMaterializedView(unresolvedView: View): Unit = {
- }
-
private def checkResolverGuard(unresolvedView: View): Unit = {
val resolverGuard = new ResolverGuard(catalogManager, tracker = tracker)
resolverGuard(unresolvedView).planUnsupportedReason match {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org