You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/12/01 06:20:38 UTC
[spark] branch master updated: [SPARK-41338][SQL] Resolve outer references and normal columns in the same analyzer batch
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8d60736abd5 [SPARK-41338][SQL] Resolve outer references and normal columns in the same analyzer batch
8d60736abd5 is described below
commit 8d60736abd56c5642ae4d3616593c94090cdf9ac
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Thu Dec 1 14:20:23 2022 +0800
[SPARK-41338][SQL] Resolve outer references and normal columns in the same analyzer batch
### What changes were proposed in this pull request?
Today, the way we resolve outer references is very inefficient. It invokes the entire analyzer to resolve the subquery plan, then transforms the plan to resolve `UnresolvedAttribute` to outer references. If the plan is still unresolved, repeat the process until the plan is resolved or the plan doesn't change any more. Ideally, we should only invoke the analyzer once to resolve subquery plans.
This PR adds a new rule to resolve outer references, and put it in the main analyzer batch. Then we can safely invoke the analyzer only once.
### Why are the changes needed?
Simplify the subquery resolution code and make it more efficient
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
existing tests
Closes #38851 from cloud-fan/outer.
Authored-by: Wenchen Fan <we...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/analysis/Analyzer.scala | 106 +++++++++++----------
1 file changed, 57 insertions(+), 49 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 1daa8ea36bf..7f66ddaa894 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -298,6 +298,7 @@ class Analyzer(override val catalogManager: CatalogManager)
ResolveOrdinalInOrderByAndGroupBy ::
ResolveAggAliasInGroupBy ::
ResolveMissingReferences ::
+ ResolveOuterReferences ::
ExtractGenerator ::
ResolveGenerate ::
ResolveFunctions ::
@@ -2109,6 +2110,51 @@ class Analyzer(override val catalogManager: CatalogManager)
}
}
+ /**
+ * Resolves `UnresolvedAttribute` to `OuterReference` if we are resolving subquery plans (when
+ * `AnalysisContext.get.outerPlan` is set).
+ */
+ object ResolveOuterReferences extends Rule[LogicalPlan] {
+ override def apply(plan: LogicalPlan): LogicalPlan = {
+ // Only apply this rule if we are resolving subquery plans.
+ if (AnalysisContext.get.outerPlan.isEmpty) return plan
+
+ // We must run these 3 rules first, as they also resolve `UnresolvedAttribute` and have
+ // higher priority than outer reference resolution.
+ val prepared = ResolveAggregateFunctions(ResolveMissingReferences(ResolveReferences(plan)))
+ prepared.resolveOperatorsDownWithPruning(_.containsPattern(UNRESOLVED_ATTRIBUTE)) {
+ // Handle `Generate` specially here, because `Generate.generatorOutput` starts with
+ // `UnresolvedAttribute` but we should never resolve it to outer references. It's a bit
+ // hacky that `Generate` uses `UnresolvedAttribute` to store the generator column names,
+ // we should clean it up later.
+ case g: Generate if g.childrenResolved && !g.resolved =>
+ val newGenerator = g.generator.transformWithPruning(
+ _.containsPattern(UNRESOLVED_ATTRIBUTE))(resolveOuterReference)
+ val resolved = g.copy(generator = newGenerator.asInstanceOf[Generator])
+ resolved.copyTagsFrom(g)
+ resolved
+ case q: LogicalPlan if q.childrenResolved && !q.resolved =>
+ q.transformExpressionsWithPruning(
+ _.containsPattern(UNRESOLVED_ATTRIBUTE))(resolveOuterReference)
+ }
+ }
+
+ private val resolveOuterReference: PartialFunction[Expression, Expression] = {
+ case u @ UnresolvedAttribute(nameParts) => withPosition(u) {
+ try {
+ AnalysisContext.get.outerPlan.get.resolveChildren(nameParts, resolver) match {
+ case Some(resolved) => wrapOuterReference(resolved)
+ case None => u
+ }
+ } catch {
+ case ae: AnalysisException =>
+ logDebug(ae.getMessage)
+ u
+ }
+ }
+ }
+ }
+
/**
* Checks whether a function identifier referenced by an [[UnresolvedFunction]] is defined in the
* function registry. Note that this rule doesn't try to resolve the [[UnresolvedFunction]]. It
@@ -2482,65 +2528,27 @@ class Analyzer(override val catalogManager: CatalogManager)
*/
object ResolveSubquery extends Rule[LogicalPlan] {
/**
- * Resolve the correlated expressions in a subquery, as if the expressions live in the outer
- * plan. All resolved outer references are wrapped in an [[OuterReference]]
- */
- private def resolveOuterReferences(plan: LogicalPlan, outer: LogicalPlan): LogicalPlan = {
- plan.resolveOperatorsDownWithPruning(_.containsPattern(UNRESOLVED_ATTRIBUTE)) {
- case q: LogicalPlan if q.childrenResolved && !q.resolved =>
- q.transformExpressionsWithPruning(_.containsPattern(UNRESOLVED_ATTRIBUTE)) {
- case u @ UnresolvedAttribute(nameParts) =>
- withPosition(u) {
- try {
- outer.resolveChildren(nameParts, resolver) match {
- case Some(outerAttr) => wrapOuterReference(outerAttr)
- case None => u
- }
- } catch {
- case ae: AnalysisException =>
- logDebug(ae.getMessage)
- u
- }
- }
- }
- }
- }
-
- /**
- * Resolves the subquery plan that is referenced in a subquery expression. The normal
- * attribute references are resolved using regular analyzer and the outer references are
- * resolved from the outer plans using the resolveOuterReferences method.
+ * Resolves the subquery plan that is referenced in a subquery expression, by invoking the
+ * entire analyzer recursively. We set outer plan in `AnalysisContext`, so that the analyzer
+ * can resolve outer references.
*
- * Outer references from the correlated predicates are updated as children of
- * Subquery expression.
+ * Outer references of the subquery are updated as children of Subquery expression.
*/
private def resolveSubQuery(
e: SubqueryExpression,
outer: LogicalPlan)(
f: (LogicalPlan, Seq[Expression]) => SubqueryExpression): SubqueryExpression = {
- // Step 1: Resolve the outer expressions.
- var previous: LogicalPlan = null
- var current = e.plan
- do {
- // Try to resolve the subquery plan using the regular analyzer.
- previous = current
- current = AnalysisContext.withOuterPlan(outer) {
- executeSameContext(current)
- }
-
- // Use the outer references to resolve the subquery plan if it isn't resolved yet.
- if (!current.resolved) {
- current = resolveOuterReferences(current, outer)
- }
- } while (!current.resolved && !current.fastEquals(previous))
+ val newSubqueryPlan = AnalysisContext.withOuterPlan(outer) {
+ executeSameContext(e.plan)
+ }
- // Step 2: If the subquery plan is fully resolved, pull the outer references and record
+ // If the subquery plan is fully resolved, pull the outer references and record
// them as children of SubqueryExpression.
- if (current.resolved) {
+ if (newSubqueryPlan.resolved) {
// Record the outer references as children of subquery expression.
- f(current, SubExprUtils.getOuterReferences(current))
+ f(newSubqueryPlan, SubExprUtils.getOuterReferences(newSubqueryPlan))
} else {
- e.withNewPlan(current)
+ e.withNewPlan(newSubqueryPlan)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org