You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/04/08 14:32:47 UTC
(spark) branch master updated: [SPARK-51611][SQL] New iteration of single-pass Analyzer functionality
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9efddcf4a50b [SPARK-51611][SQL] New iteration of single-pass Analyzer functionality
9efddcf4a50b is described below
commit 9efddcf4a50b1b809cd9ae0330a35808b1344b3a
Author: Vladimir Golubev <vl...@databricks.com>
AuthorDate: Tue Apr 8 22:32:33 2025 +0800
[SPARK-51611][SQL] New iteration of single-pass Analyzer functionality
### What changes were proposed in this pull request?
New single-pass Analyzer features:
- GROUP BY
- ORDER BY
- JOIN
- EXCEPT
- INTERSECT
- Correlated subqueries
- Other small features and bugfixes
### Why are the changes needed?
To replace the fixed-point Analyzer with the single-pass one.
### Does this PR introduce _any_ user-facing change?
No, single-pass Analyzer is not yet enabled by default.
### How was this patch tested?
Logical plans were compared in tests with `ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER`.
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #50406 from vladimirg-db/vladimir-golubev_data/single-pass-analyzer/new-iteration-of-features.
Authored-by: Vladimir Golubev <vl...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../resolver/AggregateExpressionResolver.scala | 205 +++-
.../analysis/resolver/AggregateResolver.scala | 332 ++++++
.../catalyst/analysis/resolver/AliasResolver.scala | 58 +-
.../analysis/resolver/AnalyzerBridgeState.scala | 16 +-
.../analysis/resolver/AttributeScopeStack.scala | 69 +-
.../resolver/AutoGeneratedAliasProvider.scala | 77 ++
.../resolver/BinaryArithmeticResolver.scala | 5 +-
.../resolver/BridgedRelationMetadataProvider.scala | 27 +-
.../resolver/ConditionalExpressionResolver.scala | 2 +-
.../resolver/CreateNamedStructResolver.scala | 1 +
.../sql/catalyst/analysis/resolver/CteScope.scala | 135 ++-
.../resolver/DelegatesResolutionToExtensions.scala | 1 -
.../ExplicitlyUnsupportedResolverFeature.scala | 2 +-
.../analysis/resolver/ExpressionIdAssigner.scala | 372 +++++--
.../resolver/ExpressionResolutionContext.scala | 71 +-
.../resolver/ExpressionResolutionValidator.scala | 78 +-
.../analysis/resolver/ExpressionResolver.scala | 696 +++++++++---
.../resolver/ExpressionTreeTraversal.scala | 96 ++
.../analysis/resolver/FilterResolver.scala | 104 ++
.../analysis/resolver/FunctionResolver.scala | 9 +-
.../analysis/resolver/HybridAnalyzer.scala | 55 +-
.../resolver/IdentifierAndCteSubstituor.scala | 184 +++
.../catalyst/analysis/resolver/JoinResolver.scala | 284 +++++
.../resolver/LimitLikeExpressionValidator.scala | 151 +++
.../analysis/resolver/MetadataResolver.scala | 71 +-
.../sql/catalyst/analysis/resolver/NameScope.scala | 614 ++++++++--
.../catalyst/analysis/resolver/NameTarget.scala | 9 +-
.../catalyst/analysis/resolver/PlanRewriter.scala | 62 ++
.../analysis/resolver/PredicateResolver.scala | 12 +-
.../resolver/ProducesUnresolvedSubtree.scala | 31 +-
.../analysis/resolver/ProjectResolver.scala | 186 +++-
.../analysis/resolver/PruneMetadataColumns.scala | 148 +++
...ndeterministicExpressionInExpressionTree.scala} | 36 +-
.../resolver/RelationMetadataProvider.scala | 10 +-
.../analysis/resolver/ResolutionValidator.scala | 149 ++-
.../resolver/ResolvedAggregateExpressions.scala | 43 +
.../analysis/resolver/ResolvedProjectList.scala | 3 -
....scala => ResolvedSubqueryExpressionPlan.scala} | 29 +-
.../sql/catalyst/analysis/resolver/Resolver.scala | 509 +++++----
.../catalyst/analysis/resolver/ResolverGuard.scala | 377 +++++--
.../analysis/resolver/ResolverMetricTracker.scala | 77 ++
.../analysis/resolver/ResolverRunner.scala | 75 +-
.../resolver/ResolvesExpressionChildren.scala | 91 +-
.../resolver/ResolvesNameByHiddenOutput.scala | 186 ++++
.../analysis/resolver/SemanticComparator.scala | 71 ++
...solver.scala => SetOperationLikeResolver.scala} | 263 +++--
.../catalyst/analysis/resolver/SortResolver.scala | 397 +++++++
.../resolver/SubqueryExpressionResolver.scala | 200 ++++
.../catalyst/analysis/resolver/SubqueryScope.scala | 137 +++
.../analysis/resolver/TimeAddResolver.scala | 8 +-
.../resolver/TimezoneAwareExpressionResolver.scala | 27 +-
...ssionChildren.scala => TryExtractOrdinal.scala} | 23 +-
.../analysis/resolver/UnaryMinusResolver.scala | 4 +-
...ildren.scala => UnresolvedCteRelationRef.scala} | 22 +-
...UnsupportedExpressionInOperatorValidation.scala | 68 ++
.../catalyst/analysis/resolver/ViewResolver.scala | 35 +-
.../org/apache/spark/sql/internal/SQLConf.scala | 41 +
.../LimitLikeExpressionValidatorSuite.scala | 109 ++
.../AggregateExpressionResolverSuite.scala | 11 +-
.../analysis/resolver/AggregateResolverSuite.scala | 149 +++
.../sql/analysis/resolver/AliasResolverSuite.scala | 83 ++
...ExplicitlyUnsupportedResolverFeatureSuite.scala | 13 +-
.../resolver/ExpressionIdAssignerSuite.scala | 1168 ++++++++++++++++----
.../analysis/resolver/HybridAnalyzerSuite.scala | 22 +-
.../IdentifierAndCteSubstitutorSuite.scala | 255 +++++
.../sql/analysis/resolver/NameScopeSuite.scala | 379 +++++--
.../sql/analysis/resolver/ResolverGuardSuite.scala | 134 ++-
.../sql/analysis/resolver/ViewResolverSuite.scala | 17 +-
68 files changed, 7891 insertions(+), 1493 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateExpressionResolver.scala
index ccbb82a0bac3..63642e3ab1f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateExpressionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateExpressionResolver.scala
@@ -17,15 +17,19 @@
package org.apache.spark.sql.catalyst.analysis.resolver
+import java.util.IdentityHashMap
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.{
- AnalysisErrorAt,
AnsiTypeCoercion,
CollationTypeCoercion,
TypeCoercion
}
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, OuterReference, SubExprUtils}
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, Project}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Sort}
+import org.apache.spark.sql.errors.QueryCompilationErrors
/**
* A resolver for [[AggregateExpression]]s which are introduced while resolving an
@@ -37,6 +41,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, Project}
* in the [[FunctionResolver]].
*/
class AggregateExpressionResolver(
+ operatorResolver: Resolver,
expressionResolver: ExpressionResolver,
timezoneAwareExpressionResolver: TimezoneAwareExpressionResolver)
extends TreeNodeResolver[AggregateExpression, Expression]
@@ -53,85 +58,175 @@ class AggregateExpressionResolver(
private val expressionResolutionContextStack =
expressionResolver.getExpressionResolutionContextStack
+ private val subqueryRegistry = operatorResolver.getSubqueryRegistry
+ private val traversals = expressionResolver.getExpressionTreeTraversals
+ private val autoGeneratedAliasProvider = new AutoGeneratedAliasProvider(
+ expressionResolver.getExpressionIdAssigner
+ )
/**
- * Resolves the given [[AggregateExpression]] by applying:
- * - Type coercion rules
- * - Validity checks. Those include:
- * - Whether the [[AggregateExpression]] is under a valid operator.
- * - Whether there is a nested [[AggregateExpression]].
- * - Whether there is a nondeterministic child.
- * - Updates to the [[ExpressionResolver.expressionResolutionContextStack]]
+ * Resolves the given [[AggregateExpression]]:
+ * - Apply type coercion rules;
+ * - Validate the [[AggregateExpression]]:
+ * 1. Nested aggregate functions are not allowed;
+ * 2. Nondeterministic expressions in the subtree of a related aggregate function are not
+ * allowed;
+ * 3. The mix of outer and local references is not allowed.
+ * - Update the [[ExpressionResolver.expressionResolutionContextStack]];
+ * - Handle the outer aggregate expression in a special way (see
+ * [[handleOuterAggregateExpression]]).
*/
override def resolve(aggregateExpression: AggregateExpression): Expression = {
+ val expressionResolutionContext = expressionResolutionContextStack.peek()
+
val aggregateExpressionWithTypeCoercion =
- withResolvedChildren(aggregateExpression, typeCoercionResolver.resolve)
+ withResolvedChildren(aggregateExpression, typeCoercionResolver.resolve _)
+ .asInstanceOf[AggregateExpression]
- throwIfNotUnderValidOperator(aggregateExpression)
- throwIfNestedAggregateExists(aggregateExpressionWithTypeCoercion)
- throwIfHasNondeterministicChildren(aggregateExpressionWithTypeCoercion)
+ validateResolvedAggregateExpression(aggregateExpressionWithTypeCoercion)
- expressionResolutionContextStack
- .peek()
- .hasAggregateExpressionsInASubtree = true
+ expressionResolutionContext.hasAggregateExpressions = true
// There are two different cases that we handle regarding the value of the flag:
//
// - We have an attribute under an `AggregateExpression`:
// {{{ SELECT COUNT(col1) FROM VALUES (1); }}}
- // In this case, value of the `hasAttributeInASubtree` flag should be `false` as it
- // indicates whether there is an attribute in the subtree that's not `AggregateExpression`
- // so we can throw the `MISSING_GROUP_BY` exception appropriately.
+ // In this case, value of the `hasAttributeOutsideOfAggregateExpressions` flag should be
+ // `false` as it indicates whether there is an attribute in the subtree that's not
+ // `AggregateExpression` so we can throw the `MISSING_GROUP_BY` exception appropriately.
//
// - In the following example:
// {{{ SELECT COUNT(*), col1 + 1 FROM VALUES (1); }}}
// It would be `true` as described above.
- expressionResolutionContextStack.peek().hasAttributeInASubtree = false
+ expressionResolutionContext.hasAttributeOutsideOfAggregateExpressions = false
- aggregateExpressionWithTypeCoercion
+ if (expressionResolutionContext.hasOuterReferences) {
+ handleOuterAggregateExpression(aggregateExpressionWithTypeCoercion)
+ } else {
+ traversals.current.parentOperator match {
+ case Sort(_, _, aggregate: Aggregate, _) =>
+ handleAggregateExpressionInSort(aggregateExpressionWithTypeCoercion, aggregate)
+ case other =>
+ aggregateExpressionWithTypeCoercion
+ }
+ }
}
- private def throwIfNotUnderValidOperator(aggregateExpression: AggregateExpression): Unit = {
- expressionResolver.getParentOperator.get match {
- case _: Aggregate | _: Project =>
- case filter: Filter =>
- filter.failAnalysis(
- errorClass = "INVALID_WHERE_CONDITION",
- messageParameters = Map(
- "condition" -> toSQLExpr(filter.condition),
- "expressionList" -> Seq(aggregateExpression).mkString(", ")
- )
- )
- case other =>
- other.failAnalysis(
- errorClass = "UNSUPPORTED_EXPR_FOR_OPERATOR",
- messageParameters = Map(
- "invalidExprSqls" -> Seq(aggregateExpression).mkString(", ")
- )
- )
+ private def validateResolvedAggregateExpression(
+ aggregateExpression: AggregateExpression): Unit = {
+ if (expressionResolutionContextStack.peek().hasAggregateExpressions) {
+ throwNestedAggregateFunction(aggregateExpression)
+ }
+
+ val nonDeterministicChild =
+ aggregateExpression.aggregateFunction.children.collectFirst {
+ case child if !child.deterministic => child
+ }
+ if (nonDeterministicChild.nonEmpty) {
+ throwAggregateFunctionWithNondeterministicExpression(
+ aggregateExpression,
+ nonDeterministicChild.get
+ )
}
}
- private def throwIfNestedAggregateExists(aggregateExpression: AggregateExpression): Unit = {
- if (expressionResolutionContextStack
- .peek()
- .hasAggregateExpressionsInASubtree) {
- aggregateExpression.failAnalysis(
- errorClass = "NESTED_AGGREGATE_FUNCTION",
- messageParameters = Map.empty
+ /**
+ * If the [[AggregateExpression]] has outer references in its subtree, we need to handle it in a
+ * special way. The whole process is explained in the [[SubqueryScope]] scaladoc, but in short
+ * we need to:
+ * - Validate that we don't have local references in this subtree;
+ * - Create a new subtree without [[OuterReference]]s;
+ * - Alias this subtree and put it inside the current [[SubqueryScope]];
+ * - If outer aggregates are allowed, replace the [[AggregateExpression]] with an
+ * [[OuterReference]] to the auto-generated [[Alias]] that we created. This alias will later
+ * be injected into the outer [[Aggregate]];
+ * - In case we have an [[AggregateExpression]] inside a [[Sort]] operator, we need to handle it
+ * in a special way (see [[handleAggregateExpressionInSort]] for more details).
+ * - Return the original [[AggregateExpression]] otherwise. This is done to stay compatible
+ * with the fixed-point Analyzer - a proper exception will be thrown later by
+ * [[ValidateSubqueryExpression]].
+ */
+ private def handleOuterAggregateExpression(
+ aggregateExpression: AggregateExpression): Expression = {
+ if (expressionResolutionContextStack.peek().hasLocalReferences) {
+ throw QueryCompilationErrors.mixedRefsInAggFunc(
+ aggregateExpression.sql,
+ aggregateExpression.origin
+ )
+ }
+
+ if (subqueryRegistry.currentScope.isOuterAggregateAllowed) {
+ val aggregateExpressionWithStrippedOuterReferences =
+ SubExprUtils.stripOuterReference(aggregateExpression)
+
+ val outerAggregateExpressionAlias = autoGeneratedAliasProvider.newOuterAlias(
+ child = aggregateExpressionWithStrippedOuterReferences
)
+ subqueryRegistry.currentScope.addOuterAggregateExpression(
+ outerAggregateExpressionAlias,
+ aggregateExpressionWithStrippedOuterReferences
+ )
+
+ OuterReference(outerAggregateExpressionAlias.toAttribute)
+ } else {
+ aggregateExpression
}
}
- private def throwIfHasNondeterministicChildren(aggregateExpression: AggregateExpression): Unit = {
- aggregateExpression.aggregateFunction.children.foreach(child => {
- if (!child.deterministic) {
- child.failAnalysis(
- errorClass = "AGGREGATE_FUNCTION_WITH_NONDETERMINISTIC_EXPRESSION",
- messageParameters = Map("sqlExpr" -> toSQLExpr(aggregateExpression))
- )
+ /**
+ * If we order by an [[AggregateExpression]] which is not present in the [[Aggregate]] operator
+ * (child of the [[Sort]]) we have to extract it (by adding it to the
+ * `extractedAggregateExpressionAliases` list of the current expression tree traversal) and add
+ * it to the [[Aggregate]] operator afterwards (this is done in the [[SortResolver]]).
+ */
+ private def handleAggregateExpressionInSort(
+ aggregateExpression: Expression,
+ aggregate: Aggregate): Expression = {
+ val aliasChildToAliasInAggregateExpressions = new IdentityHashMap[Expression, Alias]
+ val aggregateExpressionsSemanticComparator = new SemanticComparator(
+ aggregate.aggregateExpressions.collect {
+ case alias: Alias =>
+ aliasChildToAliasInAggregateExpressions.put(alias.child, alias)
+ alias.child
}
- })
+ )
+
+ val referencedAggregateExpression =
+ aggregateExpressionsSemanticComparator.collectFirst(aggregateExpression)
+
+ referencedAggregateExpression match {
+ case Some(expression) =>
+ aliasChildToAliasInAggregateExpressions.get(expression) match {
+ case null =>
+ throw SparkException.internalError(
+ s"No parent alias for expression $expression while extracting aggregate" +
+ s"expressions in Sort operator."
+ )
+ case alias: Alias => alias.toAttribute
+ }
+ case None =>
+ val alias = autoGeneratedAliasProvider.newAlias(child = aggregateExpression)
+ traversals.current.extractedAggregateExpressionAliases.add(alias)
+ alias.toAttribute
+ }
+ }
+
+ private def throwNestedAggregateFunction(aggregateExpression: AggregateExpression): Nothing = {
+ throw new AnalysisException(
+ errorClass = "NESTED_AGGREGATE_FUNCTION",
+ messageParameters = Map.empty,
+ origin = aggregateExpression.origin
+ )
+ }
+
+ private def throwAggregateFunctionWithNondeterministicExpression(
+ aggregateExpression: AggregateExpression,
+ nonDeterministicChild: Expression): Nothing = {
+ throw new AnalysisException(
+ errorClass = "AGGREGATE_FUNCTION_WITH_NONDETERMINISTIC_EXPRESSION",
+ messageParameters = Map("sqlExpr" -> toSQLExpr(aggregateExpression)),
+ origin = nonDeterministicChild.origin
+ )
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala
new file mode 100644
index 000000000000..f39e036807b6
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AggregateResolver.scala
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.LinkedHashMap
+
+import scala.jdk.CollectionConverters._
+
+import org.apache.spark.sql.catalyst.analysis.{
+ withPosition,
+ AnalysisErrorAt,
+ NondeterministicExpressionCollection,
+ UnresolvedAttribute
+}
+import org.apache.spark.sql.catalyst.expressions.{
+ Alias,
+ Expression,
+ ExprUtils,
+ IntegerLiteral,
+ Literal,
+ NamedExpression
+}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Project}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.types.IntegerType
+
+/**
+ * Resolves an [[Aggregate]] by resolving its child, aggregate expressions and grouping
+ * expressions. Updates the [[NameScopeStack]] with its output and performs validation
+ * related to [[Aggregate]] resolution.
+ */
+class AggregateResolver(operatorResolver: Resolver, expressionResolver: ExpressionResolver)
+ extends TreeNodeResolver[Aggregate, Aggregate] {
+ private val scopes = operatorResolver.getNameScopes
+
+ /**
+ * Resolve [[Aggregate]] operator.
+ *
+ * 1. Resolve the child (inline table).
+ * 2. Resolve aggregate expressions using [[ExpressionResolver.resolveAggregateExpressions]].
+ * 3. If there's just one [[UnresolvedAttribute]] with a single-part name "ALL", expand it using
+ * aggregate expressions which don't contain aggregate functions. There should not exist a
+ * column with that name in the lower operator's output, otherwise it takes precedence.
+ * 4. Resolve grouping expressions using [[ExpressionResolver.resolveGroupingExpressions]]. This
+ * includes alias references to aggregate expressions, which is done in
+ * [[NameScope.resolveMultipartName]].
+ * 5. Substitute ordinals with aggregate expressions in appropriate places.
+ * 6. Substitute non-deterministic expressions with derived attribute references to an
+ * artificial [[Project]] list.
+ *
+ * At the end of resolution we validate the [[Aggregate]] using the
+ * [[ExprUtils.assertValidAggregation]], update the `scopes` with the output of [[Aggregate]] and
+ * return the result.
+ */
+ def resolve(unresolvedAggregate: Aggregate): Aggregate = {
+ val resolvedAggregate = scopes.withNewScope() {
+ val resolvedChild = operatorResolver.resolve(unresolvedAggregate.child)
+
+ val resolvedAggregateExpressions = expressionResolver.resolveAggregateExpressions(
+ unresolvedAggregate.aggregateExpressions,
+ unresolvedAggregate
+ )
+
+ val resolvedGroupingExpressions =
+ if (canGroupByAll(unresolvedAggregate.groupingExpressions)) {
+ tryResolveGroupByAll(
+ resolvedAggregateExpressions,
+ unresolvedAggregate
+ )
+ } else {
+ val partiallyResolvedGroupingExpressions = expressionResolver.resolveGroupingExpressions(
+ unresolvedAggregate.groupingExpressions,
+ unresolvedAggregate
+ )
+ withPosition(unresolvedAggregate) {
+ tryReplaceOrdinalsInGroupingExpressions(
+ partiallyResolvedGroupingExpressions,
+ resolvedAggregateExpressions
+ )
+ }
+ }
+
+ val resolvedAggregate = unresolvedAggregate.copy(
+ groupingExpressions = resolvedGroupingExpressions,
+ aggregateExpressions = resolvedAggregateExpressions.expressions,
+ child = resolvedChild
+ )
+
+ tryPullOutNondeterministic(resolvedAggregate)
+ }
+
+ // TODO: This validation function does a post-traversal. This is discouraged in single-pass
+ // Analyzer.
+ ExprUtils.assertValidAggregation(resolvedAggregate)
+
+ scopes.overwriteOutputAndExtendHiddenOutput(
+ output = resolvedAggregate.aggregateExpressions.map(_.toAttribute)
+ )
+
+ resolvedAggregate
+ }
+
+ /**
+ * Replaces the ordinals with the actual expressions from the resolved aggregate expression list
+ * or throws if any of aggregate expression are irregular.
+ * There are three cases:
+ * - If the aggregate expression referenced by the ordinal is a [[Literal]] with the Integer
+ * data type - preserve the ordinal literal in order to pass logical plan comparison.
+ * - If [[SQLConf.groupByOrdinal]] flag is set to false, treat the grouping expression as
+ * a [[Literal]] instead of ordinal.
+ * - If aggregate expression is an [[Alias]] return [[Alias.child]].
+ * - Otherwise, replace the ordinal with the aggregate expression.
+ * Remove all the leftover [[Alias]]es at the end of resolution.
+ * For the query:
+ *
+ * {{{ SELECT col1 + 1, col1, col2 FROM VALUES(1, 2) GROUP BY 2, col2, 3; }}}
+ *
+ * It would replace `2` with the `col1` and `3` with `col2` so the final grouping expression list
+ * would be: [col1, col2, col2].
+ *
+ * In case of having an integer [[Literal]] in the aggregate expressions which is referenced by an
+ * ordinal, example and final grouping expression list are the following:
+ * - Example:
+ * {{{ SELECT col1 + 1, col1, 10 FROM VALUES(1, 2) GROUP BY 2, col2, 3; }}}
+ * - Grouping expressions:
+ * [col1, col2, 3] // we preserve the ordinal instead of replacing it.
+ */
+ private def tryReplaceOrdinalsInGroupingExpressions(
+ groupingExpressions: Seq[Expression],
+ aggregateExpressions: ResolvedAggregateExpressions): Seq[Expression] = {
+ val aggregateExpressionsArray = aggregateExpressions.expressions.toArray
+ val groupByOrdinal = conf.groupByOrdinal
+ groupingExpressions.map { expression =>
+ val maybeGroupByOrdinal = if (groupByOrdinal) {
+ tryReplaceOrdinalInGroupingExpression(
+ expression,
+ aggregateExpressionsArray,
+ aggregateExpressions
+ )
+ } else {
+ expression
+ }
+ maybeGroupByOrdinal match {
+ case alias: Alias =>
+ alias.child
+ case other => other
+ }
+ }
+ }
+
+ private def tryReplaceOrdinalInGroupingExpression(
+ groupingExpression: Expression,
+ aggregateExpressionsArray: Array[NamedExpression],
+ resolvedAggregateExpressions: ResolvedAggregateExpressions): Expression = {
+ TryExtractOrdinal(groupingExpression) match {
+ case Some(ordinal) =>
+ if (ordinal > aggregateExpressionsArray.length) {
+ throw QueryCompilationErrors.groupByPositionRangeError(
+ ordinal,
+ aggregateExpressionsArray.length
+ )
+ }
+
+ if (resolvedAggregateExpressions.hasStar) {
+ throw QueryCompilationErrors.starNotAllowedWhenGroupByOrdinalPositionUsedError()
+ }
+
+ if (resolvedAggregateExpressions.expressionIndexesWithAggregateFunctions
+ .contains(ordinal - 1)) {
+ throw QueryCompilationErrors.groupByPositionRefersToAggregateFunctionError(
+ ordinal,
+ aggregateExpressionsArray(ordinal - 1)
+ )
+ }
+
+ val resolvedOrdinalAggregateExpression =
+ aggregateExpressionsArray(ordinal - 1) match {
+ case alias: Alias =>
+ alias.child
+ case other => other
+ }
+
+ resolvedOrdinalAggregateExpression match {
+ case Literal(_: Int, IntegerType) =>
+ Literal(ordinal)
+ case other => other
+ }
+ case None => groupingExpression
+ }
+ }
+
+ /**
+ * Resolve `GROUP BY ALL`.
+ *
+ * Examples below show which queries should be resolved with `tryResolveGroupByAll` and which
+ * should be resolved generically (using the [[ExpressionResolver.resolveGroupingExpressions]]):
+ *
+ * Example 1:
+ *
+ * {{{
+ * -- Table `table_1` has a column `all`.
+ * SELECT * from table_1 GROUP BY all;
+ * }}}
+ * this one should be grouped by the column `all`.
+ *
+ * Example 2:
+ *
+ * {{{
+ * -- Table `table_2` doesn't have a column `all`.
+ * SELECT * from table_2 GROUP BY all;
+ * }}}
+ * this one should be grouped by all the columns from `table_1`.
+ *
+ * Example 3:
+ *
+ * {{{
+ * -- Table `table_3` doesn't have a column `all` and there other grouping expressions.
+ * SELECT * from table_3 GROUP BY all, column;
+ * }}}
+ * this one should be grouped by column `all` which doesn't exist so `UNRESOLVED_COLUMN`
+ * exception is thrown.
+ *
+ * Example 4:
+ *
+ * {{{ SELECT col1, col2 + 1, COUNT(col1 + 1) FROM VALUES(1, 2) GROUP BY ALL; }}}
+ * this one should be grouped by keyword `ALL`. It means that the grouping expressions list is
+ * going to contain all the aggregate expressions that don't have aggregate expressions in their
+ * subtrees. The grouping expressions list will be [col1, col2 + 1], and COUNT(col1 + 1) won't be
+ * included, being an [[AggregateExpression]].
+ *
+ * Example 5:
+ *
+ * {{{ SELECT col1, 5 FROM VALUES(1) GROUP BY ALL; }}}
+ * this one should be grouped by keyword `ALL`. If there is an aggregate expression which is a
+ * [[Literal]] with the Integer data type - preserve the ordinal literal in order to pass logical
+ * plan comparison. The grouping expressions list will be [col1, 2].
+ */
+ private def tryResolveGroupByAll(
+ aggregateExpressions: ResolvedAggregateExpressions,
+ aggregate: Aggregate): Seq[Expression] = {
+ if (aggregateExpressions.resolvedExpressionsWithoutAggregates.isEmpty &&
+ aggregateExpressions.hasAttributeOutsideOfAggregateExpressions) {
+ aggregate.failAnalysis(
+ errorClass = "UNRESOLVED_ALL_IN_GROUP_BY",
+ messageParameters = Map.empty
+ )
+ }
+
+ aggregateExpressions.resolvedExpressionsWithoutAggregates.zipWithIndex.map {
+ case (expression, index) =>
+ expression match {
+ case IntegerLiteral(_) =>
+ Literal(index + 1)
+ case _ => expression
+ }
+ }
+ }
+
+ /**
+ * In case there are non-deterministic expressions in either `groupingExpressions` or
+ * `aggregateExpressions` replace them with attributes created out of corresponding
+ * non-deterministic expression. Example:
+ *
+ * {{{ SELECT RAND() GROUP BY 1; }}}
+ *
+ * This query would have the following analyzed plan:
+ * Aggregate(
+ * groupingExpressions = [AttributeReference(_nonDeterministic)]
+ * aggregateExpressions = [Alias(AttributeReference(_nonDeterministic), `rand()`)]
+ * child = Project(
+ * projectList = [Alias(Rand(...), `_nondeterministic`)]
+ * child = OneRowRelation
+ * )
+ * )
+ */
+ private def tryPullOutNondeterministic(aggregate: Aggregate): Aggregate = {
+ val nondeterministicToAttributes: LinkedHashMap[Expression, NamedExpression] =
+ NondeterministicExpressionCollection.getNondeterministicToAttributes(
+ aggregate.groupingExpressions
+ )
+
+ if (!nondeterministicToAttributes.isEmpty) {
+ val newChild = Project(
+ scopes.current.output ++ nondeterministicToAttributes.values.asScala.toSeq,
+ aggregate.child
+ )
+ val resolvedAggregateExpressions = aggregate.aggregateExpressions.map { expression =>
+ PullOutNondeterministicExpressionInExpressionTree(expression, nondeterministicToAttributes)
+ }
+ val resolvedGroupingExpressions = aggregate.groupingExpressions.map { expression =>
+ PullOutNondeterministicExpressionInExpressionTree(
+ expression,
+ nondeterministicToAttributes
+ )
+ }
+ aggregate.copy(
+ groupingExpressions = resolvedGroupingExpressions,
+ aggregateExpressions = resolvedAggregateExpressions,
+ child = newChild
+ )
+ } else {
+ aggregate
+ }
+ }
+
+ private def canGroupByAll(expressions: Seq[Expression]): Boolean = {
+ val isOrderByAll = expressions match {
+ case Seq(unresolvedAttribute: UnresolvedAttribute) =>
+ unresolvedAttribute.equalsIgnoreCase("ALL")
+ case _ => false
+ }
+ isOrderByAll && scopes.current
+ .resolveMultipartName(Seq("ALL"))
+ .candidates
+ .isEmpty
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AliasResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AliasResolver.scala
index ed013232ac84..06b340738d32 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AliasResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AliasResolver.scala
@@ -35,9 +35,10 @@ class AliasResolver(expressionResolver: ExpressionResolver)
* resolution, after which they will be removed in the post-processing phase.
*/
override def resolve(unresolvedAlias: UnresolvedAlias): NamedExpression =
- scopes.top.lcaRegistry.withNewLcaScope {
+ scopes.current.lcaRegistry.withNewLcaScope {
val aliasWithResolvedChildren =
- withResolvedChildren(unresolvedAlias, expressionResolver.resolve)
+ withResolvedChildren(unresolvedAlias, expressionResolver.resolve _)
+ .asInstanceOf[UnresolvedAlias]
val resolvedAlias =
AliasResolution.resolve(aliasWithResolvedChildren).asInstanceOf[NamedExpression]
@@ -48,9 +49,7 @@ class AliasResolver(expressionResolver: ExpressionResolver)
s"unsupported expression: ${multiAlias.getClass.getName}"
)
case alias: Alias =>
- expressionResolver.getExpressionIdAssigner
- .mapExpression(alias)
- .asInstanceOf[Alias]
+ expressionResolver.getExpressionIdAssigner.mapExpression(alias)
}
}
@@ -59,11 +58,50 @@ class AliasResolver(expressionResolver: ExpressionResolver)
* resolve its children and afterwards reassign exprId to the resulting [[Alias]].
*/
def handleResolvedAlias(alias: Alias): Alias = {
- scopes.top.lcaRegistry.withNewLcaScope {
- val aliasWithResolvedChildren = withResolvedChildren(alias, expressionResolver.resolve)
- expressionResolver.getExpressionIdAssigner
- .mapExpression(aliasWithResolvedChildren)
- .asInstanceOf[Alias]
+ val resolvedAlias = scopes.current.lcaRegistry.withNewLcaScope {
+ val aliasWithResolvedChildren =
+ withResolvedChildren(alias, expressionResolver.resolve _).asInstanceOf[Alias]
+
+ expressionResolver.getExpressionIdAssigner.mapExpression(aliasWithResolvedChildren)
}
+
+ collapseAlias(resolvedAlias)
}
+
+ /**
+ * In case where there are two explicit [[Alias]]es, one on top of the other, remove the bottom
+ * one. For the example bellow:
+ *
+ * - df.select($"column".as("alias_1").as("alias_2"))
+ *
+ * the plan is:
+ *
+ * Project[
+ * Alias("alias_2")(
+ * Alias("alias_1")(id)
+ * )
+ * ]( ... )
+ *
+ * and after the `collapseAlias` call (removing the bottom one) it would be:
+ *
+ * Project[
+ * Alias("alias_2")(id)
+ * ]( ... )
+ */
+ private def collapseAlias(alias: Alias): Alias =
+ alias.child match {
+ case innerAlias: Alias =>
+ val metadata = if (alias.metadata.isEmpty) {
+ None
+ } else {
+ Some(alias.metadata)
+ }
+ alias.copy(child = innerAlias.child)(
+ exprId = alias.exprId,
+ qualifier = alias.qualifier,
+ explicitMetadata = metadata,
+ nonInheritableMetadataKeys = alias.nonInheritableMetadataKeys
+ )
+ case _ => alias
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala
index d3e93c82dfa2..e28b04178324 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala
@@ -20,21 +20,29 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import java.util.HashMap
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
/**
* The [[AnalyzerBridgeState]] is a state passed from legacy [[Analyzer]] to the single-pass
- * [[Resolver]].
+ * [[Resolver]]. It is used in dual-run mode (when
+ * [[ANALYZER_SINGLE_PASS_RESOLVER_RELATION_BRIDGING_ENABLED]] is true).
*
* @param relationsWithResolvedMetadata A map from [[UnresolvedRelation]] to the relations with
* resolved metadata. It allows us to reuse the relation metadata and avoid duplicate
- * catalog/table lookups in dual-run mode (when
- * [[ANALYZER_SINGLE_PASS_RESOLVER_RELATION_BRIDGING_ENABLED]] is true).
+ * catalog/table lookups.
+ * @param catalogRelationsWithResolvedMetadata A map from [[UnresolvedCatalogRelation]] to the
+ * relations with resolved metadata. It allows us to reuse the relation metadata and avoid
+ * duplicate catalog/table lookups.
*/
case class AnalyzerBridgeState(
relationsWithResolvedMetadata: AnalyzerBridgeState.RelationsWithResolvedMetadata =
- new AnalyzerBridgeState.RelationsWithResolvedMetadata)
+ new AnalyzerBridgeState.RelationsWithResolvedMetadata,
+ catalogRelationsWithResolvedMetadata: AnalyzerBridgeState.CatalogRelationsWithResolvedMetadata =
+ new AnalyzerBridgeState.CatalogRelationsWithResolvedMetadata
+)
object AnalyzerBridgeState {
type RelationsWithResolvedMetadata = HashMap[UnresolvedRelation, LogicalPlan]
+ type CatalogRelationsWithResolvedMetadata = HashMap[UnresolvedCatalogRelation, LogicalPlan]
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala
index 6f9d6defd2ed..b36a5ac6feb4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AttributeScopeStack.scala
@@ -21,11 +21,15 @@ import java.util.ArrayDeque
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet}
+/**
+ * A scope with registered attributes encountered during the logical plan validation process. We
+ * use [[AttributeSet]] here to check the equality of attributes based on their expression IDs.
+ */
+case class AttributeScope(attributes: AttributeSet, isSubqueryRoot: Boolean = false)
+
/**
* The [[AttributeScopeStack]] is used to validate that the attribute which was encountered by the
- * [[ExpressionResolutionValidator]] is in the current operator's visibility scope. We use
- * [[AttributeSet]] as scope implementation here to check the equality of attributes based on their
- * expression IDs.
+ * [[ExpressionResolutionValidator]] is in the current operator's visibility scope.
*
* E.g. for the following SQL query:
* {{{
@@ -46,44 +50,69 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet}
* new ID #3 for an alias of `a + col2`.
*/
class AttributeScopeStack {
- private val stack = new ArrayDeque[AttributeSet]
- push()
+ private val stack = new ArrayDeque[AttributeScope]
+ stack.push(AttributeScope(attributes = AttributeSet(Seq.empty)))
/**
- * Get the relevant attribute scope in the context of the current operator.
+ * Check if the `attribute` is present in this stack. We check the current scope by default. If
+ * `isOuterReference` is true, we check the first scope above our subquery root.
*/
- def top: AttributeSet = {
- stack.peek()
+ def contains(attribute: Attribute, isOuterReference: Boolean = false): Boolean = {
+ if (!isOuterReference) {
+ current.attributes.contains(attribute)
+ } else {
+ outer match {
+ case Some(outer) => outer.attributes.contains(attribute)
+ case _ => false
+ }
+ }
}
/**
* Overwrite current relevant scope with a sequence of attributes which is an output of some
* operator. `attributes` can have duplicate IDs if the output of the operator contains multiple
- * occurrences of the same attribute.
+ * occurencies of the same attribute.
*/
- def overwriteTop(attributes: Seq[Attribute]): Unit = {
- stack.pop()
- stack.push(AttributeSet(attributes))
+ def overwriteCurrent(attributes: Seq[Attribute]): Unit = {
+ val current = stack.pop()
+
+ stack.push(current.copy(attributes = AttributeSet(attributes)))
}
/**
* Execute `body` in the context of a fresh attribute scope. Used by [[Project]] and [[Aggregate]]
* validation code since those operators introduce a new scope with fresh expression IDs.
*/
- def withNewScope[R](body: => R): Unit = {
- push()
+ def withNewScope[R](isSubqueryRoot: Boolean = false)(body: => R): Unit = {
+ stack.push(
+ AttributeScope(
+ attributes = AttributeSet(Seq.empty),
+ isSubqueryRoot = isSubqueryRoot
+ )
+ )
try {
body
} finally {
- pop()
+ stack.pop()
}
}
- private def push(): Unit = {
- stack.push(AttributeSet(Seq.empty))
- }
+ override def toString: String = stack.toString
+
+ private def current: AttributeScope = stack.peek
+
+ private def outer: Option[AttributeScope] = {
+ var outerScope: Option[AttributeScope] = None
+
+ val iter = stack.iterator
+ while (iter.hasNext && !outerScope.isDefined) {
+ val scope = iter.next
+
+ if (scope.isSubqueryRoot && iter.hasNext) {
+ outerScope = Some(iter.next)
+ }
+ }
- private def pop(): Unit = {
- stack.pop()
+ outerScope
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AutoGeneratedAliasProvider.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AutoGeneratedAliasProvider.scala
new file mode 100644
index 000000000000..2a49581b3499
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AutoGeneratedAliasProvider.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, Expression}
+import org.apache.spark.sql.catalyst.util.toPrettySQL
+import org.apache.spark.sql.types.Metadata
+
+/**
+ * [[AutoGeneratedAliasProvider]] is a tool to create auto-generated aliases in the plan.
+ * All the auto-generated aliases have to be registered in [[ExpressionIdAssigner]].
+ */
+class AutoGeneratedAliasProvider(expressionIdAssigner: ExpressionIdAssigner) {
+
+ /**
+ * Create a new auto-generated [[Alias]]. If the `name` is not provided, [[toPrettySql]] is going
+ * to be used to generate a proper alias name. We call [[ExpressionIdAssigner.mapExpression]] to
+ * register this alias in the [[ExpressionIdAssigner]].
+ */
+ def newAlias(
+ child: Expression,
+ name: Option[String] = None,
+ explicitMetadata: Option[Metadata] = None): Alias = {
+ newAliasImpl(child = child, name = name, explicitMetadata = explicitMetadata)
+ }
+
+ /**
+ * Create a new auto-generated [[Alias]]. If the `name` is not provided, [[toPrettySql]] is going
+ * to be used to generate a proper alias name. We don't call
+ * [[ExpressionIdAssigner.mapExpression]] for outer aliases, because thy should be manually
+ * mapped in the context of an outer query.
+ */
+ def newOuterAlias(
+ child: Expression,
+ name: Option[String] = None,
+ explicitMetadata: Option[Metadata] = None): Alias = {
+ newAliasImpl(
+ child = child,
+ name = name,
+ explicitMetadata = explicitMetadata,
+ skipExpressionIdAssigner = true
+ )
+ }
+
+ def newAliasImpl(
+ child: Expression,
+ name: Option[String] = None,
+ explicitMetadata: Option[Metadata] = None,
+ skipExpressionIdAssigner: Boolean = false): Alias = {
+ var alias = Alias(
+ child = child,
+ name = name.getOrElse(toPrettySQL(child))
+ )(
+ explicitMetadata = explicitMetadata
+ )
+ if (skipExpressionIdAssigner) {
+ alias
+ } else {
+ expressionIdAssigner.mapExpression(alias)
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BinaryArithmeticResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BinaryArithmeticResolver.scala
index d2b586f3d372..0f90e8a852fe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BinaryArithmeticResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BinaryArithmeticResolver.scala
@@ -100,11 +100,14 @@ class BinaryArithmeticResolver(
override def resolve(unresolvedBinaryArithmetic: BinaryArithmetic): Expression = {
val binaryArithmeticWithResolvedChildren: BinaryArithmetic =
- withResolvedChildren(unresolvedBinaryArithmetic, expressionResolver.resolve)
+ withResolvedChildren(unresolvedBinaryArithmetic, expressionResolver.resolve _)
+ .asInstanceOf[BinaryArithmetic]
+
val binaryArithmeticWithResolvedSubtree: Expression =
withResolvedSubtree(binaryArithmeticWithResolvedChildren, expressionResolver.resolve) {
transformBinaryArithmeticNode(binaryArithmeticWithResolvedChildren)
}
+
timezoneAwareExpressionResolver.withResolvedTimezone(
binaryArithmeticWithResolvedSubtree,
conf.sessionLocalTimeZone
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BridgedRelationMetadataProvider.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BridgedRelationMetadataProvider.scala
index a33675e9dfd0..f01b9adf2a95 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BridgedRelationMetadataProvider.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/BridgedRelationMetadataProvider.scala
@@ -18,7 +18,8 @@
package org.apache.spark.sql.catalyst.analysis.resolver
import org.apache.spark.sql.catalyst.analysis.RelationResolution
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.catalog.UnresolvedCatalogRelation
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
import org.apache.spark.sql.connector.catalog.CatalogManager
/**
@@ -52,9 +53,31 @@ class BridgedRelationMetadataProvider(
(unresolvedRelation, relationWithResolvedMetadata) => {
relationsWithResolvedMetadata.put(
relationIdFromUnresolvedRelation(unresolvedRelation),
- relationWithResolvedMetadata
+ tryConvertUnresolvedCatalogRelation(relationWithResolvedMetadata)
)
}
)
}
+
+ private def tryConvertUnresolvedCatalogRelation(source: LogicalPlan): LogicalPlan = {
+ source match {
+ case unresolvedCatalogRelation: UnresolvedCatalogRelation
+ if analyzerBridgeState.catalogRelationsWithResolvedMetadata
+ .containsKey(unresolvedCatalogRelation) =>
+ analyzerBridgeState.catalogRelationsWithResolvedMetadata.get(unresolvedCatalogRelation)
+
+ case SubqueryAlias(id, unresolvedCatalogRelation: UnresolvedCatalogRelation)
+ if analyzerBridgeState.catalogRelationsWithResolvedMetadata
+ .containsKey(unresolvedCatalogRelation) =>
+ SubqueryAlias(
+ id,
+ analyzerBridgeState.catalogRelationsWithResolvedMetadata.get(
+ unresolvedCatalogRelation
+ )
+ )
+
+ case _ =>
+ source
+ }
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ConditionalExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ConditionalExpressionResolver.scala
index 4532965c6c68..428662d14dc8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ConditionalExpressionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ConditionalExpressionResolver.scala
@@ -40,7 +40,7 @@ class ConditionalExpressionResolver(
override def resolve(unresolvedConditionalExpression: ConditionalExpression): Expression = {
val conditionalExpressionWithResolvedChildren =
- withResolvedChildren(unresolvedConditionalExpression, expressionResolver.resolve)
+ withResolvedChildren(unresolvedConditionalExpression, expressionResolver.resolve _)
typeCoercionResolver.resolve(conditionalExpressionWithResolvedChildren)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CreateNamedStructResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CreateNamedStructResolver.scala
index 12c3c71b5e8b..d0e4ecea25cb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CreateNamedStructResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CreateNamedStructResolver.scala
@@ -31,6 +31,7 @@ class CreateNamedStructResolver(expressionResolver: ExpressionResolver)
override def resolve(createNamedStruct: CreateNamedStruct): Expression = {
val createNamedStructWithResolvedChildren =
withResolvedChildren(createNamedStruct, expressionResolver.resolve)
+ .asInstanceOf[CreateNamedStruct]
CreateNamedStructResolver.cleanupAliases(createNamedStructWithResolvedChildren)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CteScope.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CteScope.scala
index 4f61a25a3cd4..b0a39706ec22 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CteScope.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/CteScope.scala
@@ -21,7 +21,13 @@ import java.util.{ArrayDeque, ArrayList}
import scala.jdk.CollectionConverters._
-import org.apache.spark.sql.catalyst.plans.logical.CTERelationDef
+import org.apache.spark.sql.catalyst.plans.logical.{
+ CTERelationDef,
+ LogicalPlan,
+ UnresolvedWith,
+ WithCTE
+}
+import org.apache.spark.sql.catalyst.trees.TreePattern.UNRESOLVED_WITH
/**
* The [[CteScope]] is responsible for keeping track of visible and known CTE definitions at a given
@@ -98,6 +104,67 @@ import org.apache.spark.sql.catalyst.plans.logical.CTERelationDef
* : +- ...
* }}}
*
+ * - The [[WithCTE]] operator is placed on top of the resolved operator if one of the following
+ * conditions are met:
+ * 1. We just resolved an [[UnresolvedWith]], which is the topmost [[UnresolvedWith]] of this
+ * root query, view or an expression subquery.
+ * 2. In case there is no single topmost [[UnresolvedWith]], we pick the least common ancestor
+ * of those branches. This is going to be a multi-child operator - [[Union]], [[Join]], etc.
+ *
+ * Here's an example for the second case:
+ *
+ * {{{
+ * SELECT * FROM (
+ * WITH cte AS (
+ * SELECT 1
+ * )
+ * SELECT * FROM cte
+ * UNION ALL
+ * (
+ * WITH cte AS (
+ * SELECT 2
+ * )
+ * SELECT * FROM cte
+ * )
+ * )
+ * }}}
+ *
+ * ->
+ *
+ * {{{
+ * Project [1#60]
+ * +- SubqueryAlias __auto_generated_subquery_name
+ * +- WithCTE
+ * :- CTERelationDef 30, false
+ * : +- ...
+ * :- CTERelationDef 31, false
+ * : +- ...
+ * +- Union false, false
+ * :- Project [1#60]
+ * : +- ...
+ * +- Project [2#61]
+ * +- ...
+ * }}}
+ *
+ * Consider a different example though:
+ *
+ * {{{
+ * SELECT * FROM (
+ * SELECT 1
+ * UNION ALL
+ * (
+ * WITH cte AS (
+ * SELECT 2
+ * )
+ * SELECT * FROM cte
+ * )
+ * )
+ * }}}
+ *
+ * The [[Union]] operator is not the least common ancestor of the [[UnresolvedWith]]s in the
+ * query. In fact, there's just a single [[UnresolvedWith]], which is a proper place where we
+ * need to place a [[WithCTE]].
+ *
* - However, if we have any expression subquery (scalar/IN/EXISTS...), the top
* [[CTERelationDef]]s and subquery's [[CTERelationDef]] won't be merged together (as they are
* separated by an expression tree):
@@ -167,6 +234,19 @@ class CteScope(val isRoot: Boolean, val isOpaque: Boolean) {
*/
private val visibleCtes = new IdentifierMap[CTERelationDef]
+ /**
+ * Optionally put [[WithCTE]] on top of the `resolvedOperator`. This is done just for the root
+ * scopes in the context of a correct `unresolvedOperator`. Return the `resolvedOperator`
+ * otherwise.
+ */
+ def tryPutWithCTE(unresolvedOperator: LogicalPlan, resolvedOperator: LogicalPlan): LogicalPlan = {
+ if (!knownCtes.isEmpty && isRoot && isSuitableOperatorForWithCTE(unresolvedOperator)) {
+ WithCTE(resolvedOperator, knownCtes.asScala.toSeq)
+ } else {
+ resolvedOperator
+ }
+ }
+
/**
* Register a new CTE definition in this scope. Since the scope is created per single WITH clause,
* there can be no name conflicts, but this is validated by the Parser in [[AstBuilder]]
@@ -197,11 +277,17 @@ class CteScope(val isRoot: Boolean, val isOpaque: Boolean) {
}
/**
- * Get all known (from this and child scopes) [[CTERelationDef]]s. This is used to construct
- * [[WithCTE]] from a root scope.
+ * This predicate returns `true` if the `unresolvedOperator` is suitable to place a [[WithCTE]]
+ * on top of its resolved counterpart. This is the case for:
+ * - [[UnresolvedWith]];
+ * - Multi-child operators with [[UnresolvedWith]]s in multiple subtrees.
*/
- def getKnownCtes: Seq[CTERelationDef] = {
- knownCtes.asScala.toSeq
+ private def isSuitableOperatorForWithCTE(unresolvedOperator: LogicalPlan): Boolean = {
+ unresolvedOperator match {
+ case _: UnresolvedWith => true
+ case _ =>
+ CteRegistry.isSuitableMultiChildOperatorForWithCTE(unresolvedOperator)
+ }
}
}
@@ -215,6 +301,29 @@ class CteRegistry {
def currentScope: CteScope = stack.peek()
+ /**
+ * This is a [[withNewScope]] variant specifically designed to be called above multi-child
+ * operator children resolution (e.g. for children of a [[Join]] or [[Union]]).
+ *
+ * The `isRoot` flag has to be propagated from the parent scope if all of the following
+ * conditions are met:
+ * - The current scope is a root scope
+ * - The multi-child `unresolvedOperator` IS NOT suitable to place a [[WithCTE]]
+ * - Some operator in `unresolvedChild` subtree IS suitable to place a [[WithCTE]].
+ */
+ def withNewScopeUnderMultiChildOperator[R](
+ unresolvedOperator: LogicalPlan,
+ unresolvedChild: LogicalPlan
+ )(body: => R): R = {
+ withNewScope(
+ isRoot = currentScope.isRoot &&
+ !CteRegistry.isSuitableMultiChildOperatorForWithCTE(unresolvedOperator) &&
+ CteRegistry.hasSuitableOperatorForWithCTEInSubtree(unresolvedChild)
+ ) {
+ body
+ }
+ }
+
/**
* A RAII-wrapper for pushing/popping scopes. This is used by the [[Resolver]] to create a new
* scope for each WITH clause.
@@ -255,3 +364,19 @@ class CteRegistry {
result
}
}
+
+object CteRegistry {
+
+ /**
+ * This predicate returns `true` if the `unresolvedOperator` is a multi-child operator that
+ * contains multiple [[UnresolvedWith]] operators in its subtrees. This way we determine if
+ * this operator is suitable to place a [[WithCTE]] on top of its resolved counterpart.
+ */
+ def isSuitableMultiChildOperatorForWithCTE(unresolvedOperator: LogicalPlan): Boolean = {
+ unresolvedOperator.children.count(hasSuitableOperatorForWithCTEInSubtree(_)) > 1
+ }
+
+ def hasSuitableOperatorForWithCTEInSubtree(unresolvedOperator: LogicalPlan): Boolean = {
+ unresolvedOperator.containsPattern(UNRESOLVED_WITH)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DelegatesResolutionToExtensions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DelegatesResolutionToExtensions.scala
index e9de72e6fa36..acac8edd4a73 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DelegatesResolutionToExtensions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/DelegatesResolutionToExtensions.scala
@@ -51,7 +51,6 @@ trait DelegatesResolutionToExtensions {
matchedExtension match {
case None =>
resolutionResult = extension.resolveOperator(unresolvedOperator, resolver)
-
if (resolutionResult.isDefined) {
matchedExtension = Some(extension)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala
index c9b8dece77ce..475f38cabcff 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExplicitlyUnsupportedResolverFeature.scala
@@ -21,7 +21,7 @@ package org.apache.spark.sql.catalyst.analysis.resolver
* This is an addon to [[ResolverGuard]] functionality for features that cannot be determined by
* only looking at the unresolved plan. [[Resolver]] will throw this control-flow exception
* when it encounters some explicitly unsupported feature. Later behavior depends on the value of
- * [[HybridAnalyzer.checkSupportedSinglePassFeatures]] flag:
+ * [[HybridAnalyzer.exposeExplicitlyUnsupportedResolverFeature]] flag:
* - If it is true: It will later be caught by [[HybridAnalyzer]] to abort single-pass
* analysis without comparing single-pass and fixed-point results. The motivation for this
* feature is the same as for the [[ResolverGuard]] - we want to have an explicit allowlist of
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala
index 2d8c332d8527..cbd772d5a678 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionIdAssigner.scala
@@ -27,6 +27,8 @@ import org.apache.spark.sql.catalyst.expressions.{
ExprId,
NamedExpression
}
+import org.apache.spark.sql.catalyst.plans.logical.{CTERelationRef, LeafNode}
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.errors.QueryCompilationErrors
/**
@@ -35,11 +37,15 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
* that Optimizer performs its work correctly and does not produce correctness issues.
*
* The framework works the following way:
- * - Each leaf operator must have unique output IDs (even if it's the same table, view, or CTE).
+ * - Each leaf operator must have globally unique output IDs (even if it's the same table, view,
+ * or CTE).
* - The [[AttributeReference]]s get propagated "upwards" through the operator tree with their IDs
- * preserved.
- * - Each [[Alias]] gets assigned a new unique ID and it sticks with it after it gets converted to
- * an [[AttributeReference]] when it is outputted from the operator that produced it.
+ * preserved. In case of correlated subqueries [[AttributeReference]]s may propagate downwards
+ * from the outer scope to the point of correlated reference in the subquery. Currently only
+ * one level of correlation is supported.
+ * - Each [[Alias]] gets assigned a new globally unique ID and it sticks with it after it gets
+ * converted to an [[AttributeReference]] when it is outputted from the operator that produced
+ * it.
* - Any operator may have [[AttributeReference]]s with the same IDs in its output given it is the
* same attribute.
* Thus, **no multi-child operator may have children with conflicting [[AttributeReference]] IDs**.
@@ -116,9 +122,48 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
* Because the latter case will confuse the Optimizer and the top [[Project]] will be eliminated
* leading to incorrect result.
*
- * There's an important caveat here: the leftmost branch of a logical plan tree. In this branch we
- * need to preserve the expression IDs wherever possible because DataFrames may reference each other
- * using their attributes. This also makes sense for performance reasons.
+ * In case of partially resolved DataFrame subtrees with correlated subqueries inside we need to
+ * remap [[OuterReference]]s as well:
+ *
+ * {{{
+ * val df = spark.sql("""
+ * SELECT * FROM t1 WHERE EXISTS (
+ * SELECT * FROM t2 WHERE t2.id == t1.id
+ * )
+ * """)
+ * df.union(df)
+ * }}}
+ *
+ * The analyzed plan should be:
+ * {{{
+ * Union false, false
+ * :- Project [id#1]
+ * : +- Filter exists#9 [id#1]
+ * : : +- Project [id#16]
+ * : : +- Filter (id#16 = outer(id#1))
+ * : : +- SubqueryAlias spark_catalog.default.t2
+ * : : +- Relation spark_catalog.default.t2[id#16] parquet
+ * : +- SubqueryAlias spark_catalog.default.t1
+ * : +- Relation spark_catalog.default.t1[id#1] parquet
+ * +- Project [id#17 AS id#19]
+ * +- Project [id#17]
+ * +- Filter exists#9 [id#17]
+ * : +- Project [id#18]
+ * : +- Filter (id#18 = outer(id#17))
+ * : +- SubqueryAlias spark_catalog.default.t2
+ * : +- Relation spark_catalog.default.t2[id#18] parquet
+ * +- SubqueryAlias spark_catalog.default.t1
+ * +- Relation spark_catalog.default.t1[id#17] parquet
+ * }}}
+ *
+ * Note how id#17 is the same in outer branch and in a subquery - is was properly remapped, because
+ * the right subtree of [[Union]] contained identical expression IDs as the left subtree. That's
+ * why we pass main mapping as outer mapping to the correlated subquery branch.
+ *
+ * There's an important caveat here: those branches of a logical plan tree where outputs do not
+ * conflict. We should preserve expression IDs on those branches wherever possible because
+ * DataFrames may reference each other using their attributes. This also makes sense for
+ * performance reasons.
*
* Consider this example:
*
@@ -130,10 +175,27 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
*
* In this example `df("id")` references lower `id` attribute by expression ID, so `union` must not
* reassign expression IDs in `df1` (left child). Referencing `df2` (right child) is not supported
- * in Spark.
+ * in Spark, because [[Union]] does not output it, but we don't have to regenerate expression IDs
+ * in that branch either.
+ *
+ * However:
+ *
+ * {{{
+ * val df1 = spark.range(0, 10).select($"id")
+ * df1.union(df1).filter(df1("id") === 5)
+ * }}}
+ *
+ * Here we need to regenerate expression IDs in the right branch, because those would conflict
+ * (both branches are the same plan). Expression IDs in the left branch may be preserved.
+ *
+ * CTE references are handled in a special way to stay compatible with the fixed-point Analyzer.
+ * First [[CTERelationRef]] that we meet in the query plan can preserve its output expression IDs,
+ * and the plan will be inlined by the [[InlineCTE]] without any artificial [[Alias]]es that
+ * "stitch" expression IDs together. This way we ensure that Optimizer behavior is the same as
+ * after the fixed-point Analyzer and that no extra projections are introduced.
*
* The [[ExpressionIdAssigner]] covers both SQL and DataFrame scenarios with single approach and is
- * integrated in the single-pass analysis framework.
+ * integrated into the single-pass analysis framework.
*
* The [[ExpressionIdAssigner]] is used in the following way:
* - When the [[Resolver]] traverses the tree downwards prior to starting bottom-up analysis,
@@ -144,88 +206,171 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
* the mapping needs to be initialized later with the correct output of a resolved operator.
* - When the bottom-up analysis starts, we assign IDs to all the [[NamedExpression]]s which are
* present in operators starting from the [[LeafNode]]s using [[mapExpression]].
- * [[createMapping]] is called right after each [[LeafNode]] is resolved, and first remapped
- * attributes come from that [[LeafNode]]. This is done in [[Resolver.handleLeafOperator]] for
- * each logical plan tree branch except the leftmost.
+ * [[createMappingForLeafOperator]] is called right after each [[LeafNode]] is resolved, and
+ * first remapped attributes come from that [[LeafNode]]. This is done if leaf operator output
+ * doesn't conflict with `globalExpressionIds`.
* - Once the child branch is resolved, [[withNewMapping]] ends by calling [[mappingStack.pop]].
- * - After the multi-child operator is resolved, we call [[createMapping]] to
- * initialize the mapping with attributes _chosen_ (e.g. [[Union.mergeChildOutputs]]) by that
- * operator's resolution algorithm and remap _old_ expression IDs to those chosen attributes.
+ * - After the multi-child operator is resolved, we call [[createMappingFromChildMappings]] to
+ * initialize the mapping with attributes collected in [[withNewMapping]] with
+ * `collectChildMapping = true`.
+ * - While traversing the expression tree, we may meet a [[SubqueryExpression]] and resolve its
+ * plan. In this case we call [[withNewMapping]] with `isSubqueryRoot = true` to pass the
+ * current mapping as outer mapping to the subquery branches. Any subquery branch may reference
+ * outer attributes, so if `isSubqueryRoot` is `false`, we pass the previous `outerMapping` to
+ * lower branches. Since we only support one level of correlation, for every subquery level
+ * current `mapping` becomes `outerMapping` for the next level.
* - Continue remapping expressions until we reach the root of the operator tree.
*/
class ExpressionIdAssigner {
- private val mappingStack = new ExpressionIdAssigner.Stack
- mappingStack.push(ExpressionIdAssigner.StackEntry(isLeftmostBranch = true))
+ private val globalExpressionIds = new HashSet[ExprId]
+ private val cteRelationRefOutputIds = new HashSet[ExprId]
- /**
- * Returns `true` if the current logical plan branch is the leftmost branch. This is important
- * in the context of preserving expression IDs in DataFrames. See class doc for more details.
- */
- def isLeftmostBranch: Boolean = mappingStack.peek().isLeftmostBranch
+ private val mappingStack = new ExpressionIdAssigner.Stack
+ mappingStack.push(ExpressionIdAssigner.StackEntry())
/**
* A RAII-wrapper for [[mappingStack.push]] and [[mappingStack.pop]]. [[Resolver]] uses this for
* every child of a multi-child operator to ensure that each operator branch uses an isolated
* expression ID mapping.
*
- * @param isLeftmostChild whether the current child is the leftmost child of the operator that is
- * being resolved. This is used to determine whether the new stack entry is gonna be in the
- * leftmost logical plan branch. It's `false` by default, because it's safer to remap attributes
- * than to leave duplicates (to prevent correctness issues).
+ * @param isSubqueryRoot whether the new branch is related to a subquery root. In this case we
+ * pass current `mapping` as `outerMapping` to the subquery branches. Otherwise we just
+ * propagate `outerMapping` itself, because any nested subquery operator may reference outer
+ * attributes.
+ * @param collectChildMapping whether to collect a child mapping into the current stack entry.
+ * This is used in multi-child operators to automatically propagate mapped expression IDs
+ * upwards using [[createMappingFromChildMappings]].
*/
- def withNewMapping[R](isLeftmostChild: Boolean = false)(body: => R): R = {
+ def withNewMapping[R](
+ isSubqueryRoot: Boolean = false,
+ collectChildMapping: Boolean = false
+ )(body: => R): R = {
+ val currentStackEntry = mappingStack.peek()
+
mappingStack.push(
ExpressionIdAssigner.StackEntry(
- isLeftmostBranch = isLeftmostChild && isLeftmostBranch
+ outerMapping = if (isSubqueryRoot) {
+ currentStackEntry.mapping.map(new ExpressionIdAssigner.Mapping(_))
+ } else {
+ currentStackEntry.outerMapping
+ }
)
)
+
try {
- body
+ val result = body
+
+ val childStackEntry = mappingStack.peek()
+ if (collectChildMapping) {
+ childStackEntry.mapping match {
+ case Some(childMapping) =>
+ currentStackEntry.childMappings.push(childMapping)
+ case None =>
+ throw SparkException.internalError("Child mapping doesn't exist")
+ }
+ }
+
+ result
} finally {
mappingStack.pop()
}
}
/**
- * Create mapping with the given `newOutput` that rewrites the `oldOutput`. This
- * is used by the [[Resolver]] after the multi-child operator is resolved to fill the current
- * mapping with the attributes _chosen_ by that operator's resolution algorithm and remap _old_
- * expression IDs to those chosen attributes. It's also used by the [[ExpressionResolver]] right
- * before remapping the attributes of a [[LeafNode]].
+ * Create mapping for the given `newOperator` that replaces the `oldOperator`. This is used by
+ * the [[Resolver]] after a certain [[LeafNode]] is resolved to make sure that leaf node outputs
+ * in the query don't have conflicting expression IDs.
*
- * `oldOutput` is present for already resolved subtrees (e.g. DataFrames), but for SQL queries
+ * `oldOperator` is present for already resolved subtrees (e.g. DataFrames), but for SQL queries
* is will be `None`, because that logical plan is analyzed for the first time.
*/
- def createMapping(
- newOutput: Seq[Attribute] = Seq.empty,
- oldOutput: Option[Seq[Attribute]] = None): Unit = {
+ def createMappingForLeafOperator(
+ newOperator: LeafNode,
+ oldOperator: Option[LeafNode] = None): Unit = {
if (mappingStack.peek().mapping.isDefined) {
throw SparkException.internalError(
- s"Attempt to overwrite existing mapping. New output: $newOutput, old output: $oldOutput"
+ "Attempt to overwrite existing mapping. " +
+ s"New operator: $newOperator, old operator: $oldOperator"
)
}
val newMapping = new ExpressionIdAssigner.Mapping
- oldOutput match {
- case Some(oldOutput) =>
- if (newOutput.length != oldOutput.length) {
+ oldOperator match {
+ case Some(oldOperator) =>
+ if (newOperator.output.length != oldOperator.output.length) {
throw SparkException.internalError(
- s"Outputs have different lengths. New output: $newOutput, old output: $oldOutput"
+ "Outputs have different lengths. " +
+ s"New operator: $newOperator, old operator: $oldOperator"
)
}
- newOutput.zip(oldOutput).foreach {
+ newOperator.output.zip(oldOperator.output).foreach {
case (newAttribute, oldAttribute) =>
newMapping.put(oldAttribute.exprId, newAttribute.exprId)
newMapping.put(newAttribute.exprId, newAttribute.exprId)
+
+ registerLeafOperatorAttribute(newOperator, newAttribute)
}
case None =>
- newOutput.foreach { newAttribute =>
+ newOperator.output.foreach { newAttribute =>
newMapping.put(newAttribute.exprId, newAttribute.exprId)
+
+ registerLeafOperatorAttribute(newOperator, newAttribute)
}
}
- mappingStack.push(mappingStack.pop().copy(mapping = Some(newMapping)))
+ setCurrentMapping(newMapping)
+ }
+
+ /**
+ * Create new mapping in current scope based on collected child mappings. The calling code
+ * must pass `collectChildMapping = true` to all the [[withNewMapping]] calls beforehand.
+ *
+ * Since nodes are resolved from left to right (the Analyzer is guaranteed to resolve left
+ * branches first), we know that by calling [[childMappings.pop]] we get the mappings from right
+ * to left. This approach leads to duplicate expression IDs from right mapping keys being
+ * overwritten by the left ones. This order is very important, because in case of duplicate
+ * DataFrame subtrees like self-joins, expression IDs from right duplicate branch cannot be
+ * accessed:
+ *
+ * {{{
+ * val df1 = spark.range(0, 10)
+ * val df2 = df1.select(($"id" + 1).as("id"))
+ *
+ * // Both branches originate from the same `df1`, and have duplicate IDs, so right branch IDs
+ * // are regenedated. Thus, it's important to prioritize left mapping values for the same keys.
+ * val df3 = df2.join(df1, "id")
+ *
+ * // This succeeds because left branch IDs are preserved.
+ * df3.where(df2("id") === 1)
+ *
+ * // This fails because right branch IDs are regenerated.
+ * df3.where(df1("id") === 1)
+ * }}}
+ *
+ * This is used by multi child operators like [[Join]] or [[Union]] to propagate mapped
+ * expression IDs upwards.
+ */
+ def createMappingFromChildMappings(): Unit = {
+ if (mappingStack.peek().mapping.isDefined) {
+ throw SparkException.internalError(
+ "Attempt to overwrite existing mapping with child mappings"
+ )
+ }
+
+ val currentStackEntry = mappingStack.peek()
+ if (currentStackEntry.childMappings.isEmpty) {
+ throw SparkException.internalError("No child mappings to create new current mapping")
+ }
+
+ val newMapping = new ExpressionIdAssigner.Mapping
+ while (!currentStackEntry.childMappings.isEmpty) {
+ val nextMapping = currentStackEntry.childMappings.pop()
+
+ newMapping.putAll(nextMapping)
+ }
+
+ setCurrentMapping(newMapping)
}
/**
@@ -233,9 +378,9 @@ class ExpressionIdAssigner {
* expression, or return a corresponding new instance of the same attribute, that was previously
* reallocated and is present in the current [[mappingStack]] entry.
*
- * For [[Alias]]es: Try to preserve them if we are in the leftmost logical plan tree branch and
- * unless they conflict. Conflicting [[Alias]] IDs are never acceptable. Otherwise, reallocate
- * with a new ID and return that instance.
+ * For [[Alias]]es: Try to preserve it if the alias ID doesn't conflict with
+ * `globalExpressionIds`. Conflicting [[Alias]] IDs are never acceptable.
+ * Otherwise, reallocate with a new ID and return that instance.
*
* For [[AttributeReference]]s: If the attribute is present in the current [[mappingStack]] entry,
* return that instance, otherwise reallocate with a new ID and return that instance. The mapping
@@ -267,32 +412,39 @@ class ExpressionIdAssigner {
* spark.sql("SELECT col1 FROM VALUES (1)").select(col("col1").as("a", metadata1)).to(schema)
* }}}
*/
- def mapExpression(originalExpression: NamedExpression): NamedExpression = {
+ def mapExpression[NamedExpressionType <: NamedExpression](
+ originalExpression: NamedExpressionType): NamedExpressionType = {
if (mappingStack.peek().mapping.isEmpty) {
throw SparkException.internalError(
- "Expression ID mapping doesn't exist. Please call createMapping(...) first. " +
- s"Original expression: $originalExpression"
+ "Expression ID mapping doesn't exist. Please first call " +
+ "createMappingForLeafOperator(...) for leaf nodes or createMappingFromChildMappings(...) " +
+ s"for multi-child nodes. Original expression: $originalExpression"
)
}
val currentMapping = mappingStack.peek().mapping.get
val resultExpression = originalExpression match {
- case alias: Alias if isLeftmostBranch =>
- val resultAlias = currentMapping.get(alias.exprId) match {
- case null =>
- alias
- case _ =>
- alias.newInstance()
+ case alias: Alias =>
+ val resultAlias = if (globalExpressionIds.contains(alias.exprId)) {
+ val newAlias = newAliasInstance(alias)
+ currentMapping.put(alias.exprId, newAlias.exprId)
+ newAlias
+ } else {
+ alias
}
+
currentMapping.put(resultAlias.exprId, resultAlias.exprId)
+
+ globalExpressionIds.add(resultAlias.exprId)
+
resultAlias
- case alias: Alias =>
- reassignExpressionId(alias, currentMapping)
case attributeReference: AttributeReference =>
currentMapping.get(attributeReference.exprId) match {
case null =>
- reassignExpressionId(attributeReference, currentMapping)
+ throw SparkException.internalError(
+ s"Encountered a dangling attribute reference $attributeReference"
+ )
case mappedExpressionId =>
attributeReference.withExprId(mappedExpressionId)
}
@@ -302,31 +454,96 @@ class ExpressionIdAssigner {
)
}
- resultExpression.copyTagsFrom(originalExpression)
- resultExpression
+ resultExpression.asInstanceOf[NamedExpressionType]
+ }
+
+ /**
+ * Map [[AttributeReference]] which is a child of [[OuterReference]]. When [[ExpressionResolver]]
+ * meets an attribute under a resolved [[OuterReference]], it remaps it using the outer
+ * mapping passed from the parent plan of the [[SubqueryExpression]] that is currently being
+ * re-analyzed. This mapping must exist, as well as a mapped expression ID. Otherwise we met a
+ * danging outer reference, which is an internal error.
+ */
+ def mapOuterReference(attributeReference: AttributeReference): AttributeReference = {
+ if (mappingStack.peek().outerMapping.isEmpty) {
+ throw SparkException.internalError(
+ "Outer expression ID mapping doesn't exist while remapping outer reference " +
+ s"$attributeReference"
+ )
+ }
+
+ mappingStack.peek().outerMapping.get.get(attributeReference.exprId) match {
+ case null =>
+ throw SparkException.internalError(
+ s"No mapped expression ID for outer reference $attributeReference"
+ )
+ case mappedExpressionId =>
+ attributeReference.withExprId(mappedExpressionId)
+ }
+ }
+
+ /**
+ * Returns `true` if expression IDs for the current [[LeafNode]] should be preserved. This is
+ * important for DataFrames that reference columns by their IDs. See class doc for more details.
+ *
+ * Expression IDs of outputs of the first CTE reference are not regenerated for compatibility
+ * with the fixed-point Analyzer.
+ */
+ def shouldPreserveLeafOperatorIds(leafOperator: LeafNode): Boolean = {
+ leafOperator match {
+ case cteRelationRef: CTERelationRef =>
+ cteRelationRef.output.forall { attribute =>
+ !cteRelationRefOutputIds.contains(attribute.exprId)
+ }
+ case _ =>
+ leafOperator.output.forall { attribute =>
+ !globalExpressionIds.contains(attribute.exprId)
+ }
+ }
}
- private def reassignExpressionId(
- originalExpression: NamedExpression,
- currentMapping: ExpressionIdAssigner.Mapping): NamedExpression = {
- val newExpression = originalExpression.newInstance()
+ private def setCurrentMapping(mapping: ExpressionIdAssigner.Mapping): Unit = {
+ val currentEntry = mappingStack.pop()
+ mappingStack.push(currentEntry.copy(mapping = Some(mapping)))
+ }
- currentMapping.put(originalExpression.exprId, newExpression.exprId)
- currentMapping.put(newExpression.exprId, newExpression.exprId)
+ private def newAliasInstance(alias: Alias): Alias = {
+ val newAlias = withOrigin(alias.origin) {
+ alias.newInstance().asInstanceOf[Alias]
+ }
+ newAlias.copyTagsFrom(alias)
+ newAlias
+ }
- newExpression
+ private def registerLeafOperatorAttribute(leafOperator: LeafNode, attribute: Attribute): Unit = {
+ globalExpressionIds.add(attribute.exprId)
+ if (leafOperator.isInstanceOf[CTERelationRef]) {
+ cteRelationRefOutputIds.add(attribute.exprId)
+ }
}
}
object ExpressionIdAssigner {
type Mapping = HashMap[ExprId, ExprId]
- case class StackEntry(mapping: Option[Mapping] = None, isLeftmostBranch: Boolean = false)
+ case class StackEntry(
+ mapping: Option[Mapping] = None,
+ outerMapping: Option[Mapping] = None,
+ childMappings: ArrayDeque[Mapping] = new ArrayDeque[Mapping])
type Stack = ArrayDeque[StackEntry]
/**
- * Assert that `outputs` don't have conflicting expression IDs. This is only relevant for child
+ * Assert that `outputs` don't have conflicting expression IDs.
+ */
+ def assertOutputsHaveNoConflictingExpressionIds(outputs: Seq[Seq[Attribute]]): Unit = {
+ if (doOutputsHaveConflictingExpressionIds(outputs)) {
+ throw SparkException.internalError(s"Conflicting expression IDs in child outputs: $outputs")
+ }
+ }
+
+ /**
+ * Check whether `outputs` have conflicting expression IDs. This is only relevant for child
* outputs of multi-child operators. Conflicting attributes are only checked between different
* child branches, since one branch may output the same attribute multiple times. Hence, we use
* only distinct expression IDs from each output.
@@ -344,14 +561,11 @@ object ExpressionIdAssigner {
* SELECT col1 FROM t1
* ;
* }}}
+ *
+ * One edge case is [[WithCTE]] - we don't have to check conflicts between [[CTERelationDef]]s and
+ * the plan itself.
*/
- def assertOutputsHaveNoConflictingExpressionIds(outputs: Seq[Seq[Attribute]]): Unit = {
- if (doOutputsHaveConflictingExpressionIds(outputs)) {
- throw SparkException.internalError(s"Conflicting expression IDs in child outputs: $outputs")
- }
- }
-
- private def doOutputsHaveConflictingExpressionIds(outputs: Seq[Seq[Attribute]]): Boolean = {
+ def doOutputsHaveConflictingExpressionIds(outputs: Seq[Seq[Attribute]]): Boolean = {
outputs.length > 1 && {
val expressionIds = new HashSet[ExprId]
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionContext.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionContext.scala
index 822cda228962..66fa5a4226e6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionContext.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionContext.scala
@@ -23,21 +23,66 @@ package org.apache.spark.sql.catalyst.analysis.resolver
* of [[ExpressionResolver.resolve]] call, which are not the resolved child itself, from children
* to parents.
*
- * @param hasAggregateExpressionsInASubtree A flag that highlights that a specific node
- * corresponding to [[ExpressionResolutionContext]] has
- * aggregate expressions in its subtree.
- * @param hasAttributeInASubtree A flag that highlights that a specific node corresponding to
- * [[ExpressionResolutionContext]] has attributes in its subtree.
+ * @param isRoot A flag indicating that we are resolving the root of the expression tree. It's
+ * used by the [[ExpressionResolver]] to correctly propagate top-level information like
+ * [[isTopOfProjectList]]. It's going to be set to `true` for the top-level context when we are
+ * entering expression resolution from a specific operator (this can be either a top-level query
+ * or a subquery).
+ * @param hasLocalReferences A flag that highlights that a specific node corresponding to
+ * [[ExpressionResolutionContext]] has local [[AttributeReferences]] in its subtree.
+ * @param hasOuterReferences A flag that highlights that a specific node corresponding to
+ * [[ExpressionResolutionContext]] has outer [[AttributeReferences]] in its subtree.
+ * @param hasAggregateExpressions A flag that highlights that a specific node
+ * corresponding to [[ExpressionResolutionContext]] has aggregate expressions in its subtree.
+ * @param hasAttributeOutsideOfAggregateExpressions A flag that highlights that a specific node
+ * corresponding to [[ExpressionResolutionContext]] has attributes in its subtree which are not
+ * under an [[AggregateExpression]].
* @param hasLateralColumnAlias A flag that highlights that a specific node corresponding to
- * [[ExpressionResolutionContext]] has LCA in its subtree.
+ * [[ExpressionResolutionContext]] has LCA in its subtree.
+ * @param isTopOfProjectList A flag indicating that we are resolving top of [[Project]] list.
+ * Otherwise, extra [[Alias]]es have to be stripped away.
+ * @param resolvingGroupingExpressions A flag indicating whether an expression we are resolving is
+ * one of [[Aggregate.groupingExpressions]].
*/
class ExpressionResolutionContext(
- var hasAggregateExpressionsInASubtree: Boolean = false,
- var hasAttributeInASubtree: Boolean = false,
- var hasLateralColumnAlias: Boolean = false) {
- def merge(other: ExpressionResolutionContext): Unit = {
- hasAggregateExpressionsInASubtree |= other.hasAggregateExpressionsInASubtree
- hasAttributeInASubtree |= other.hasAttributeInASubtree
- hasLateralColumnAlias |= other.hasLateralColumnAlias
+ val isRoot: Boolean = false,
+ var hasLocalReferences: Boolean = false,
+ var hasOuterReferences: Boolean = false,
+ var hasAggregateExpressions: Boolean = false,
+ var hasAttributeOutsideOfAggregateExpressions: Boolean = false,
+ var hasLateralColumnAlias: Boolean = false,
+ var isTopOfProjectList: Boolean = false,
+ var resolvingGroupingExpressions: Boolean = false) {
+
+ /**
+ * Propagate generic information that is valid across the whole expression tree from the
+ * [[child]] context.
+ */
+ def mergeChild(child: ExpressionResolutionContext): Unit = {
+ hasLocalReferences |= child.hasLocalReferences
+ hasOuterReferences |= child.hasOuterReferences
+ hasAggregateExpressions |= child.hasAggregateExpressions
+ hasAttributeOutsideOfAggregateExpressions |= child.hasAttributeOutsideOfAggregateExpressions
+ hasLateralColumnAlias |= child.hasLateralColumnAlias
+ }
+}
+
+object ExpressionResolutionContext {
+
+ /**
+ * Create a new child [[ExpressionResolutionContext]]. Propagates the relevant information
+ * top-down.
+ */
+ def createChild(parent: ExpressionResolutionContext): ExpressionResolutionContext = {
+ if (parent.isRoot) {
+ new ExpressionResolutionContext(
+ isTopOfProjectList = parent.isTopOfProjectList,
+ resolvingGroupingExpressions = parent.resolvingGroupingExpressions
+ )
+ } else {
+ new ExpressionResolutionContext(
+ resolvingGroupingExpressions = parent.resolvingGroupingExpressions
+ )
+ }
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionValidator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionValidator.scala
index 3ca62348e892..e0508e924678 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionValidator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolutionValidator.scala
@@ -22,10 +22,15 @@ import org.apache.spark.sql.catalyst.expressions.{
ArraysZip,
AttributeReference,
BinaryExpression,
+ Exists,
Expression,
+ InSubquery,
+ ListQuery,
Literal,
NamedExpression,
+ OuterReference,
Predicate,
+ ScalarSubquery,
TimeZoneAwareExpression
}
import org.apache.spark.sql.types.BooleanType
@@ -36,6 +41,12 @@ import org.apache.spark.sql.types.BooleanType
* logical plan. You can find more info in the [[ResolutionValidator]] scaladoc.
*/
class ExpressionResolutionValidator(resolutionValidator: ResolutionValidator) {
+ private val attributeScopeStack = resolutionValidator.getAttributeScopeStack
+
+ /**
+ * The flag to indicate if the validation process traverses the outer reference subtree.
+ */
+ private var inOuterReferenceSubtree = false
/**
* Validate resolved expression tree. The principle is the same as
@@ -51,6 +62,14 @@ class ExpressionResolutionValidator(resolutionValidator: ResolutionValidator) {
validateBinaryExpression(binaryExpression)
case literal: Literal =>
validateLiteral(literal)
+ case scalarSubquery: ScalarSubquery =>
+ validateScalarSubquery(scalarSubquery)
+ case inSubquery: InSubquery =>
+ validateInSubquery(inSubquery)
+ case listQuery: ListQuery =>
+ validateListQuery(listQuery)
+ case exists: Exists =>
+ validateExists(exists)
case predicate: Predicate =>
validatePredicate(predicate)
case arraysZip: ArraysZip =>
@@ -68,6 +87,8 @@ class ExpressionResolutionValidator(resolutionValidator: ResolutionValidator) {
validateAttributeReference(attributeReference)
case alias: Alias =>
validateAlias(alias)
+ case outerReference: OuterReference =>
+ validateOuterReference(outerReference)
}
}
@@ -82,9 +103,8 @@ class ExpressionResolutionValidator(resolutionValidator: ResolutionValidator) {
private def validateAttributeReference(attributeReference: AttributeReference): Unit = {
assert(
- resolutionValidator.attributeScopeStack.top.contains(attributeReference),
- s"Attribute $attributeReference is missing from attribute scope: " +
- s"${resolutionValidator.attributeScopeStack.top}"
+ attributeScopeStack.contains(attributeReference, isOuterReference = inOuterReferenceSubtree),
+ s"Attribute $attributeReference is missing from attribute scope stack: $attributeScopeStack"
)
}
@@ -92,6 +112,15 @@ class ExpressionResolutionValidator(resolutionValidator: ResolutionValidator) {
validate(alias.child)
}
+ private def validateOuterReference(outerReference: OuterReference): Unit = {
+ inOuterReferenceSubtree = true
+ try {
+ validate(outerReference.e)
+ } finally {
+ inOuterReferenceSubtree = false
+ }
+ }
+
private def validateBinaryExpression(binaryExpression: BinaryExpression): Unit = {
validate(binaryExpression.left)
validate(binaryExpression.right)
@@ -106,6 +135,49 @@ class ExpressionResolutionValidator(resolutionValidator: ResolutionValidator) {
private def validateLiteral(literal: Literal): Unit = {}
+ private def validateScalarSubquery(scalarSubquery: ScalarSubquery): Unit = {
+ attributeScopeStack.withNewScope(isSubqueryRoot = true) {
+ resolutionValidator.validate(scalarSubquery.plan)
+ }
+
+ for (outerAttribute <- scalarSubquery.outerAttrs) {
+ validate(outerAttribute)
+ }
+
+ assert(
+ scalarSubquery.plan.output.size == 1,
+ s"Scalar subquery returns more than one column: ${scalarSubquery.plan.output}"
+ )
+ }
+
+ private def validateInSubquery(inSubquery: InSubquery): Unit = {
+ for (value <- inSubquery.values) {
+ validate(value)
+ }
+
+ validate(inSubquery.query)
+ }
+
+ private def validateListQuery(listQuery: ListQuery): Unit = {
+ attributeScopeStack.withNewScope(isSubqueryRoot = true) {
+ resolutionValidator.validate(listQuery.plan)
+ }
+
+ for (outerAttribute <- listQuery.outerAttrs) {
+ validate(outerAttribute)
+ }
+ }
+
+ private def validateExists(exists: Exists): Unit = {
+ attributeScopeStack.withNewScope(isSubqueryRoot = true) {
+ resolutionValidator.validate(exists.plan)
+ }
+
+ for (outerAttribute <- exists.outerAttrs) {
+ validate(outerAttribute)
+ }
+ }
+
private def validateArraysZip(arraysZip: ArraysZip): Unit = {
arraysZip.children.foreach(validate)
arraysZip.names.foreach(validate)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala
index d6fbdad46fb1..6975493bf758 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionResolver.scala
@@ -17,7 +17,11 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import java.util.ArrayDeque
+import java.util.{ArrayDeque, HashMap, HashSet}
+
+import scala.annotation.nowarn
+import scala.collection.mutable
+import scala.jdk.CollectionConverters._
import org.apache.spark.SparkException
import org.apache.spark.sql.catalyst.analysis.{
@@ -30,31 +34,12 @@ import org.apache.spark.sql.catalyst.analysis.{
UnresolvedStar,
UpCastResolution
}
-import org.apache.spark.sql.catalyst.expressions.{
- Alias,
- AttributeReference,
- BinaryArithmetic,
- ConditionalExpression,
- CreateNamedStruct,
- DateAddYMInterval,
- Expression,
- ExtractIntervalPart,
- GetTimeField,
- Literal,
- MakeTimestamp,
- NamedExpression,
- Predicate,
- RuntimeReplaceable,
- TimeAdd,
- TimeZoneAwareExpression,
- UnaryMinus,
- UnresolvedNamedLambdaVariable,
- UpCast
-}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, LogicalPlan, Sort}
import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.catalyst.trees.TreeNodeTag
+import org.apache.spark.sql.catalyst.util.CollationFactory
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.MetadataBuilder
@@ -85,13 +70,36 @@ class ExpressionResolver(
extends TreeNodeResolver[Expression, Expression]
with ProducesUnresolvedSubtree
with ResolvesExpressionChildren {
- private val isLcaEnabled = conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)
/**
- * This is a flag indicating that we are resolving top of [[Project]] list. Otherwise extra
- * [[Alias]]es have to be stripped away.
+ * This field stores referenced attributes from the most recently resolved expression tree. It is
+ * populated in [[resolveExpressionTreeInOperatorImpl]] when [[ExpressionTreeTraversal]] is
+ * popped from the stack. It is the responsibility of the parent operator resolver to collect
+ * referenced attributes, before this field is overwritten.
+ */
+ private var lastReferencedAttributes: Option[HashMap[ExprId, Attribute]] = None
+
+ /**
+ * This field stores invalid expressions in the context of the parent operator from the most
+ * recently resolved expression tree. It is populated in [[resolveExpressionTreeInOperatorImpl]]
+ * when [[ExpressionTreeTraversal]] is popped from the stack. It is the responsibility of the
+ * parent operator resolver to collect invalid expressions, before this field is overwritten.
+ */
+ private var lastInvalidExpressionsInTheContextOfOperator: Option[Seq[Expression]] = None
+
+ /**
+ * This field contains the aliases of [[AggregateExpression]]s that were extracted during the
+ * most recent expression tree resolution. It is populated in
+ * [[resolveExpressionTreeInOperatorImpl]] when [[ExpressionTreeTraversal]] is popped from the
+ * stack.
*/
- private var isTopOfProjectList: Boolean = false
+ private var lastExtractedAggregateExpressionAliases: Option[Seq[Alias]] = None
+
+ /**
+ * This is a flag indicating that we are re-analyzing a resolved [[OuterReference]] subtree. It's
+ * managed by [[handleResolvedOuterReference]].
+ */
+ private var inOuterReferenceSubtree: Boolean = false
/**
* The stack of parent operators which were encountered during the resolution of a certain
@@ -117,10 +125,11 @@ class ExpressionResolver(
*
* Project -> Project
*/
- private val parentOperators = new ArrayDeque[LogicalPlan]
private val expressionIdAssigner = new ExpressionIdAssigner
+ private val traversals = new ExpressionTreeTraversalStack
private val expressionResolutionContextStack = new ArrayDeque[ExpressionResolutionContext]
private val scopes = resolver.getNameScopes
+ private val subqueryRegistry = resolver.getSubqueryRegistry
private val aliasResolver = new AliasResolver(this)
private val timezoneAwareExpressionResolver = new TimezoneAwareExpressionResolver(this)
@@ -132,10 +141,10 @@ class ExpressionResolver(
this,
timezoneAwareExpressionResolver
)
- private val limitExpressionResolver = new LimitExpressionResolver
+ private val limitLikeExpressionValidator = new LimitLikeExpressionValidator
private val typeCoercionResolver = new TypeCoercionResolver(timezoneAwareExpressionResolver)
private val aggregateExpressionResolver =
- new AggregateExpressionResolver(this, timezoneAwareExpressionResolver)
+ new AggregateExpressionResolver(resolver, this, timezoneAwareExpressionResolver)
private val functionResolver = new FunctionResolver(
this,
timezoneAwareExpressionResolver,
@@ -145,6 +154,51 @@ class ExpressionResolver(
)
private val timeAddResolver = new TimeAddResolver(this, timezoneAwareExpressionResolver)
private val unaryMinusResolver = new UnaryMinusResolver(this, timezoneAwareExpressionResolver)
+ private val subqueryExpressionResolver = new SubqueryExpressionResolver(this, resolver)
+
+ /**
+ * Get the expression tree traversal stack.
+ */
+ def getExpressionTreeTraversals: ExpressionTreeTraversalStack = traversals
+
+ /**
+ * Get the expression resolution context stack.
+ */
+ def getExpressionResolutionContextStack: ArrayDeque[ExpressionResolutionContext] =
+ expressionResolutionContextStack
+
+ def getExpressionIdAssigner: ExpressionIdAssigner = expressionIdAssigner
+
+ /**
+ * Get [[NameScopeStack]] bound to the used [[Resolver]].
+ */
+ def getNameScopes: NameScopeStack = scopes
+
+ /**
+ * Get the [[TypeCoercionResolver]] which contains all the transformations for generic coercion.
+ */
+ def getGenericTypeCoercionResolver: TypeCoercionResolver = typeCoercionResolver
+
+ /**
+ * Returns all attributes that have been referenced during the most recent expression tree
+ * resolution.
+ */
+ def getLastReferencedAttributes: HashMap[ExprId, Attribute] =
+ lastReferencedAttributes.getOrElse(new HashMap[ExprId, Attribute])
+
+ /**
+ * Returns all invalid expressions in the context of the parent operator from the most recent
+ * expression tree resolution.
+ */
+ def getLastInvalidExpressionsInTheContextOfOperator: Seq[Expression] =
+ lastInvalidExpressionsInTheContextOfOperator.getOrElse(Seq.empty)
+
+ /**
+ * Returns all aliases of [[AggregateExpression]]s that were extracted during the most recent
+ * expression tree resolution.
+ */
+ def getLastExtractedAggregateExpressionAliases: Seq[Alias] =
+ lastExtractedAggregateExpressionAliases.getOrElse(Seq.empty)
/**
* Resolve `unresolvedExpression` which is a child of `parentOperator`. This is the main entry
@@ -206,7 +260,11 @@ class ExpressionResolver(
case unresolvedLiteral: Literal =>
resolveLiteral(unresolvedLiteral)
case unresolvedPredicate: Predicate =>
- predicateResolver.resolve(unresolvedPredicate)
+ resolvePredicate(unresolvedPredicate)
+ case unresolvedScalarSubquery: ScalarSubquery =>
+ subqueryExpressionResolver.resolveScalarSubquery(unresolvedScalarSubquery)
+ case unresolvedListQuery: ListQuery =>
+ subqueryExpressionResolver.resolveListQuery(unresolvedListQuery)
case unresolvedTimeAdd: TimeAdd =>
timeAddResolver.resolve(unresolvedTimeAdd)
case unresolvedUnaryMinus: UnaryMinus =>
@@ -227,6 +285,8 @@ class ExpressionResolver(
timezoneAwareExpressionResolver.resolve(unresolvedTimezoneExpression)
case unresolvedUpCast: UpCast =>
resolveUpCast(unresolvedUpCast)
+ case unresolvedCollation: UnresolvedCollation =>
+ resolveCollation(unresolvedCollation)
case expression: Expression =>
resolveExpressionGenericallyWithTypeCoercion(expression)
}
@@ -234,8 +294,8 @@ class ExpressionResolver(
preserveTags(unresolvedExpression, resolvedExpression)
popResolutionContext()
- if (!resolvedExpression.resolved) {
- throwSinglePassFailedToResolveExpression(resolvedExpression)
+ withPosition(unresolvedExpression) {
+ validateResolvedExpressionGenerically(resolvedExpression)
}
planLogger.logExpressionTreeResolution(unresolvedExpression, resolvedExpression)
@@ -244,41 +304,20 @@ class ExpressionResolver(
}
/**
- * Get the expression resolution context stack.
- */
- def getExpressionResolutionContextStack: ArrayDeque[ExpressionResolutionContext] = {
- expressionResolutionContextStack
- }
-
- def getExpressionIdAssigner: ExpressionIdAssigner = expressionIdAssigner
-
- /**
- * Get the most recent operator (bottommost) from the `parentOperators` stack.
+ * Resolve and validate the limit like expressions from either [[LocalLimit]], [[GlobalLimit]],
+ * [[Offset]] or [[Tail]] operator.
*/
- def getParentOperator: Option[LogicalPlan] = {
- if (parentOperators.size() > 0) {
- Some(parentOperators.peek())
- } else {
- None
- }
- }
-
- /**
- * Get [[NameScopeStack]] bound to the used [[Resolver]].
- */
- def getNameScopes: NameScopeStack = scopes
-
- /**
- * Resolve the limit expression from either a [[LocalLimit]] or a [[GlobalLimit]] operator.
- */
- def resolveLimitExpression(
- unresolvedLimitExpr: Expression,
- unresolvedLimit: LogicalPlan): Expression = {
- val resolvedLimitExpr = resolveExpressionTreeInOperator(
- unresolvedLimitExpr,
- unresolvedLimit
+ def resolveLimitLikeExpression(
+ unresolvedLimitLikeExpr: Expression,
+ partiallyResolvedLimitLike: LogicalPlan): Expression = {
+ val resolvedLimitLikeExpr = resolveExpressionTreeInOperator(
+ unresolvedLimitLikeExpr,
+ partiallyResolvedLimitLike
+ )
+ limitLikeExpressionValidator.validateLimitLikeExpr(
+ resolvedLimitLikeExpr,
+ partiallyResolvedLimitLike
)
- limitExpressionResolver.resolve(resolvedLimitExpr)
}
/**
@@ -303,52 +342,220 @@ class ExpressionResolver(
* ResolvedProjectList(
* expressions = [count(col1) as count(col1), 2 AS 2],
* hasAggregateExpressions = true, // because it contains `count(col1)` in the project list
- * hasAttributes = false // because it doesn't contain any [[AttributeReference]]s in the
- * // project list (only under the aggregate expression, please check
- * // [[AggregateExpressionResolver]] for more details).
+ * )
*/
def resolveProjectList(
- unresolvedProjectList: Seq[NamedExpression],
+ sourceUnresolvedProjectList: Seq[NamedExpression],
operator: LogicalPlan): ResolvedProjectList = {
- val projectListResolutionContext = new ExpressionResolutionContext
- val resolvedProjectList = unresolvedProjectList.flatMap {
+ val unresolvedProjectList = tryDrainLazySequences(sourceUnresolvedProjectList)
+
+ var hasAggregateExpressions = false
+ var hasLateralColumnAlias = false
+
+ val unresolvedProjectListWithStarsExpanded = unresolvedProjectList.flatMap {
case unresolvedStar: UnresolvedStar =>
resolveStar(unresolvedStar)
- case other =>
- val (resolvedElement, resolvedElementContext) =
- resolveExpressionTreeInOperatorImpl(other, operator)
- projectListResolutionContext.merge(resolvedElementContext)
- Seq(resolvedElement.asInstanceOf[NamedExpression])
+ case other => Seq(other)
}
+
+ val resolvedProjectList = unresolvedProjectListWithStarsExpanded.flatMap { expression =>
+ val (resolvedElement, resolvedElementContext) = {
+ resolveExpressionTreeInOperatorImpl(
+ expression,
+ operator,
+ inProjectList = true
+ )
+ }
+
+ hasAggregateExpressions |= resolvedElementContext.hasAggregateExpressions
+ hasLateralColumnAlias |= resolvedElementContext.hasLateralColumnAlias
+
+ Seq(resolvedElement.asInstanceOf[NamedExpression])
+ }
+
ResolvedProjectList(
expressions = resolvedProjectList,
- hasAggregateExpressions = projectListResolutionContext.hasAggregateExpressionsInASubtree,
- hasAttributes = projectListResolutionContext.hasAttributeInASubtree,
- hasLateralColumnAlias = projectListResolutionContext.hasLateralColumnAlias
+ hasAggregateExpressions = hasAggregateExpressions,
+ hasLateralColumnAlias = hasLateralColumnAlias
)
}
/**
- * Resolves [[Expression]] by resolving its children and applying generic type coercion
- * transformations to the resulting expression. This resolution method is used for nodes that
- * require type coercion on top of [[resolveExpressionGenerically]].
+ * Resolve aggregate expressions in [[Aggregate]] operator.
+ *
+ * The [[Aggregate]] list can contain different unresolved expressions before the resolution,
+ * which will be resolved using generic [[resolve]]. However, [[UnresolvedStar]] is a special
+ * case, because it is expanded into a sequence of [[NamedExpression]]s. Because of that this
+ * method returns a sequence and doesn't conform to generic [[resolve]] interface - it's called
+ * directly from the [[AggregateResolver]] during [[Aggregate]] resolution.
+ *
+ * Besides resolution, we do the following:
+ * - If there is a [[UnresolvedStar]] in the list we set `hasStar` to true in order to throw
+ * if there are any ordinals in grouping expressions.
+ * Example of an invalid query:
+ * {{{ SELECT * FROM VALUES(1) GROUP BY 1; }}}
+ *
+ * - If there is an expression which has aggregate function in its subtree, we add it to the
+ * `expressionsWithAggregateFunctions` list in order to throw if there is any ordinal in
+ * grouping expressions which references that aggregate expression.
+ * Example of an invalid query:
+ * {{{ SELECT count(col1) FROM VALUES(1) GROUP BY 1; }}}
+ *
+ * - If the resolved expression is an [[Alias]], add it to
+ * `scopes.current.topAggregateExpressionsByAliasName` so it can be used for grouping
+ * expressions resolution, if needed.
+ * Example of a query with an [[Alias]]:
+ * 1. Implicit alias:
+ * {{{ SELECT col1 + col2 FROM VALUES(1, 2) GROUP BY `(col1 + col2)`; }}}
+ * 2. Explicit alias:
+ * {{{ SELECT 1 AS column GROUP BY column; }}}
+ *
+ * While resolving the list, we have to keep track of all the expressions that don't have
+ * [[AggregateExpression]]s in their subtrees (`expressionsWithoutAggregates`) and whether any of
+ * aggregate expressions (that are not `expressionsWithoutAggregates`) has attributes in the
+ * subtree outside of [[AggregateExpressions]]s (`hasAttributeOutsideOfAggregateExpressions`).
+ * This is used when resolving `GROUP BY ALL` in the [[AggregateResolver.resolveGroupByAll]].
+ *
+ * @returns List of resolved expressions, list of expressions that don't have
+ * [[AggregateExpression]] in their subtrees, if any of resolved expressions have
+ * attributes in the subtree that are not under an [[AggregateExpression]], if any of
+ * expressions is a star (`*`) and list of indices of expressions that have aggregate
+ * functions in the subtree encapsulated in [[ResolvedAggregateExpressions]].
*/
- def resolveExpressionGenericallyWithTypeCoercion(expression: Expression): Expression = {
- val expressionWithResolvedChildren = withResolvedChildren(expression, resolve)
- typeCoercionResolver.resolve(expressionWithResolvedChildren)
+ def resolveAggregateExpressions(
+ sourceUnresolvedAggregateExpressions: Seq[NamedExpression],
+ unresolvedAggregate: Aggregate): ResolvedAggregateExpressions = {
+ val unresolvedAggregateExpressions = tryDrainLazySequences(sourceUnresolvedAggregateExpressions)
+
+ val expressionsWithoutAggregates = new mutable.ArrayBuffer[NamedExpression]
+ val expressionIndexesWithAggregateFunctions = new HashSet[Int]
+ var hasAttributeOutsideOfAggregateExpressions = false
+ var hasStar = false
+
+ val unresolvedAggregateExpressionsWithStarsExpanded = unresolvedAggregateExpressions.flatMap {
+ case unresolvedStar: UnresolvedStar =>
+ hasStar = true
+ resolveStar(unresolvedStar)
+ case other => Seq(other)
+ }
+
+ val resolvedAggregateExpressions =
+ unresolvedAggregateExpressionsWithStarsExpanded.zipWithIndex.flatMap {
+ case (expression, index) =>
+ val (resolvedElement, resolvedElementContext) = resolveExpressionTreeInOperatorImpl(
+ expression,
+ unresolvedAggregate,
+ inProjectList = true
+ )
+
+ resolvedElement match {
+ case alias: Alias =>
+ scopes.current.addTopAggregateExpression(alias)
+ case other =>
+ }
+
+ if (resolvedElementContext.hasAggregateExpressions) {
+ expressionIndexesWithAggregateFunctions.add(index)
+ hasAttributeOutsideOfAggregateExpressions |=
+ resolvedElementContext.hasAttributeOutsideOfAggregateExpressions
+ } else {
+ expressionsWithoutAggregates += resolvedElement.asInstanceOf[NamedExpression]
+ }
+
+ Seq(resolvedElement.asInstanceOf[NamedExpression])
+ }
+
+ val isLcaEnabled = conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)
+ if (isLcaEnabled && scopes.current.lcaRegistry.getAliasDependencyLevels().size() > 1) {
+ throw new ExplicitlyUnsupportedResolverFeature("LateralColumnAlias in aggregate expressions")
+ }
+
+ ResolvedAggregateExpressions(
+ expressions = resolvedAggregateExpressions,
+ resolvedExpressionsWithoutAggregates = expressionsWithoutAggregates.toSeq,
+ hasAttributeOutsideOfAggregateExpressions = hasAttributeOutsideOfAggregateExpressions,
+ hasStar = hasStar,
+ expressionIndexesWithAggregateFunctions = expressionIndexesWithAggregateFunctions
+ )
+ }
+
+ /**
+ * Resolve grouping expressions in [[Aggregate]] operator.
+ *
+ * It's done for every expression using the `resolveExpressionTreeInOperatorImpl`. For cases where
+ * grouping is done based on aliases the resolution is following:
+ * - If the expression can be resolved using the child's output (`scopes.current.output`),
+ * resolve it that way.
+ * Example:
+ * {{{ SELECT col1 FROM VALUES(1) GROUP BY `col1`; }}}
+ *
+ * - If not, try to resolve it as a top level [[Alias]] (which was populated during the
+ * resolution of the aggregate expressions).
+ * Example:
+ * 1. Group by implicit alias
+ * {{{ SELECT concat_ws(' ', 'a', 'b') GROUP BY `concat_ws( , a, b)`; }}}
+ * 2. Group by explicit alias
+ * {{{ SELECT col1 AS column_1 FROM VALUES(1) GROUP BY column_1; }}}
+ */
+ def resolveGroupingExpressions(
+ sourceUnresolvedGroupingExpressions: Seq[Expression],
+ unresolvedAggregate: Aggregate): Seq[Expression] = {
+ val unresolvedGroupingExpressions = tryDrainLazySequences(sourceUnresolvedGroupingExpressions)
+
+ unresolvedGroupingExpressions.map { expression =>
+ val (resolvedExpression, _) = resolveExpressionTreeInOperatorImpl(
+ expression,
+ unresolvedAggregate,
+ resolvingGroupingExpressions = true
+ )
+
+ resolvedExpression
+ }
+ }
+
+ /**
+ * Validate if `expression` is under supported operator or not. In case it's not, add `expression`
+ * to the [[ExpressionTreeTraversal.invalidExpressionsInTheContextOfOperator]] list to throw
+ * error later, when [[getLastInvalidExpressionsInTheContextOfOperator]] is called by the
+ * [[Resolver]].
+ */
+ def validateExpressionUnderSupportedOperator(expression: Expression): Unit = {
+ if (UnsupportedExpressionInOperatorValidation.isExpressionInUnsupportedOperator(
+ expression,
+ traversals.current.parentOperator
+ )) {
+ traversals.current.invalidExpressionsInTheContextOfOperator.add(expression)
+ }
}
private def resolveExpressionTreeInOperatorImpl(
unresolvedExpression: Expression,
- parentOperator: LogicalPlan): (Expression, ExpressionResolutionContext) = {
- this.parentOperators.push(parentOperator)
- expressionResolutionContextStack.push(new ExpressionResolutionContext)
- try {
- val resolvedExpression = resolve(unresolvedExpression)
- (resolvedExpression, expressionResolutionContextStack.peek())
- } finally {
- expressionResolutionContextStack.pop()
- this.parentOperators.pop()
+ parentOperator: LogicalPlan,
+ inProjectList: Boolean = false,
+ resolvingGroupingExpressions: Boolean = false
+ ): (Expression, ExpressionResolutionContext) = {
+ traversals.withNewTraversal(parentOperator) {
+ expressionResolutionContextStack.push(
+ new ExpressionResolutionContext(
+ isRoot = true,
+ isTopOfProjectList = inProjectList,
+ resolvingGroupingExpressions = resolvingGroupingExpressions
+ )
+ )
+
+ try {
+ val resolvedExpression = resolve(unresolvedExpression)
+
+ lastReferencedAttributes = Some(traversals.current.referencedAttributes)
+ lastInvalidExpressionsInTheContextOfOperator =
+ Some(traversals.current.invalidExpressionsInTheContextOfOperator.asScala.toSeq)
+ lastExtractedAggregateExpressionAliases =
+ Some(traversals.current.extractedAggregateExpressionAliases.asScala.toSeq)
+
+ (resolvedExpression, expressionResolutionContextStack.peek())
+ } finally {
+ expressionResolutionContextStack.pop()
+ }
}
}
@@ -365,6 +572,8 @@ class ExpressionResolver(
throw new ExplicitlyUnsupportedResolverFeature("Star outside of Project list")
case attributeReference: AttributeReference =>
handleResolvedAttributeReference(attributeReference)
+ case outerReference: OuterReference =>
+ handleResolvedOuterReference(outerReference)
case _: UnresolvedNamedLambdaVariable =>
throw new ExplicitlyUnsupportedResolverFeature("Lambda variables")
case _ =>
@@ -373,6 +582,32 @@ class ExpressionResolver(
}
}
+ /**
+ * [[UnresolvedStar]] resolution relies on the [[NameScope]]'s ability to get the attributes by a
+ * multipart name ([[UnresolvedStar]]'s `target` field):
+ *
+ * - Star target is defined:
+ *
+ * {{{
+ * SELECT t.* FROM VALUES (1) AS t;
+ * ->
+ * Project [col1#19]
+ * }}}
+ *
+ *
+ * - Star target is not defined:
+ *
+ * {{{
+ * SELECT * FROM (SELECT 1 as col1), (SELECT 2 as col2);
+ * ->
+ * Project [col1#19, col2#20]
+ * }}}
+ */
+ private def resolveStar(unresolvedStar: UnresolvedStar): Seq[NamedExpression] =
+ withPosition(unresolvedStar) {
+ scopes.current.expandStar(unresolvedStar)
+ }
+
/**
* [[UnresolvedAttribute]] resolution relies on [[NameScope]] to lookup the attribute by its
* multipart name. The resolution can result in three different outcomes which are handled in the
@@ -391,94 +626,170 @@ class ExpressionResolver(
* attribute that is a lateral column alias reference. In that case we mark the referenced
* attribute as referenced and tag the LCA attribute for further [[Alias]] resolution.
*
+ * In case that attribute is resolved as a literal function (i.e. result is [[CurrentDate]]),
+ * perform additional resolution on it.
+ *
* If the attribute is at the top of the project list (which is indicated by
- * [[isTopOfProjectList]]), we preserve the [[Alias]] or remove it otherwise.
+ * [[ExpressionResolutionContext.isTopOfProjectList]]), we preserve the [[Alias]] or remove it
+ * otherwise.
*
- * Finally, we remap the expression ID of a top [[NamedExpression]]. It's not necessary to remap
- * the expression ID of lower expressions, because they already got the appropriate ID from the
+ * Finally, we remap the expression ID of a top [[Alias]]. It's not necessary to remap the
+ * expression ID of lower expressions, because they already got the appropriate ID from the
* current scope output in [[resolveMultipartName]].
*/
private def resolveAttribute(unresolvedAttribute: UnresolvedAttribute): Expression =
withPosition(unresolvedAttribute) {
- expressionResolutionContextStack.peek().hasAttributeInASubtree = true
+ val isLcaEnabled = conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)
+ val expressionResolutionContext = expressionResolutionContextStack.peek()
- val nameTarget: NameTarget =
- scopes.top.resolveMultipartName(unresolvedAttribute.nameParts, isLcaEnabled)
+ val nameTarget: NameTarget = scopes.resolveMultipartName(
+ multipartName = unresolvedAttribute.nameParts,
+ canLaterallyReferenceColumn = canLaterallyReferenceColumn(isLcaEnabled),
+ canReferenceAggregateExpressionAliases = (
+ expressionResolutionContextStack
+ .peek()
+ .resolvingGroupingExpressions && conf.groupByAliases
+ ),
+ canResolveNameByHiddenOutput = canResolveNameByHiddenOutput
+ )
val candidate = nameTarget.pickCandidate(unresolvedAttribute)
+ expressionResolutionContext.hasAttributeOutsideOfAggregateExpressions = true
+ if (nameTarget.isOuterReference) {
+ expressionResolutionContext.hasOuterReferences = true
+ } else {
+ expressionResolutionContext.hasLocalReferences = true
+ }
+
if (isLcaEnabled) {
nameTarget.lateralAttributeReference match {
case Some(lateralAttributeReference) =>
- scopes.top.lcaRegistry
+ scopes.current.lcaRegistry
.markAttributeLaterallyReferenced(lateralAttributeReference)
candidate.setTagValue(ExpressionResolver.SINGLE_PASS_IS_LCA, ())
- expressionResolutionContextStack.peek().hasLateralColumnAlias = true
+ expressionResolutionContext.hasLateralColumnAlias = true
case None =>
}
}
+ tryAddReferencedAttribute(candidate)
+
+ val candidateOrLiteralFunction = candidate match {
+ case currentDate: CurrentDate =>
+ timezoneAwareExpressionResolver.resolve(currentDate)
+ case other => other
+ }
+
val properlyAliasedExpressionTree =
- if (isTopOfProjectList && nameTarget.aliasName.isDefined) {
- Alias(candidate, nameTarget.aliasName.get)()
+ if (expressionResolutionContext.isTopOfProjectList && nameTarget.aliasName.isDefined) {
+ Alias(candidateOrLiteralFunction, nameTarget.aliasName.get)()
} else {
- candidate
+ candidateOrLiteralFunction
}
properlyAliasedExpressionTree match {
- case namedExpression: NamedExpression =>
- expressionIdAssigner.mapExpression(namedExpression)
+ case alias: Alias =>
+ expressionIdAssigner.mapExpression(alias)
case _ =>
properlyAliasedExpressionTree
}
}
+ private def canResolveNameByHiddenOutput = traversals.current.parentOperator match {
+ case operator @ (_: Filter | _: Sort) => true
+ case other => false
+ }
+
/**
* [[AttributeReference]] is already resolved if it's passed to us from DataFrame `col(...)`
* function, for example.
+ *
+ * After mapping the [[AttributeReference]] to a correct [[ExprId]], we need to assert that the
+ * attribute exists in current [[NameScope]]. If the attribute in the current scope is nullable,
+ * we need to preserve this nullability in the [[AttributeReference]] as well. This is necessary
+ * because of the following case:
+ *
+ * {{{
+ * val df1 = Seq((1, 1)).toDF("a", "b")
+ * val df2 = Seq((2, 2)).toDF("a", "b")
+ * df1.join(df2, df1("a") === df2("a"), "outer")
+ * .select(coalesce(df1("a"), df1("b")), coalesce(df2("a"), df2("b"))),
+ * }}}
+ *
+ * Because of the outer join, after df1.join(df2), the output of the join will have nullable
+ * attributes "a" and "b" that come from df2. When these attributes are referenced in
+ * `select(coalesce(df2("a"), df2("b"))`, they need to inherit nullability of join's output, or
+ * retain nullability of [[AttributeReference]], if it was true.
+ *
+ * Without this, a nullable column's nullable field can be actually set as non-nullable, which
+ * can cause illegal optimization (e.g., NULL propagation) and wrong answers. See SPARK-13484 and
+ * SPARK-13801 for the concrete queries of this case.
*/
private def handleResolvedAttributeReference(attributeReference: AttributeReference) = {
+ val expressionResolutionContext = expressionResolutionContextStack.peek()
+
+ expressionResolutionContext.hasAttributeOutsideOfAggregateExpressions = true
+
val strippedAttributeReference = tryStripAmbiguousSelfJoinMetadata(attributeReference)
- val resultAttribute = expressionIdAssigner.mapExpression(strippedAttributeReference)
- if (!scopes.top.hasAttributeWithId(resultAttribute.exprId)) {
- throw new ExplicitlyUnsupportedResolverFeature("DataFrame missing attribute propagation")
+ val resultAttribute = if (!inOuterReferenceSubtree) {
+ expressionResolutionContext.hasLocalReferences = true
+
+ expressionIdAssigner.mapExpression(strippedAttributeReference)
+ } else {
+ expressionResolutionContext.hasOuterReferences = true
+
+ expressionIdAssigner.mapOuterReference(strippedAttributeReference)
+ }
+
+ val existingAttributeWithId = scopes.current.getAttributeById(resultAttribute.exprId)
+ val resultAttributeWithNullability = if (existingAttributeWithId.isEmpty) {
+ resultAttribute
+ } else {
+ val nullability = existingAttributeWithId.get.nullable || resultAttribute.nullable
+ resultAttribute.withNullability(nullability)
}
- resultAttribute
+ tryAddReferencedAttribute(resultAttributeWithNullability)
+
+ resultAttributeWithNullability
}
/**
- * [[UnresolvedStar]] resolution relies on the [[NameScope]]'s ability to get the attributes by a
- * multipart name ([[UnresolvedStar]]'s `target` field):
- *
- * - Star target is defined:
- *
- * {{{
- * SELECT t.* FROM VALUES (1) AS t;
- * ->
- * Project [col1#19]
- * }}}
- *
- *
- * - Star target is not defined:
- *
- * {{{
- * SELECT * FROM (SELECT 1 as col1), (SELECT 2 as col2);
- * ->
- * Project [col1#19, col2#20]
- * }}}
+ * While handling the resolved [[OuterReference]] we need to set [[inOuterReferenceSubtree]] to
+ * `true` to correctly remap [[AttributeReference]] expression IDs using [[ExpressionIdAssigner]]
+ * using outer expression ID mapping.
*/
- def resolveStar(unresolvedStar: UnresolvedStar): Seq[NamedExpression] =
- withPosition(unresolvedStar) {
- scopes.top.expandStar(unresolvedStar)
+ private def handleResolvedOuterReference(outerReference: OuterReference): Expression = {
+ inOuterReferenceSubtree = true
+ try {
+ OuterReference(e = resolve(outerReference.e).asInstanceOf[NamedExpression])
+ } finally {
+ inOuterReferenceSubtree = false
}
+ }
/**
* [[Literal]] resolution doesn't require any specific resolution logic at this point.
*/
private def resolveLiteral(literal: Literal): Expression = literal
+ /**
+ * Resolve [[Predicate]] expression using [[PredicateResolver]]. Subquery expressions are a
+ * special case and require special resolution logic.
+ */
+ private def resolvePredicate(unresolvedPredicate: Predicate): Expression = {
+ unresolvedPredicate match {
+ case unresolvedInSubquery: InSubquery =>
+ subqueryExpressionResolver.resolveInSubquery(unresolvedInSubquery)
+ case unresolvedExists: Exists =>
+ subqueryExpressionResolver.resolveExists(unresolvedExists)
+ case _ =>
+ predicateResolver.resolve(unresolvedPredicate)
+ }
+ }
+
/**
* The [[GetViewColumnByNameAndOrdinal]] is a special internal expression that is placed by the
* [[SessionCatalog]] in the top [[Project]] operator of the freshly reconstructed unresolved
@@ -540,7 +851,7 @@ class ExpressionResolver(
*/
private def resolveGetViewColumnByNameAndOrdinal(
getViewColumnByNameAndOrdinal: GetViewColumnByNameAndOrdinal): Expression = {
- val candidates = scopes.top.findAttributesByName(getViewColumnByNameAndOrdinal.colName)
+ val candidates = scopes.current.findAttributesByName(getViewColumnByNameAndOrdinal.colName)
if (candidates.length != getViewColumnByNameAndOrdinal.expectedNumCandidates) {
throw QueryCompilationErrors.incompatibleViewSchemaChangeError(
getViewColumnByNameAndOrdinal.viewName,
@@ -569,44 +880,44 @@ class ExpressionResolver(
}
/**
- * Resolves [[Expression]] by calling [[timezoneAwareExpressionResolver]] to resolve
- * expression's children and apply timezone if needed. Applies generic type coercion
- * rules to the result.
+ * Collation resolution requires resolving its collation name using [[CollationFactory]].
*/
- private def resolveExpressionGenericallyWithTimezoneWithTypeCoercion(
- timezoneAwareExpression: TimeZoneAwareExpression): Expression = {
- val expressionWithTimezone = timezoneAwareExpressionResolver.resolve(timezoneAwareExpression)
- typeCoercionResolver.resolve(expressionWithTimezone)
+ private def resolveCollation(unresolvedCollation: UnresolvedCollation): Expression = {
+ ResolvedCollation(
+ CollationFactory.resolveFullyQualifiedName(unresolvedCollation.collationName.toArray)
+ )
}
- /**
- * Resolves [[Expression]] only by resolving its children. This resolution method is used for
- * nodes that don't require any special resolution other than resolving its children.
- */
- private def resolveExpressionGenerically(expression: Expression): Expression =
- withResolvedChildren(expression, resolve)
+ private def pushResolutionContext(): Unit = {
+ val parentContext = expressionResolutionContextStack.peek()
+ expressionResolutionContextStack.push(ExpressionResolutionContext.createChild(parentContext))
+ }
private def popResolutionContext(): Unit = {
- val currentExpressionResolutionContext = expressionResolutionContextStack.pop()
- expressionResolutionContextStack.peek().merge(currentExpressionResolutionContext)
+ val childContext = expressionResolutionContextStack.pop()
+ expressionResolutionContextStack.peek().mergeChild(childContext)
}
- private def pushResolutionContext(): Unit = {
- isTopOfProjectList = expressionResolutionContextStack
- .size() == 1 && parentOperators.peek().isInstanceOf[Project]
-
- expressionResolutionContextStack.push(new ExpressionResolutionContext)
+ private def tryAddReferencedAttribute(expression: Expression) = expression match {
+ case attribute: Attribute =>
+ traversals.current.referencedAttributes.put(attribute.exprId, attribute)
+ case extractValue: ExtractValue =>
+ extractValue.foreach {
+ case attribute: Attribute =>
+ traversals.current.referencedAttributes.put(attribute.exprId, attribute)
+ case _ =>
+ }
+ case _ =>
}
- private def tryPopSinglePassSubtreeBoundary(unresolvedExpression: Expression): Boolean = {
- if (unresolvedExpression
- .getTagValue(ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY)
- .isDefined) {
- unresolvedExpression.unsetTagValue(ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY)
- true
- } else {
- false
- }
+ /**
+ * Returns true if LateralColumnAlias resolution is enabled, current operator is not created
+ * because of a generated column and current expression is a grouping one (grouping expressions)
+ * can't reference an LCA.
+ */
+ private def canLaterallyReferenceColumn(isLcaEnabled: Boolean): Boolean = {
+ isLcaEnabled &&
+ !expressionResolutionContextStack.peek().resolvingGroupingExpressions
}
/**
@@ -644,6 +955,63 @@ class ExpressionResolver(
}
}
+ /**
+ * Resolves [[Expression]] only by resolving its children. This resolution method is used for
+ * nodes that don't require any special resolution other than resolving its children.
+ */
+ private def resolveExpressionGenerically(expression: Expression): Expression =
+ withResolvedChildren(expression, resolve _)
+
+ /**
+ * Resolves [[Expression]] by resolving its children and applying generic type coercion
+ * transformations to the resulting expression. This resolution method is used for nodes that
+ * require type coercion on top of [[resolveExpressionGenerically]].
+ */
+ private def resolveExpressionGenericallyWithTypeCoercion(expression: Expression): Expression = {
+ val expressionWithResolvedChildren = withResolvedChildren(expression, resolve _)
+ typeCoercionResolver.resolve(expressionWithResolvedChildren)
+ }
+
+ /**
+ * Resolves [[Expression]] by calling [[timezoneAwareExpressionResolver]] to resolve
+ * expression's children and apply timezone if needed. Applies generic type coercion
+ * rules to the result.
+ */
+ private def resolveExpressionGenericallyWithTimezoneWithTypeCoercion(
+ timezoneAwareExpression: TimeZoneAwareExpression): Expression = {
+ val expressionWithTimezone = timezoneAwareExpressionResolver.resolve(timezoneAwareExpression)
+ typeCoercionResolver.resolve(expressionWithTimezone)
+ }
+
+ private def validateResolvedExpressionGenerically(resolvedExpression: Expression): Unit = {
+ if (!resolvedExpression.resolved) {
+ throwSinglePassFailedToResolveExpression(resolvedExpression)
+ }
+
+ validateExpressionUnderSupportedOperator(resolvedExpression)
+ }
+
+ /**
+ * Transform list to a non-lazy one. This is needed in order to enforce determinism in
+ * single-pass resolver. Example:
+ *
+ * val groupByColumns = LazyList(col("key"))
+ * val df = Seq((1, 2)).toDF("key", "value")
+ * df.groupBy(groupByCols: _*)
+ *
+ * For this case, it is necessary to transform `LazyList` (lazy object) to `List` (non-lazy).
+ * Other object which has to be transformed is `Stream`.
+ */
+ private def tryDrainLazySequences(list: Seq[Expression]): Seq[Expression] = {
+ @nowarn("cat=deprecation")
+ val result = list match {
+ case lazyObject @ (_: LazyList[_] | _: Stream[_]) =>
+ lazyObject.toList
+ case other => other
+ }
+ result
+ }
+
private def throwUnsupportedSinglePassAnalyzerFeature(unresolvedExpression: Expression): Nothing =
throw QueryCompilationErrors.unsupportedSinglePassAnalyzerFeature(
s"${unresolvedExpression.getClass} expression resolution"
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionTreeTraversal.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionTreeTraversal.scala
new file mode 100644
index 000000000000..ef3feb11c399
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ExpressionTreeTraversal.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.{ArrayDeque, ArrayList, HashMap}
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Expression, ExprId}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+/**
+ * Properties of a current expression tree traversal.
+ *
+ * @param parentOperator The parent operator of the current expression tree.
+ * @param invalidExpressionsInTheContextOfOperator The expressions that are invalid in the context
+ * of the current expression tree and its parent operator.
+ * @param referencedAttributes All attributes that are referenced during the resolution of
+ * expression trees.
+ * @param extractedAggregateExpressionAliases The aliases of the [[AggregateExpressions]] that are
+ * extracted during the resolution of expression trees.
+ */
+case class ExpressionTreeTraversal(
+ parentOperator: LogicalPlan,
+ invalidExpressionsInTheContextOfOperator: ArrayList[Expression] = new ArrayList[Expression],
+ referencedAttributes: HashMap[ExprId, Attribute] = new HashMap[ExprId, Attribute],
+ extractedAggregateExpressionAliases: ArrayList[Alias] = new ArrayList[Alias]
+)
+
+/**
+ * The stack of expression tree traversal properties which are accumulated during the resolution
+ * of a certain expression tree. This is filled by the
+ * [[ExpressionResolver.resolveExpressionTreeInOperatorImpl]],
+ * and will usually have size 1. However, in case of subquery expressions we would call
+ * [[ExpressionResolver.resolveExpressionTreeInOperatorImpl]] several times recursively
+ * for each expression tree in the operator tree -> expression tree -> operator tree ->
+ * expression tree -> ... chain. Consider this example:
+ *
+ * {{{
+ * SELECT
+ * col1
+ * FROM
+ * VALUES (1) AS t1
+ * WHERE EXISTS (
+ * SELECT
+ * *
+ * FROM
+ * VALUES (2) AS t2
+ * WHERE
+ * (SELECT col1 FROM VALUES (3) AS t3) == t1.col1
+ * )
+ * }}}
+ *
+ * We would have 3 nested stack entries for while resolving the lower scalar subquery (with the `t3`
+ * table).
+ */
+class ExpressionTreeTraversalStack {
+ private val stack = new ArrayDeque[ExpressionTreeTraversal]
+
+ /**
+ * Current expression tree traversal properties. Must exist when resolving an expression tree.
+ */
+ def current: ExpressionTreeTraversal = {
+ if (stack.isEmpty) {
+ throw SparkException.internalError("No current expression tree traversal")
+ }
+ stack.peek()
+ }
+
+ /**
+ * Pushes a new [[ExpressionTreeTraversal]] object, executes the `body` and finally pops the
+ * traversal from the stack.
+ */
+ def withNewTraversal[R](parentOperator: LogicalPlan)(body: => R): R = {
+ stack.push(ExpressionTreeTraversal(parentOperator = parentOperator))
+ try {
+ body
+ } finally {
+ stack.pop()
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FilterResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FilterResolver.scala
new file mode 100644
index 000000000000..d6192ee3d273
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FilterResolver.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.withPosition
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan}
+import org.apache.spark.sql.types.BooleanType
+
+/**
+ * Resolves [[Filter]] node and its condition.
+ */
+class FilterResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
+ extends TreeNodeResolver[Filter, LogicalPlan]
+ with ResolvesNameByHiddenOutput {
+ override protected val scopes: NameScopeStack = resolver.getNameScopes
+
+ /**
+ * Resolve [[Filter]] by resolving its child and its condition. If an attribute that is used in
+ * the condition is not present in child's output, we need to try and find the attribute in
+ * hidden output. If found, update child's output and place a [[Project]] node on top of original
+ * [[Filter]] with the original output of a [[Filter]]'s child.
+ *
+ * See [[ResolvesNameByHiddenOutput]] doc for more context.
+ */
+ override def resolve(unresolvedFilter: Filter): LogicalPlan = {
+ val resolvedChild = resolver.resolve(unresolvedFilter.child)
+
+ val partiallyResolvedFilter = unresolvedFilter.copy(child = resolvedChild)
+ val resolvedCondition = expressionResolver.resolveExpressionTreeInOperator(
+ partiallyResolvedFilter.condition,
+ partiallyResolvedFilter
+ )
+
+ val referencedAttributes = expressionResolver.getLastReferencedAttributes
+
+ val resolvedFilter = Filter(resolvedCondition, resolvedChild)
+
+ checkValidFilter(unresolvedFilter, resolvedFilter)
+
+ val missingAttributes: Seq[Attribute] =
+ scopes.current.resolveMissingAttributesByHiddenOutput(referencedAttributes)
+ val resolvedChildWithMissingAttributes =
+ insertMissingExpressions(resolvedChild, missingAttributes)
+ val finalFilter = resolvedFilter.copy(child = resolvedChildWithMissingAttributes)
+
+ retainOriginalOutput(finalFilter, missingAttributes)
+ }
+
+ private def checkValidFilter(unresolvedFilter: Filter, resolvedFilter: Filter): Unit = {
+ withPosition(unresolvedFilter) {
+ val invalidExpressions = expressionResolver.getLastInvalidExpressionsInTheContextOfOperator
+ if (invalidExpressions.nonEmpty) {
+ throwInvalidWhereCondition(resolvedFilter, invalidExpressions)
+ }
+
+ if (resolvedFilter.condition.dataType != BooleanType) {
+ throwDataTypeMismatchFilterNotBoolean(resolvedFilter)
+ }
+ }
+ }
+
+ private def throwInvalidWhereCondition(
+ filter: Filter,
+ invalidExpressions: Seq[Expression]): Nothing = {
+ throw new AnalysisException(
+ errorClass = "INVALID_WHERE_CONDITION",
+ messageParameters = Map(
+ "condition" -> toSQLExpr(filter.condition),
+ "expressionList" -> invalidExpressions.map(_.sql).mkString(", ")
+ )
+ )
+ }
+
+ private def throwDataTypeMismatchFilterNotBoolean(filter: Filter): Nothing =
+ throw new AnalysisException(
+ errorClass = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
+ messageParameters = Map(
+ "sqlExpr" -> makeCommaSeparatedExpressionString(filter.expressions),
+ "filter" -> toSQLExpr(filter.condition),
+ "type" -> toSQLType(filter.condition.dataType)
+ )
+ )
+
+ private def makeCommaSeparatedExpressionString(expressions: Seq[Expression]): String = {
+ expressions.map(toSQLExpr).mkString(", ")
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolver.scala
index 04fd03d17a86..e02cd600b888 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/FunctionResolver.scala
@@ -97,7 +97,8 @@ class FunctionResolver(
if (isCountStarExpansionAllowed(unresolvedFunction)) {
normalizeCountExpression(unresolvedFunction)
} else {
- withResolvedChildren(unresolvedFunction, expressionResolver.resolve)
+ withResolvedChildren(unresolvedFunction, expressionResolver.resolve _)
+ .asInstanceOf[UnresolvedFunction]
}
var resolvedFunction = functionResolution.resolveFunction(functionWithResolvedChildren)
@@ -116,7 +117,9 @@ class FunctionResolver(
// Since this [[InheritAnalysisRules]] node is created by
// [[FunctionResolution.resolveFunction]], we need to re-resolve its replacement
// expression.
- expressionResolver.resolveExpressionGenericallyWithTypeCoercion(inheritAnalysisRules)
+ val resolvedInheritAnalysisRules =
+ withResolvedChildren(inheritAnalysisRules, expressionResolver.resolve _)
+ typeCoercionResolver.resolve(resolvedInheritAnalysisRules)
case aggregateExpression: AggregateExpression =>
// In case `functionResolution.resolveFunction` produces a `AggregateExpression` we
// need to apply further resolution which is done in the
@@ -135,7 +138,7 @@ class FunctionResolver(
typeCoercionResolver.resolve(other)
}
- timezoneAwareExpressionResolver.withResolvedTimezoneCopyTags(
+ timezoneAwareExpressionResolver.withResolvedTimezone(
resolvedFunction,
conf.sessionLocalTimeZone
)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala
index 0c1ed75e1e15..8d55a52bc0d6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.analysis.resolver
+import java.util.Random
+
import scala.util.control.NonFatal
import org.apache.spark.sql.catalyst.{QueryPlanningTracker, SQLConfHelper}
@@ -37,9 +39,10 @@ import org.apache.spark.sql.internal.SQLConf
* - If the "spark.sql.analyzer.singlePassResolver.dualRunEnabled" is "true", the
* [[HybridAnalyzer]] will invoke the legacy analyzer and optionally _also_ the fixed-point
* one depending on the structure of the unresolved plan. This decision is based on which
- * features are supported by the single-pass Analyzer, and the checking is implemented in
- * the [[ResolverGuard]]. After that we validate the results using the following
- * logic:
+ * features are supported by the single-pass Analyzer, and the checking is implemented in the
+ * [[ResolverGuard]]. It's also determined if the query should be run in dual run mode by
+ * the [[SQLConf.ANALYZER_DUAL_RUN_SAMPLE_RATE]] flag value. After that we validate the
+ * results using the following logic:
* - If the fixed-point Analyzer fails and the single-pass one succeeds, we throw an
* appropriate exception (please check the
* [[QueryCompilationErrors.fixedPointFailedSinglePassSucceeded]] method)
@@ -55,23 +58,29 @@ class HybridAnalyzer(
resolverGuard: ResolverGuard,
resolver: Resolver,
extendedResolutionChecks: Seq[LogicalPlan => Unit] = Seq.empty,
- checkSupportedSinglePassFeatures: Boolean = true)
+ exposeExplicitlyUnsupportedResolverFeature: Boolean = false)
extends SQLConfHelper {
private var singlePassResolutionDuration: Option[Long] = None
private var fixedPointResolutionDuration: Option[Long] = None
private val resolverRunner: ResolverRunner =
new ResolverRunner(resolver, extendedResolutionChecks)
+ private val sampleRateGenerator = new Random()
def apply(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
+ val passedResolvedGuard = resolverGuard.apply(plan)
val dualRun =
conf.getConf(SQLConf.ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER) &&
- checkResolverGuard(plan)
+ passedResolvedGuard && checkDualRunSampleRate()
withTrackedAnalyzerBridgeState(dualRun) {
if (dualRun) {
resolveInDualRun(plan, tracker)
} else if (conf.getConf(SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED)) {
- resolveInSinglePass(plan)
+ resolveInSinglePass(plan, tracker)
+ } else if (passedResolvedGuard && conf.getConf(
+ SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY
+ )) {
+ resolveInSinglePassTentatively(plan, tracker)
} else {
resolveInFixedPoint(plan, tracker)
}
@@ -136,7 +145,7 @@ class HybridAnalyzer(
var singlePassException: Option[Throwable] = None
val singlePassResult = try {
val (resolutionDuration, result) = recordDuration {
- Some(resolveInSinglePass(plan))
+ Some(resolveInSinglePass(plan, tracker))
}
singlePassResolutionDuration = Some(resolutionDuration)
result
@@ -160,7 +169,7 @@ class HybridAnalyzer(
case None =>
singlePassException match {
case Some(singlePassEx: ExplicitlyUnsupportedResolverFeature)
- if checkSupportedSinglePassFeatures =>
+ if !exposeExplicitlyUnsupportedResolverFeature =>
fixedPointResult.get
case Some(singlePassEx) =>
throw singlePassEx
@@ -175,12 +184,31 @@ class HybridAnalyzer(
}
}
+ /**
+ * Run the single-pass Analyzer, but fall back to the fixed-point if
+ * [[ExplicitlyUnsupportedResolverFeature]] is thrown.
+ */
+ private def resolveInSinglePassTentatively(
+ plan: LogicalPlan,
+ tracker: QueryPlanningTracker): LogicalPlan = {
+ try {
+ resolveInSinglePass(plan, tracker)
+ } catch {
+ case _: ExplicitlyUnsupportedResolverFeature =>
+ resolveInFixedPoint(plan, tracker)
+ }
+ }
+
/**
* This method is used to run the single-pass Analyzer which will return the resolved plan
* or throw an exception if the resolution fails. Both cases are handled in the caller method.
* */
- private def resolveInSinglePass(plan: LogicalPlan): LogicalPlan =
- resolverRunner.resolve(plan, AnalysisContext.get.getSinglePassResolverBridgeState)
+ private def resolveInSinglePass(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan =
+ resolverRunner.resolve(
+ plan = plan,
+ analyzerBridgeState = AnalysisContext.get.getSinglePassResolverBridgeState,
+ tracker = tracker
+ )
/**
* This method is used to run the legacy Analyzer which will return the resolved plan
@@ -194,6 +222,10 @@ class HybridAnalyzer(
resolvedPlan
}
+ private def checkDualRunSampleRate(): Boolean = {
+ sampleRateGenerator.nextDouble() < conf.getConf(SQLConf.ANALYZER_DUAL_RUN_SAMPLE_RATE)
+ }
+
private def validateLogicalPlans(fixedPointResult: LogicalPlan, singlePassResult: LogicalPlan) = {
if (fixedPointResult.schema != singlePassResult.schema) {
throw QueryCompilationErrors.hybridAnalyzerOutputSchemaComparisonMismatch(
@@ -213,9 +245,6 @@ class HybridAnalyzer(
NormalizePlan(plan)
}
- private def checkResolverGuard(plan: LogicalPlan): Boolean =
- !checkSupportedSinglePassFeatures || resolverGuard.apply(plan)
-
private def recordDuration[T](thunk: => T): (Long, T) = {
val start = System.nanoTime()
val res = thunk
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/IdentifierAndCteSubstituor.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/IdentifierAndCteSubstituor.scala
new file mode 100644
index 000000000000..b527162f5ba2
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/IdentifierAndCteSubstituor.scala
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.plans.logical.{
+ CTERelationDef,
+ LogicalPlan,
+ SubqueryAlias,
+ UnresolvedWith
+}
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
+import org.apache.spark.sql.catalyst.trees.TreePattern.{CTE, PLAN_EXPRESSION, UNRESOLVED_RELATION}
+
+/**
+ * The [[IdentifierAndCteSubstitutor]] is responsible for substituting the IDENTIFIERs (not yet
+ * implemented) and CTE references in the unresolved logical plan before the actual resolution
+ * starts (specifically before metadata resolution). This is important for SQL features like WITH
+ * (that could confuse [[MetadataResolver]] with extra [[UnresolvedRelation]]s)
+ * or IDENTIFIER (that "hides" the actual [[UnresolvedRelation]]s).
+ *
+ * We only recurse into the plan if [[IdentifierAndCteSubstitutor.NODES_OF_INTEREST]] are present.
+ * This is done so that [[IdentifierAndCteSubstitutor]] is fast and not invasive.
+ */
+class IdentifierAndCteSubstitutor {
+ private var cteRegistry = new CteRegistry
+
+ /**
+ * This is the main entry point to the substitution process. It takes `unresolvedPlan` and
+ * substitutes CTEs and IDENTIFIERs (not yet implemented) that would otherwise confuse the
+ * metadata resolution process.
+ *
+ * [[CteRegistry]] has to be reset for each new invocation, because CTEs in views are analyzed
+ * in isolation.
+ */
+ def substitutePlan(unresolvedPlan: LogicalPlan): LogicalPlan = {
+ cteRegistry = new CteRegistry
+
+ substitute(unresolvedPlan)
+ }
+
+ private def substitute(unresolvedPlan: LogicalPlan): LogicalPlan = {
+ unresolvedPlan match {
+ case unresolvedWith: UnresolvedWith =>
+ handleWith(unresolvedWith)
+ case unresolvedRelation: UnresolvedRelation =>
+ handleUnresolvedRelation(unresolvedRelation)
+ case _ =>
+ handleOperator(unresolvedPlan)
+ }
+ }
+
+ /**
+ * Handle [[UnresolvedWith]] operator. WITH clause produces unresolved relations in the plan,
+ * that are not actual tables or views, but are just potential [[CTERelationRef]]s. To correctly
+ * detect those CTE references we use the [[CteRegistry]] framework. The actual CTE resolution
+ * is left to the main algebraic pass in the [[Resolver]] - here we replace the
+ * [[UnresolvedRelation]]s with [[UnresolvedCteRelationRef]] to avoid isuing useless catalog
+ * RPCs later on in the [[MetadataResolver]].
+ *
+ * We need to use the [[CteRegistry]] framework, because whether the relation is a CTE reference
+ * or not depends on its position in the logical plan tree:
+ * {{{
+ * CREATE TABLE rel2 (col1 INT);
+ *
+ * WITH rel1 AS (
+ * WITH rel2 AS (
+ * SELECT 1
+ * )
+ * SELECT * FROM rel2 -- `rel2` is a CTE reference
+ * )
+ * SELECT * FROM rel2 -- `rel2` is a table reference
+ * }}}
+ */
+ private def handleWith(unresolvedWith: UnresolvedWith): LogicalPlan = {
+ val cteRelationsAfterSubstitution = unresolvedWith.cteRelations.map { cteRelation =>
+ val (cteName, ctePlan) = cteRelation
+
+ val ctePlanAfter = cteRegistry.withNewScope() {
+ substitute(ctePlan).asInstanceOf[SubqueryAlias]
+ }
+
+ cteRegistry.currentScope.registerCte(cteName, CTERelationDef(ctePlanAfter))
+
+ (cteName, ctePlanAfter)
+ }
+
+ val childAfterSubstitution = cteRegistry.withNewScope() {
+ substitute(unresolvedWith.child)
+ }
+
+ val result = withOrigin(unresolvedWith.origin) {
+ unresolvedWith.copy(
+ child = childAfterSubstitution,
+ cteRelations = cteRelationsAfterSubstitution
+ )
+ }
+ result.copyTagsFrom(unresolvedWith)
+ result
+ }
+
+ /**
+ * Handle [[UnresolvedRelation]] operator, which could be a CTE reference. If that's the case, we
+ * replace it with [[UnresolvedCteRelationRef]].
+ */
+ private def handleUnresolvedRelation(unresolvedRelation: UnresolvedRelation): LogicalPlan = {
+ if (unresolvedRelation.multipartIdentifier.size == 1) {
+ cteRegistry.resolveCteName(unresolvedRelation.multipartIdentifier.head) match {
+ case Some(_) =>
+ val result = withOrigin(unresolvedRelation.origin) {
+ UnresolvedCteRelationRef(unresolvedRelation.multipartIdentifier.head)
+ }
+ result.copyTagsFrom(unresolvedRelation)
+ result
+ case None =>
+ unresolvedRelation
+ }
+ } else {
+ unresolvedRelation
+ }
+ }
+
+ /**
+ * Handle `unresolvedOperator` generically. We use the [[CteRegistry]] framework to recurse into
+ * its children and subquery expressions.
+ */
+ private def handleOperator(unresolvedOperator: LogicalPlan): LogicalPlan = {
+ val operatorAfterSubstitution = unresolvedOperator match {
+ case operator
+ if !operator.containsAnyPattern(IdentifierAndCteSubstitutor.NODES_OF_INTEREST: _*) =>
+ operator
+
+ case operator if operator.children.size > 1 =>
+ val newChildren = operator.children.map { child =>
+ cteRegistry.withNewScopeUnderMultiChildOperator(operator, child) {
+ substitute(child)
+ }
+ }
+ operator.withNewChildren(newChildren)
+
+ case operator if operator.children.size == 1 =>
+ val newChildren = Seq(substitute(operator.children.head))
+ operator.withNewChildren(newChildren)
+
+ case operator =>
+ operator
+ }
+
+ operatorAfterSubstitution.transformExpressionsWithPruning(
+ _.containsPattern(PLAN_EXPRESSION)
+ ) {
+ case subqueryExpression: SubqueryExpression =>
+ val newPlan = cteRegistry.withNewScope(isRoot = true) {
+ substitute(subqueryExpression.plan)
+ }
+
+ val result = withOrigin(subqueryExpression.origin) {
+ subqueryExpression.withNewPlan(newPlan)
+ }
+ result.copyTagsFrom(subqueryExpression)
+ result
+ }
+ }
+}
+
+object IdentifierAndCteSubstitutor {
+ val NODES_OF_INTEREST = Seq(CTE, UNRESOLVED_RELATION)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/JoinResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/JoinResolver.scala
new file mode 100644
index 000000000000..b7a5a733159f
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/JoinResolver.scala
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.analysis.NaturalAndUsingJoinResolution
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.plans.{JoinType, NaturalJoin, UsingJoin}
+import org.apache.spark.sql.catalyst.plans.logical.{Join, JoinHint, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.util._
+
+/**
+ * Resolves [[Join]] operator by resolving its left and right children and its join condition. If
+ * the unresolved join is [[NaturalJoin]] or [[UsingJoin]], the resulting operator will be
+ * [[Project]], otherwise it will be [[Join]].
+ */
+class JoinResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
+ extends TreeNodeResolver[Join, LogicalPlan] {
+ private val scopes = resolver.getNameScopes
+ private val expressionIdAssigner = expressionResolver.getExpressionIdAssigner
+ private val cteRegistry = resolver.getCteRegistry
+
+ /**
+ * Resolves [[Join]] operator:
+ * - Retrieve old output and child outputs if the operator is already resolved. This is relevant
+ * for partially resolved subtrees from DataFrame programs. Do not regenerate ExprIds if there
+ * are no conflicting ids.
+ * - Resolve each child in the context of a) New [[NameScope]] b) New [[ExpressionIdAssigner]]
+ * mapping. Collect children name scopes to use in [[Join]] output computation.
+ * - Based on the type of [[Join]] (natural, using or other) perform additional transformations
+ * and resolve join condition.
+ * - Return the resulting [[Project]] or [[Join]] with new children optionally wrapped in
+ * [[WithCTE]]. See [[CteScope]] scaladoc for more info.
+ */
+ override def resolve(unresolvedJoin: Join): LogicalPlan = {
+ val (resolvedLeftOperator: LogicalPlan, leftNameScope: NameScope) = resolveJoinChild(
+ unresolvedJoin = unresolvedJoin,
+ child = unresolvedJoin.left
+ )
+
+ val (resolvedRightOperator: LogicalPlan, rightNameScope: NameScope) = resolveJoinChild(
+ unresolvedJoin = unresolvedJoin,
+ child = unresolvedJoin.right
+ )
+
+ ExpressionIdAssigner.assertOutputsHaveNoConflictingExpressionIds(
+ Seq(leftNameScope.output, rightNameScope.output)
+ )
+
+ expressionIdAssigner.createMappingFromChildMappings()
+
+ val partiallyResolvedJoin = unresolvedJoin.copy(
+ left = resolvedLeftOperator,
+ right = resolvedRightOperator
+ )
+
+ handleDifferentTypesOfJoin(
+ unresolvedJoin = unresolvedJoin,
+ partiallyResolvedJoin = partiallyResolvedJoin,
+ leftNameScope = leftNameScope,
+ rightNameScope = rightNameScope
+ )
+ }
+
+ private def resolveJoinChild(
+ unresolvedJoin: Join,
+ child: LogicalPlan): (LogicalPlan, NameScope) = {
+ scopes.withNewScope() {
+ expressionIdAssigner.withNewMapping(collectChildMapping = true) {
+ cteRegistry.withNewScopeUnderMultiChildOperator(
+ unresolvedOperator = unresolvedJoin,
+ unresolvedChild = child
+ ) {
+ val resolvedLeftOperator = resolver.resolve(child)
+ (resolvedLeftOperator, scopes.current)
+ }
+ }
+ }
+ }
+
+ /**
+ * If the type of join is [[NaturalJoin]] or [[UsingJoin]], perform additional transformations in
+ * [[commonNaturalJoinProcessing]]. Otherwise, overwrite current name scope output with the
+ * result of [[Join.computeOutput]].
+ */
+ private def handleDifferentTypesOfJoin(
+ unresolvedJoin: Join,
+ partiallyResolvedJoin: Join,
+ leftNameScope: NameScope,
+ rightNameScope: NameScope): LogicalPlan = partiallyResolvedJoin match {
+ case Join(left, right, UsingJoin(joinType, usingCols), _, hint) =>
+ commonNaturalJoinProcessing(
+ unresolvedJoin = unresolvedJoin,
+ left = left,
+ leftNameScope = leftNameScope,
+ right = right,
+ rightNameScope = rightNameScope,
+ joinType = joinType,
+ joinNames = usingCols,
+ condition = None,
+ hint = hint
+ )
+ case Join(left, right, NaturalJoin(joinType), condition, hint) =>
+ val joinNames = getJoinNamesForNaturalJoin(leftNameScope, rightNameScope)
+ commonNaturalJoinProcessing(
+ unresolvedJoin = unresolvedJoin,
+ left = left,
+ leftNameScope = leftNameScope,
+ right = right,
+ rightNameScope = rightNameScope,
+ joinType = joinType,
+ joinNames = joinNames,
+ condition = condition,
+ hint = hint
+ )
+ case partiallyResolvedJoin: Join =>
+ handleRegularJoin(
+ unresolvedJoin = unresolvedJoin,
+ partiallyResolvedJoin = partiallyResolvedJoin,
+ leftNameScope = leftNameScope,
+ rightNameScope = rightNameScope
+ )
+ }
+
+ /**
+ * This method handles [[NaturalJoin]] and [[UsingJoin]] by computing their correct outputs and
+ * placing a [[Project]] node on top of them.
+ * The order of necessary operations is as follows:
+ * - Compute output list, hidden list and new condition with join pairs, if there are any.
+ * [[NaturalAndUsingJoinResolution.computeJoinOutputsAndNewCondition]] introduces new
+ * aliased expressions (e.g. [[Coalesce]] for keys), so we need to run the output list through
+ * [[ExpressionIdAssigner.mapExpression]].
+ * - Resolve the new condition.
+ * - Compute new `hiddenOutput` by appending elements from computed hidden list that are not
+ * already in current `hiddenOutput`. Hidden list must be qualified access only.
+ * - Overwrite current name scope with output list and newly computed hidden output.
+ * - Finally, put a [[Project]] node on top of the original [[Join]] by:
+ * - New project list becomes output list.
+ * - If [[Join]] was not a top level operator, append current hidden output to the project
+ * list.
+ * - Add new hidden output as a tag to project node in order to stay compatible with
+ * fixed-point. This should never be used in single-pass, but it can happen that fixed-point
+ * uses the single-pass result, therefore we need to set the tag.
+ */
+ private def commonNaturalJoinProcessing(
+ unresolvedJoin: Join,
+ left: LogicalPlan,
+ leftNameScope: NameScope,
+ right: LogicalPlan,
+ rightNameScope: NameScope,
+ joinType: JoinType,
+ joinNames: Seq[String],
+ condition: Option[Expression],
+ hint: JoinHint): LogicalPlan = {
+ val (outputList, hiddenList, newCondition) =
+ NaturalAndUsingJoinResolution.computeJoinOutputsAndNewCondition(
+ left = left,
+ leftOutput = leftNameScope.output,
+ right = right,
+ rightOutput = rightNameScope.output,
+ joinType = joinType,
+ joinNames = joinNames,
+ condition = condition,
+ resolveName = conf.resolver
+ )
+
+ val newOutputList = outputList.map(expressionIdAssigner.mapExpression)
+
+ val resolvedCondition =
+ resolveJoinCondition(unresolvedJoin, newCondition, leftNameScope, rightNameScope)
+
+ val hiddenListWithQualifiedAccess = hiddenList.map(_.markAsQualifiedAccessOnly())
+
+ val newHiddenOutput = hiddenListWithQualifiedAccess ++ scopes.current.hiddenOutput
+
+ scopes.overwriteCurrent(
+ output = Some(newOutputList.map(_.toAttribute)),
+ hiddenOutput = Some(newHiddenOutput)
+ )
+
+ val newProjectList =
+ if (unresolvedJoin.getTagValue(Resolver.TOP_LEVEL_OPERATOR).isEmpty) {
+ newOutputList ++ scopes.current.hiddenOutput
+ .filter(attribute => attribute.qualifiedAccessOnly)
+ } else {
+ newOutputList
+ }
+
+ val project = Project(newProjectList, Join(left, right, joinType, resolvedCondition, hint))
+
+ project.setTagValue(Project.hiddenOutputTag, newHiddenOutput)
+
+ project
+ }
+
+ /**
+ * Resolve a join that is not [[NaturalJoin]] or [[UsingJoin]]. In order to resolve the join we
+ * do the following:
+ * - Resolve join condition.
+ * - Overwrite [[NameScope.output]] with join's output.
+ * - Wrap the [[Join]] in [[WithCTE]] if necessary.
+ */
+ private def handleRegularJoin(
+ unresolvedJoin: Join,
+ partiallyResolvedJoin: Join,
+ leftNameScope: NameScope,
+ rightNameScope: NameScope) = {
+ val resolvedCondition = resolveJoinCondition(
+ unresolvedJoin = unresolvedJoin,
+ unresolvedCondition = partiallyResolvedJoin.condition,
+ leftNameScope = leftNameScope,
+ rightNameScope = rightNameScope
+ )
+
+ scopes.overwriteCurrent(
+ output = Some(
+ Join.computeOutput(
+ partiallyResolvedJoin.joinType,
+ leftNameScope.output,
+ rightNameScope.output
+ )
+ ),
+ hiddenOutput = Some(leftNameScope.hiddenOutput ++ rightNameScope.hiddenOutput)
+ )
+
+ val resolvedJoin = partiallyResolvedJoin.copy(condition = resolvedCondition)
+
+ cteRegistry.currentScope.tryPutWithCTE(
+ unresolvedOperator = unresolvedJoin,
+ resolvedOperator = resolvedJoin
+ )
+ }
+
+ /**
+ * Computes the intersection of two child name scopes, by name.
+ */
+ private def getJoinNamesForNaturalJoin(
+ leftNameScope: NameScope,
+ rightNameScope: NameScope): Seq[String] = {
+ leftNameScope.output
+ .flatMap(attribute => rightNameScope.findAttributesByName(attribute.name))
+ .map(_.name)
+ }
+
+ /**
+ * Resolves join condition by __all__ attributes from child scopes. We need to overwrite current
+ * scope first to prepare for [[resolveExpressionTreeInOperator]]. [[Join]] will actually produce
+ * different output than the one we are setting here, so additional overwrite with correct values
+ * will be needed. Two overwrites are necessary because condition is resolved from original
+ * children outputs, whereas output of [[Join]] will either not contain all attributes or their
+ * nullabilities will be different.
+ */
+ private def resolveJoinCondition(
+ unresolvedJoin: Join,
+ unresolvedCondition: Option[Expression],
+ leftNameScope: NameScope,
+ rightNameScope: NameScope) = {
+ scopes.overwriteCurrent(
+ output = Some(leftNameScope.output ++ rightNameScope.output),
+ hiddenOutput = Some(leftNameScope.hiddenOutput ++ rightNameScope.hiddenOutput)
+ )
+
+ unresolvedCondition.map { condition =>
+ expressionResolver.resolveExpressionTreeInOperator(
+ condition,
+ unresolvedJoin
+ )
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/LimitLikeExpressionValidator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/LimitLikeExpressionValidator.scala
new file mode 100644
index 000000000000..b35516fce314
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/LimitLikeExpressionValidator.scala
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.errors.QueryErrorsBase
+import org.apache.spark.sql.types.IntegerType
+
+/**
+ * The [[LimitLikeExpressionValidator]] validates [[LocalLimit]], [[GlobalLimit]], [[Offset]] or
+ * [[Tail]] integer expressions.
+ */
+class LimitLikeExpressionValidator extends QueryErrorsBase {
+ def validateLimitLikeExpr(
+ limitLikeExpression: Expression,
+ partiallyResolvedLimitLike: LogicalPlan
+ ): Expression = {
+ val evaluatedExpression =
+ evaluateLimitLikeExpression(limitLikeExpression, partiallyResolvedLimitLike)
+
+ checkValidLimitWithOffset(evaluatedExpression, partiallyResolvedLimitLike)
+
+ limitLikeExpression
+ }
+
+ private def checkValidLimitWithOffset(evaluatedExpression: Int, plan: LogicalPlan): Unit = {
+ plan match {
+ case LocalLimit(_, Offset(offsetExpr, _)) =>
+ val offset = offsetExpr.eval().asInstanceOf[Int]
+ if (Int.MaxValue - evaluatedExpression < offset) {
+ throw throwInvalidLimitWithOffsetSumExceedsMaxInt(evaluatedExpression, offset, plan)
+ }
+ case _ =>
+ }
+ }
+
+ /**
+ * Evaluate a resolved limit expression of [[GlobalLimit]], [[LocalLimit]], [[Offset]] or
+ * [[Tail]], while performing required checks:
+ * - The expression has to be foldable
+ * - The result data type has to be [[IntegerType]]
+ * - The evaluated expression has to be non-null
+ * - The evaluated expression has to be positive
+ *
+ * The `foldable` check is implemented in some expressions
+ * as a recursive expression tree traversal.
+ * It is not an ideal approach for the single-pass [[ExpressionResolver]],
+ * but __is__ practical, since:
+ * - We have to call `eval` here anyway, and it's recursive
+ * - In practice `LIMIT`, `OFFSET` and `TAIL` expression trees are very small
+ *
+ * The return type of evaluation is Int, as we perform check that the expression has
+ * IntegerType.
+ */
+ private def evaluateLimitLikeExpression(expression: Expression, plan: LogicalPlan): Int = {
+ val operatorName = plan match {
+ case _: Offset => "offset"
+ case _: Tail => "tail"
+ case _: LocalLimit | _: GlobalLimit => "limit"
+ case other =>
+ throw SparkException.internalError(
+ s"Unexpected limit like operator type: ${other.getClass.getName}"
+ )
+ }
+ if (!expression.foldable) {
+ throwInvalidLimitLikeExpressionIsUnfoldable(operatorName, expression)
+ }
+ if (expression.dataType != IntegerType) {
+ throwInvalidLimitLikeExpressionDataType(operatorName, expression)
+ }
+ expression.eval() match {
+ case null =>
+ throwInvalidLimitLikeExpressionIsNull(operatorName, expression)
+ case value: Int if value < 0 =>
+ throwInvalidLimitLikeExpressionIsNegative(operatorName, expression, value)
+ case result =>
+ result.asInstanceOf[Int]
+ }
+ }
+
+ private def throwInvalidLimitWithOffsetSumExceedsMaxInt(
+ limit: Int,
+ offset: Int,
+ plan: LogicalPlan): Nothing =
+ throw new AnalysisException(
+ errorClass = "SUM_OF_LIMIT_AND_OFFSET_EXCEEDS_MAX_INT",
+ messageParameters = Map("limit" -> limit.toString, "offset" -> offset.toString),
+ origin = plan.origin
+ )
+
+ private def throwInvalidLimitLikeExpressionIsUnfoldable(
+ name: String,
+ expression: Expression): Nothing =
+ throw new AnalysisException(
+ errorClass = "INVALID_LIMIT_LIKE_EXPRESSION.IS_UNFOLDABLE",
+ messageParameters = Map("name" -> name, "expr" -> toSQLExpr(expression)),
+ origin = expression.origin
+ )
+
+ private def throwInvalidLimitLikeExpressionDataType(
+ name: String,
+ expression: Expression): Nothing =
+ throw new AnalysisException(
+ errorClass = "INVALID_LIMIT_LIKE_EXPRESSION.DATA_TYPE",
+ messageParameters = Map(
+ "name" -> name,
+ "expr" -> toSQLExpr(expression),
+ "dataType" -> toSQLType(expression.dataType)
+ ),
+ origin = expression.origin
+ )
+
+ private def throwInvalidLimitLikeExpressionIsNull(name: String, expression: Expression): Nothing =
+ throw new AnalysisException(
+ errorClass = "INVALID_LIMIT_LIKE_EXPRESSION.IS_NULL",
+ messageParameters = Map(
+ "name" -> name,
+ "expr" -> toSQLExpr(expression)
+ ),
+ origin = expression.origin
+ )
+
+ private def throwInvalidLimitLikeExpressionIsNegative(
+ name: String,
+ expression: Expression,
+ value: Int): Nothing =
+ throw new AnalysisException(
+ errorClass = "INVALID_LIMIT_LIKE_EXPRESSION.IS_NEGATIVE",
+ messageParameters =
+ Map("name" -> name, "expr" -> toSQLExpr(expression), "v" -> toSQLValue(value, IntegerType)),
+ origin = expression.origin
+ )
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/MetadataResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/MetadataResolver.scala
index 0d57eae0be7a..b54824c34819 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/MetadataResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/MetadataResolver.scala
@@ -20,8 +20,13 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import java.util.ArrayDeque
import org.apache.spark.sql.catalyst.analysis.{RelationResolution, UnresolvedRelation}
-import org.apache.spark.sql.catalyst.expressions.{Expression, PlanExpression}
-import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan}
+import org.apache.spark.sql.catalyst.expressions.{Expression, SubqueryExpression}
+import org.apache.spark.sql.catalyst.plans.logical.{
+ AnalysisHelper,
+ LogicalPlan,
+ SubqueryAlias,
+ UnresolvedWith
+}
import org.apache.spark.sql.connector.catalog.CatalogManager
/**
@@ -54,9 +59,8 @@ class MetadataResolver(
* calls for the [[UnresolvedRelation]]s present in that tree. During the `unresolvedPlan`
* traversal we fill [[relationsWithResolvedMetadata]] with resolved metadata by relation id.
* This map will be used to resolve the plan in single-pass by the [[Resolver]] using
- * [[getRelationWithResolvedMetadata]]. If the generic metadata resolution using
- * [[RelationResolution]] wasn't successful, we resort to using [[extensions]].
- * Otherwise, we fail with an exception.
+ * [[getRelationWithResolvedMetadata]]. We always try to complete the default resolution using
+ * extensions.
*/
override def resolve(unresolvedPlan: LogicalPlan): Unit = {
traverseLogicalPlanTree(unresolvedPlan) {
@@ -64,20 +68,26 @@ class MetadataResolver(
val relationId = relationIdFromUnresolvedRelation(unresolvedRelation)
if (!relationsWithResolvedMetadata.containsKey(relationId)) {
- val relationWithResolvedMetadata = resolveRelation(unresolvedRelation).orElse {
- // In case the generic metadata resolution returned `None`, we try to check if any
- // of the [[extensions]] matches this `unresolvedRelation`, and resolve it using
- // that extension.
- tryDelegateResolutionToExtension(unresolvedRelation, prohibitedResolver)
+ val relationAfterDefaultResolution =
+ resolveRelation(unresolvedRelation).getOrElse(unresolvedRelation)
+
+ val relationAfterExtensionResolution = relationAfterDefaultResolution match {
+ case subqueryAlias: SubqueryAlias =>
+ tryDelegateResolutionToExtension(subqueryAlias.child, prohibitedResolver).map {
+ relation =>
+ subqueryAlias.copy(child = relation)
+ }
+ case _ =>
+ tryDelegateResolutionToExtension(relationAfterDefaultResolution, prohibitedResolver)
}
- relationWithResolvedMetadata match {
- case Some(relationWithResolvedMetadata) =>
+ relationAfterExtensionResolution.getOrElse(relationAfterDefaultResolution) match {
+ case _: UnresolvedRelation =>
+ case relationWithResolvedMetadata =>
relationsWithResolvedMetadata.put(
relationId,
relationWithResolvedMetadata
)
- case None =>
}
}
case _ =>
@@ -109,18 +119,21 @@ class MetadataResolver(
case Left(logicalPlan) =>
visitor(logicalPlan)
- for (child <- logicalPlan.children) {
- stack.push(Left(child))
- }
- for (innerChild <- logicalPlan.innerChildren) {
- innerChild match {
- case plan: LogicalPlan =>
- stack.push(Left(plan))
- case _ =>
- }
- }
- for (expression <- logicalPlan.expressions) {
- stack.push(Right(expression))
+ logicalPlan match {
+ case unresolvedWith: UnresolvedWith =>
+ for (cteRelation <- unresolvedWith.cteRelations) {
+ stack.push(Left(cteRelation._2))
+ }
+
+ stack.push(Left(unresolvedWith.child))
+ case _ =>
+ for (child <- logicalPlan.children) {
+ stack.push(Left(child))
+ }
+
+ for (expression <- logicalPlan.expressions) {
+ stack.push(Right(expression))
+ }
}
case Right(expression) =>
for (child <- expression.children) {
@@ -128,12 +141,8 @@ class MetadataResolver(
}
expression match {
- case planExpression: PlanExpression[_] =>
- planExpression.plan match {
- case plan: LogicalPlan =>
- stack.push(Left(plan))
- case _ =>
- }
+ case subqueryExpression: SubqueryExpression =>
+ stack.push(Left(subqueryExpression.plan))
case _ =>
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala
index 5e6aa6583040..86fc43fd5224 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameScope.scala
@@ -17,14 +17,30 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import java.util.{ArrayDeque, HashSet}
+import java.util.{ArrayDeque, HashMap, LinkedHashMap}
import scala.collection.mutable
+import scala.jdk.CollectionConverters._
import org.apache.spark.sql.catalyst.SQLConfHelper
-import org.apache.spark.sql.catalyst.analysis.{Resolver => NameComparator, UnresolvedStar}
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSeq, ExprId, NamedExpression}
+import org.apache.spark.sql.catalyst.analysis.{
+ LiteralFunctionResolution,
+ Resolver => NameComparator,
+ UnresolvedStar
+}
+import org.apache.spark.sql.catalyst.expressions.{
+ Alias,
+ Attribute,
+ AttributeSeq,
+ Expression,
+ ExprId,
+ ExtractValue,
+ NamedExpression,
+ OuterReference
+}
+import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.Metadata
/**
* The [[NameScope]] is used to control the resolution of names (table, column, alias identifiers).
@@ -32,8 +48,8 @@ import org.apache.spark.sql.internal.SQLConf
* program operators.
*
* The [[NameScope]] output is immutable. If it's necessary to update the output,
- * [[NameScopeStack]] methods are used ([[overwriteTop]] or [[withNewScope]]). The [[NameScope]]
- * is always used through the [[NameScopeStack]].
+ * [[NameScopeStack]] methods are used ([[overwriteCurrent]] or [[withNewScope]]). The
+ * [[NameScope]] is always used through the [[NameScopeStack]].
*
* The resolution of identifiers is case-insensitive.
*
@@ -41,16 +57,35 @@ import org.apache.spark.sql.internal.SQLConf
*
* 1. Resolution of local references:
* - column reference
+ * - parameterless function reference
* - struct field or map key reference
* 2. Resolution of lateral column aliases (if enabled).
+ * 3. In the context of [[Aggregate]]: resolution of names in groping expressions list referencing
+ * aliases in aggregate expressions.
*
- * For example, in a query like:
+ * Following examples showcase the priority of name resolution:
*
* {{{ SELECT 1 AS col1, col1 FROM VALUES (2) }}}
*
* Because column resolution has a higher priority than LCA resolution, the result will be [1, 2]
* and not [1, 1].
*
+ * {{{
+ * CREATE TABLE t AS SELECT col1 as current_date FROM VALUES (2);
+ *
+ * SELECT
+ * 1 AS current_timestamp,
+ * current_timestamp,
+ * current_date
+ * FROM
+ * foo;
+ * }}}
+ *
+ * Result of the previous SELECT will be: [1, 2025-02-13T07:55:26.206+00:00, 2]. As can be seen,
+ * because of resolution precedence, current_date is resolved as a table column, but
+ * current_timestamp is resolved as a function without parenthesis instead of a lateral column
+ * reference.
+ *
* Approximate tree of [[NameScope]] manipulations is shown in the following example:
*
* {{{
@@ -71,33 +106,50 @@ import org.apache.spark.sql.internal.SQLConf
* unionAttributes = withNewScope {
* lhsOutput = withNewScope {
* expandedStar = withNewScope {
- * scope.overwriteTop(localRelation.output)
+ * scopes.overwriteCurrent(localRelation.output)
* scope.expandStar(star)
* }
- * scope.overwriteTop(expandedStar)
+ * scopes.overwriteCurrent(expandedStar)
* scope.output
* }
* rhsOutput = withNewScope {
* subqueryAttributes = withNewScope {
- * scope.overwriteTop(t1.output)
- * scope.overwriteTop(prependQualifier(scope.output, "t2"))
+ * scopes.overwriteCurrent(t1.output)
+ * scopes.overwriteCurrent(prependQualifier(scope.output, "t2"))
* [scope.matchMultiPartName("t2", "col1"), scope.matchMultiPartName("t2", "col2")]
* }
- * scope.overwriteTop(subqueryAttributes)
+ * scopes.overwriteCurrent(subqueryAttributes)
* scope.output
* }
- * scope.overwriteTop(coerce(lhsOutput, rhsOutput))
+ * scopes.overwriteCurrent(coerce(lhsOutput, rhsOutput))
* [scope.matchMultiPartName("col1"), alias(scope.matchMultiPartName("col2"), "alias1")]
* }
- * scope.overwriteTop(unionAttributes)
+ * scopes.overwriteCurrent(unionAttributes)
* }}}
*
* @param output These are the attributes visible for lookups in the current scope.
* These may be:
* - Transformed outputs of lower scopes (e.g. type-coerced outputs of [[Union]]'s children).
* - Output of a current operator that is being resolved (leaf nodes like [[Relations]]).
+ * @param hiddenOutput Attributes that are not directly visible in the scope, but available for
+ * lookup in case the resolved attribute is not found in `output`.
+ * @param isSubqueryRoot Indicates that the current scope is a root of a subquery. This is used by
+ * [[NameScopeStack.resolveMultipartName]] to detect the nearest outer scope.
*/
-class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
+class NameScope(
+ val output: Seq[Attribute] = Seq.empty,
+ val hiddenOutput: Seq[Attribute] = Seq.empty,
+ val isSubqueryRoot: Boolean = false)
+ extends SQLConfHelper {
+
+ /**
+ * This is an internal class used to store resolved multipart name, with correct precedence as
+ * specified by [[NameScope]] class doc.
+ */
+ private case class ResolvedMultipartName(
+ candidates: Seq[Expression],
+ referencedAttribute: Option[Attribute],
+ aliasMetadata: Option[Metadata] = None)
/**
* [[nameComparator]] is a function that is used to compare two identifiers. Its implementation
@@ -107,10 +159,28 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
private val nameComparator: NameComparator = conf.resolver
/**
- * [[attributesForResolution]] is an [[AttributeSeq]] that is used for resolution of
- * multipart attribute names. It's created from the `attributes` when [[NameScope]] is updated.
+ * [[attributesForResolution]] is an [[AttributeSeq]] that is used for resolution of multipart
+ * attribute names, by output. It's created from the `output` when
+ * [[NameScope]] is updated.
+ */
+ private val attributesForResolution: AttributeSeq =
+ AttributeSeq.fromNormalOutput(output)
+
+ /**
+ * [[hiddenAttributesForResolution]] is an [[AttributeSeq]] that is used for resolution of
+ * multipart attribute names, by hidden output. It's created from the `hiddenOutput` when
+ * [[NameScope]] is updated.
+ */
+ private lazy val hiddenAttributesForResolution: AttributeSeq =
+ AttributeSeq.fromNormalOutput(hiddenOutput)
+
+ /**
+ * [[metadataAttributesForResolution]] is an [[AttributeSeq]] that is used for resolution of
+ * multipart attribute names, by qualified access only columns from hidden output. It's created
+ * from the `hiddenOutput` when [[NameScope]] is updated.
*/
- private val attributesForResolution: AttributeSeq = AttributeSeq.fromNormalOutput(output)
+ private lazy val metadataAttributesForResolution: AttributeSeq =
+ AttributeSeq.fromNormalOutput(hiddenOutput.filter(_.qualifiedAccessOnly))
/**
* [[attributesByName]] is used to look up attributes by one-part name from the operator's output.
@@ -120,15 +190,73 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
private lazy val attributesByName = createAttributesByName(output)
/**
- * Expression IDs from `output`. See [[hasAttributeWithId]] for more details.
+ * Returns a map of [[ExprId]] to [[Attribute]] from `output`. See [[getAttributeById]] for
+ * more details.
+ */
+ private lazy val attributesById: HashMap[ExprId, Attribute] = createAttributeIds(output)
+
+ /**
+ * Returns a map of [[ExprId]] to [[Attribute]] from `hiddenOutput`.
*/
- private lazy val attributeIds = createAttributeIds(output)
+ private lazy val hiddenAttributesById: HashMap[ExprId, Attribute] =
+ createAttributeIds(hiddenOutput)
- private val isLcaEnabled = conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)
- lazy val lcaRegistry: LateralColumnAliasRegistry = if (isLcaEnabled) {
- new LateralColumnAliasRegistryImpl(output)
- } else {
- new LateralColumnAliasProhibitedRegistry
+ lazy val lcaRegistry: LateralColumnAliasRegistry =
+ if (conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)) {
+ new LateralColumnAliasRegistryImpl(output)
+ } else {
+ new LateralColumnAliasProhibitedRegistry
+ }
+
+ /**
+ * All aliased aggregate expressions from an [[Aggregate]] that is currently being resolved.
+ * Used in [[resolveMultipartName]] to resolve names in grouping expressions list referencing
+ * aggregate expressions.
+ */
+ private lazy val topAggregateExpressionsByAliasName: IdentifierMap[Alias] =
+ new IdentifierMap[Alias]
+
+ /**
+ * Returns new [[NameScope]] which preserves all the immutable [[NameScope]] properties but
+ * overwrites `output` and `hiddenOutput` if provided. Mutable state like `lcaRegistry` is not
+ * preserved.
+ */
+ def overwriteOutput(
+ output: Option[Seq[Attribute]] = None,
+ hiddenOutput: Option[Seq[Attribute]] = None): NameScope = {
+ new NameScope(
+ output = output.getOrElse(this.output),
+ hiddenOutput = hiddenOutput.getOrElse(this.hiddenOutput),
+ isSubqueryRoot = isSubqueryRoot
+ )
+ }
+
+ /**
+ * Given referenced attributes, returns all attributes that are referenced and missing from
+ * current output, but can be found in hidden output.
+ */
+ def resolveMissingAttributesByHiddenOutput(
+ referencedAttributes: HashMap[ExprId, Attribute]): Seq[Attribute] = {
+ val distinctMissingAttributes = new LinkedHashMap[ExprId, Attribute]
+ hiddenOutput.foreach(
+ attribute =>
+ if (referencedAttributes.containsKey(attribute.exprId) &&
+ !attributesById.containsKey(attribute.exprId) &&
+ !distinctMissingAttributes.containsKey(attribute.exprId)) {
+ distinctMissingAttributes.put(attribute.exprId, attribute)
+ }
+ )
+ distinctMissingAttributes.asScala.values.toSeq
+ }
+
+ /**
+ * Add a top level alias to the map so it can be used when resolving a grouping expression.
+ */
+ def addTopAggregateExpression(aliasedAggregateExpression: Alias): Unit = {
+ topAggregateExpressionsByAliasName.put(
+ aliasedAggregateExpression.name,
+ aliasedAggregateExpression
+ )
}
/**
@@ -174,7 +302,7 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
def expandStar(unresolvedStar: UnresolvedStar): Seq[NamedExpression] = {
unresolvedStar.expandStar(
childOperatorOutput = output,
- childOperatorMetadataOutput = Seq.empty,
+ childOperatorMetadataOutput = hiddenOutput,
resolve =
(nameParts, nameComparator) => attributesForResolution.resolve(nameParts, nameComparator),
suggestedAttributes = output,
@@ -258,6 +386,23 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
* SELECT COL1 FROM t;
* }}}
*
+ * Name resolution can be done using the hidden output for certain operators (e.g [[Sort]],
+ * [[Filter]]). This is indicated by `canResolveNameByHiddenOutput` which is passed from
+ * [[ExpressionResolver.resolveAttribute]] based on the parent operator.
+ * Example:
+ *
+ * {{{
+ * -- Project's output = [`col1`]; Project's hidden output = [`col1`, `col2`]
+ * SELECT col1 FROM VALUES(1, 2) ORDER BY col2;
+ * }}}
+ *
+ * The names in [[Aggregate.groupingExpressions]] can reference
+ * [[Aggregate.aggregateExpressions]] aliases. `canReferenceAggregateExpressionAliases` will be
+ * true when we are resolving the grouping expressions.
+ * Example:
+ *
+ * {{ SELECT col1 + col2 AS a FROM VALUES (1, 2) GROUP BY a; }}}
+ *
* We are relying on the [[AttributeSeq]] to perform that work, since it requires complex
* resolution logic involving nested field extraction and multipart name matching.
*
@@ -265,36 +410,57 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
*/
def resolveMultipartName(
multipartName: Seq[String],
- canLaterallyReferenceColumn: Boolean = true): NameTarget = {
- val (candidates, nestedFields) =
- attributesForResolution.getCandidatesForResolution(multipartName, nameComparator)
+ canLaterallyReferenceColumn: Boolean = true,
+ canReferenceAggregateExpressionAliases: Boolean = false,
+ canResolveNameByHiddenOutput: Boolean = false): NameTarget = {
- val (candidatesWithLCAs: Seq[Attribute], referencedAttribute: Option[Attribute]) =
- if (candidates.isEmpty && canLaterallyReferenceColumn) {
- getLcaCandidates(multipartName)
- } else {
- (candidates, None)
- }
-
- val resolvedCandidates = attributesForResolution.resolveCandidates(
- multipartName,
- nameComparator,
- candidatesWithLCAs,
- nestedFields
- )
+ val resolvedMultipartName: ResolvedMultipartName =
+ tryResolveMultipartNameByOutput(
+ multipartName,
+ nameComparator,
+ attributesForResolution,
+ canResolveByProposedAttributes = true
+ ).orElse(
+ tryResolveMultipartNameByOutput(
+ multipartName,
+ nameComparator,
+ metadataAttributesForResolution,
+ canResolveByProposedAttributes = true
+ )
+ )
+ .orElse(
+ tryResolveMultipartNameByOutput(
+ multipartName,
+ nameComparator,
+ hiddenAttributesForResolution,
+ canResolveByProposedAttributes = canResolveNameByHiddenOutput
+ )
+ )
+ .orElse(tryResolveMultipartNameAsLiteralFunction(multipartName))
+ .orElse(
+ tryResolveMultipartNameAsLateralColumnReference(
+ multipartName,
+ canLaterallyReferenceColumn
+ )
+ )
+ .orElse(
+ tryResolveAttributeAsGroupByAlias(multipartName, canReferenceAggregateExpressionAliases)
+ )
+ .getOrElse(ResolvedMultipartName(candidates = Seq.empty, referencedAttribute = None))
- resolvedCandidates match {
+ resolvedMultipartName.candidates match {
case Seq(Alias(child, aliasName)) =>
NameTarget(
candidates = Seq(child),
aliasName = Some(aliasName),
- lateralAttributeReference = referencedAttribute,
+ aliasMetadata = resolvedMultipartName.aliasMetadata,
+ lateralAttributeReference = resolvedMultipartName.referencedAttribute,
output = output
)
case other =>
NameTarget(
candidates = other,
- lateralAttributeReference = referencedAttribute,
+ lateralAttributeReference = resolvedMultipartName.referencedAttribute,
output = output
)
}
@@ -323,11 +489,99 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
}
/**
- * Check if `output` contains attributes with `expressionId`. This is used to disable missing
- * attribute propagation for DataFrames, because we don't support it yet.
+ * Returns attribute with `expressionId` if `output` contains it. This is used to preserve
+ * nullability for resolved [[AttributeReference]].
*/
- def hasAttributeWithId(expressionId: ExprId): Boolean = {
- attributeIds.contains(expressionId)
+ def getAttributeById(expressionId: ExprId): Option[Attribute] =
+ Option(attributesById.get(expressionId))
+
+ /**
+ * Returns attribute with `expressionId` if `hiddenOutput` contains it.
+ */
+ def getHiddenAttributeById(expressionId: ExprId): Option[Attribute] =
+ Option(hiddenAttributesById.get(expressionId))
+
+ private def tryResolveMultipartNameByOutput(
+ multipartName: Seq[String],
+ nameComparator: NameComparator,
+ attributesForResolution: AttributeSeq,
+ canResolveByProposedAttributes: Boolean): Option[ResolvedMultipartName] = {
+ if (canResolveByProposedAttributes) {
+ val (candidates, nestedFields) =
+ attributesForResolution.getCandidatesForResolution(multipartName, nameComparator)
+ val resolvedCandidates = attributesForResolution.resolveCandidates(
+ multipartName,
+ nameComparator,
+ candidates,
+ nestedFields
+ )
+ if (resolvedCandidates.nonEmpty) {
+ Some(ResolvedMultipartName(candidates = resolvedCandidates, referencedAttribute = None))
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ }
+
+ private def tryResolveMultipartNameAsLiteralFunction(
+ multipartName: Seq[String]): Option[ResolvedMultipartName] = {
+ val literalFunction = LiteralFunctionResolution.resolve(multipartName).toSeq
+ if (literalFunction.nonEmpty) {
+ Some(ResolvedMultipartName(candidates = literalFunction, referencedAttribute = None))
+ } else {
+ None
+ }
+ }
+
+ private def tryResolveMultipartNameAsLateralColumnReference(
+ multipartName: Seq[String],
+ canLaterallyReferenceColumn: Boolean): Option[ResolvedMultipartName] = {
+ val (candidatesForLca, nestedFields, referencedAttribute) =
+ if (canLaterallyReferenceColumn) {
+ getLcaCandidates(multipartName)
+ } else {
+ (Seq.empty, Seq.empty, None)
+ }
+
+ val resolvedCandidatesForLca = attributesForResolution.resolveCandidates(
+ multipartName,
+ nameComparator,
+ candidatesForLca,
+ nestedFields
+ )
+ if (resolvedCandidatesForLca.nonEmpty) {
+ Some(
+ ResolvedMultipartName(
+ candidates = resolvedCandidatesForLca,
+ referencedAttribute = referencedAttribute
+ )
+ )
+ } else {
+ None
+ }
+ }
+
+ private def tryResolveAttributeAsGroupByAlias(
+ multipartName: Seq[String],
+ canReferenceAggregateExpressionAliases: Boolean): Option[ResolvedMultipartName] = {
+ if (canReferenceAggregateExpressionAliases) {
+ topAggregateExpressionsByAliasName.get(multipartName.head) match {
+ case None =>
+ None
+ case Some(alias) =>
+ Some(
+ ResolvedMultipartName(
+ candidates = Seq(alias.child),
+ referencedAttribute = None,
+ aliasMetadata = Some(alias.metadata)
+ )
+ )
+ }
+ } else {
+ None
+ }
}
/**
@@ -335,15 +589,16 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
* columns. Here we do [[AttributeSeq.fromNormalOutput]] because a struct field can also be
* laterally referenced and we need to properly resolve [[GetStructField]] node.
*/
- private def getLcaCandidates(multipartName: Seq[String]): (Seq[Attribute], Option[Attribute]) = {
+ private def getLcaCandidates(
+ multipartName: Seq[String]): (Seq[Attribute], Seq[String], Option[Attribute]) = {
val referencedAttribute = lcaRegistry.getAttribute(multipartName.head)
if (referencedAttribute.isDefined) {
val attributesForResolution = AttributeSeq.fromNormalOutput(Seq(referencedAttribute.get))
- val (newCandidates, _) =
+ val (newCandidates, nestedFields) =
attributesForResolution.getCandidatesForResolution(multipartName, nameComparator)
- (newCandidates, Some(referencedAttribute.get))
+ (newCandidates, nestedFields, Some(referencedAttribute.get))
} else {
- (Seq.empty, None)
+ (Seq.empty, Seq.empty, None)
}
}
@@ -365,10 +620,10 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
result
}
- private def createAttributeIds(attributes: Seq[Attribute]): HashSet[ExprId] = {
- val result = new HashSet[ExprId]
+ private def createAttributeIds(attributes: Seq[Attribute]): HashMap[ExprId, Attribute] = {
+ val result = new HashMap[ExprId, Attribute](attributes.size)
for (attribute <- attributes) {
- result.add(attribute.exprId)
+ result.put(attribute.exprId, attribute)
}
result
@@ -376,35 +631,40 @@ class NameScope(val output: Seq[Attribute] = Seq.empty) extends SQLConfHelper {
}
/**
- * The [[NameScopeStack]] is a stack of [[NameScope]]s managed by the [[Resolver]]. Usually a top
- * scope is used for name resolution, but in case of correlated subqueries we can lookup names in
- * the parent scopes. Low-level scope creation is managed internally, and only high-level api like
- * [[withNewScope]] is available to the resolvers. Freshly-created [[NameScopeStack]] contains an
- * empty root [[NameScope]], which in the context of [[Resolver]] corresponds to the query output.
+ * The [[NameScopeStack]] is a stack of [[NameScope]]s managed by the [[Resolver]]. Usually the
+ * current scope is used for name resolution, but in case of correlated subqueries we can lookup
+ * names in the parent scopes. Low-level scope creation is managed internally, and only high-level
+ * api like [[withNewScope]] is available to the resolvers. Freshly-created [[NameScopeStack]]
+ * contains an empty root [[NameScope]], which in the context of [[Resolver]] corresponds to the
+ * query output.
*/
class NameScopeStack extends SQLConfHelper {
private val stack = new ArrayDeque[NameScope]
stack.push(new NameScope)
/**
- * Get the top scope, which is a default choice for name resolution.
+ * Get the current scope, which is a default choice for name resolution.
*/
- def top: NameScope = {
+ def current: NameScope = {
stack.peek()
}
/**
- * Completely overwrite the top scope state with operator `output`.
+ * Completely overwrite the current scope state with operator `output` and `hiddenOutput`. If
+ * `hiddenOutput` is not provided, preserve the previous `hiddenOutput`. Additionally, update
+ * nullabilities of attributes in hidden output from new output, so that if attribute was
+ * nullable in either old hidden output or new output, it must stay nullable in new hidden
+ * output as well.
*
* This method is called by the [[Resolver]] when we've calculated the output of an operator that
* is being resolved. The new output is calculated based on the outputs of operator's children.
*
- * Example for [[SubqueryAlias]], here we rewrite the top [[NameScope]]'s attributes to prepend
- * subquery qualifier to their names:
+ * Example for [[SubqueryAlias]], here we rewrite the current [[NameScope]]'s attributes to
+ * prepend subquery qualifier to their names:
*
* {{{
* val qualifier = sa.identifier.qualifier :+ sa.alias
- * scope.overwriteTop(scope.output.map(attribute => attribute.withQualifier(qualifier)))
+ * scopes.overwriteCurrent(scope.output.map(attribute => attribute.withQualifier(qualifier)))
* }}}
*
* Trivially, we would call this method for every operator in the query plan,
@@ -413,10 +673,57 @@ class NameScopeStack extends SQLConfHelper {
*
* This method should be preferred over [[withNewScope]].
*/
- def overwriteTop(output: Seq[Attribute]): Unit = {
- val newScope = new NameScope(output)
+ def overwriteCurrent(
+ output: Option[Seq[Attribute]] = None,
+ hiddenOutput: Option[Seq[Attribute]] = None): Unit = {
+ val hiddenOutputWithUpdatedNullabilities = updateNullabilitiesInHiddenOutput(
+ output.getOrElse(stack.peek().output),
+ hiddenOutput.getOrElse(stack.peek().hiddenOutput)
+ )
+ val newScope = stack.pop.overwriteOutput(output, Some(hiddenOutputWithUpdatedNullabilities))
+
+ stack.push(newScope)
+ }
+
+ /**
+ * Overwrites output of the current [[NameScope]] entry and:
+ * 1. extends hidden output with the provided output (only attributes that are not in the hidden
+ * output are added). This is done because resolution of arguments can be done through certain
+ * operators by hidden output. This use case is specific to Dataframe programs. Example:
+ *
+ * {{{
+ * val df = (1 to 100).map { i => (i, i % 10, i % 2 == 0) }.toDF("a", "b", "c")
+ * df.select($"a", $"b").filter($"c")
+ * }}}
+ *
+ * Unresolved tree would be:
+ *
+ * Filter 'c
+ * +- 'Project ['a, 'b]
+ * +- Project [_1 AS a, _2 AS b, _3 AS c]
+ * +- LocalRelation [_1, _2, _3]
+ *
+ * As it can be seen in the example above, `c` from the [[Filter]] condition should be resolved
+ * using the `hiddenOutput` (because its child output doesn't contain `c`). That's why in hidden
+ * output we have to have both hidden output from the previous scope and the provided output.
+ * This is done for [[Project]] and [[Aggregate]] operators.
+ *
+ * 2. updates nullabilities of attributes in hidden output from new output, so that if attribute
+ * was nullable in either old hidden output or new output, it must stay nullable in new hidden
+ * output as well.
+ */
+ def overwriteOutputAndExtendHiddenOutput(output: Seq[Attribute]): Unit = {
+ val prevScope = stack.pop
+ val hiddenOutputWithUpdatedNullabilities =
+ updateNullabilitiesInHiddenOutput(output, prevScope.hiddenOutput)
+ val hiddenOutput = hiddenOutputWithUpdatedNullabilities ++ output.filter { attribute =>
+ prevScope.getHiddenAttributeById(attribute.exprId).isEmpty
+ }
+ val newScope = prevScope.overwriteOutput(
+ output = Some(output),
+ hiddenOutput = Some(hiddenOutput)
+ )
- stack.pop()
stack.push(newScope)
}
@@ -455,13 +762,176 @@ class NameScopeStack extends SQLConfHelper {
* resolve(unresolvedExcept.right)
* }
* }}}
+ *
+ * After finishing execution of the body within the `withNewScope`, pops the stack. It also
+ * propagates `hiddenOutput` upwards because of name resolution by overwriting the current
+ * [[NameScope.hiddenOutput]] with the popped one. This is not done in case `withNewScope` was
+ * called in the context of subquery resolution (which is indicated by `isSubqueryRoot` flag),
+ * because we don't want to overwrite the existing `hiddenOutput` of the main plan.
+ *
+ * @param isSubqueryRoot Indicates that the current scope is a root of a subquery. This is used by
+ * [[NameScopeStack.resolveMultipartName]] to detect the nearest outer scope.
*/
- def withNewScope[R](body: => R): R = {
- stack.push(new NameScope)
+ def withNewScope[R](isSubqueryRoot: Boolean = false)(body: => R): R = {
+ stack.push(new NameScope(isSubqueryRoot = isSubqueryRoot))
try {
body
} finally {
- stack.pop()
+ val childScope = stack.pop()
+ if (stack.size() > 0 && !childScope.isSubqueryRoot) {
+ val currentScope = stack.pop()
+ stack.push(currentScope.overwriteOutput(hiddenOutput = Some(childScope.hiddenOutput)))
+ }
+ }
+ }
+
+ /**
+ * Resolve multipart name into a [[NameTarget]] from current or outer scopes. Currently we only
+ * support one level of correlation, so we look up `multipartName` in current scope, and if the
+ * name was not found, we look it up in the nearest outer scope:
+ *
+ * {{{
+ * -- 'a' is a simple lookup from the current scope.
+ * SELECT a FROM (SELECT col1 AS a FROM VALUES (1));
+ * }}}
+ *
+ * {{{
+ * -- `a` in `(SELECT a + 1)` will be wrapped in [[OuterReference]].
+ * SELECT a, (SELECT a + 1) AS b FROM (SELECT col1 AS a FROM VALUES (1));
+ * }}}
+ *
+ * The ambiguity between local and outer references is resolved in favour of current:
+ * {{{
+ * -- There's no correlation here, subquery references its column from the current scope.
+ * -- This returns [1, 2].
+ * SELECT col1, (SELECT col1 FROM VALUES (2)) AS b FROM VALUES (1)
+ * }}}
+ *
+ * Correlations beyond one level are not supported:
+ * {{{
+ * -- 3 levels, fails with `UNRESOLVED_COLUMN`.
+ * SELECT (
+ * SELECT (
+ * SELECT t1.col1 FROM VALUES (3) AS t3
+ * ) FROM VALUES (2) AS t2
+ * ) FROM VALUES (1) AS t1;
+ * }}}
+ *
+ * Correlated references are accessible from lower subquery operators:
+ * {{{
+ * -- Returns [1, 1]
+ * SELECT
+ * col1, (SELECT * FROM (SELECT t1.col1 FROM VALUES (2) AS t2))
+ * FROM
+ * VALUES (1) AS t1;
+ * }}}
+ *
+ * We cannot reference LCA or aggregate expression by alias in the outer scope:
+ * {{{
+ * -- These examples fail with `UNRESOLVED_COLUMN`.
+ * -- LCA in outer scope.
+ * SELECT col1 AS a, (SELECT a + 1) AS b FROM VALUES (1);
+ * -- Aliased aggerate expression in outer scope.
+ * SELECT col1 AS a FROM VALUES (1) GROUP BY a, (SELECT a + 1);
+ * }}}
+ *
+ * Only [[Attribute]]s are wrapped in [[OuterReference]]:
+ * {{{
+ * -- The subquery's [[Project]] list will contain outer(col1#0).f1.f2.
+ * SELECT
+ * col1, (SELECT col1.f1.f2 + 1) AS b
+ * FROM
+ * VALUES (named_struct('f1', named_struct('f2', 1)));
+ * }}}
+ */
+ def resolveMultipartName(
+ multipartName: Seq[String],
+ canLaterallyReferenceColumn: Boolean = true,
+ canReferenceAggregateExpressionAliases: Boolean = false,
+ canResolveNameByHiddenOutput: Boolean = false): NameTarget = {
+ val nameTargetFromCurrentScope = current.resolveMultipartName(
+ multipartName,
+ canLaterallyReferenceColumn = canLaterallyReferenceColumn,
+ canReferenceAggregateExpressionAliases = canReferenceAggregateExpressionAliases,
+ canResolveNameByHiddenOutput = canResolveNameByHiddenOutput
+ )
+
+ if (nameTargetFromCurrentScope.candidates.nonEmpty) {
+ nameTargetFromCurrentScope
+ } else {
+ outer match {
+ case Some(outer) =>
+ val nameTarget = outer.resolveMultipartName(
+ multipartName,
+ canLaterallyReferenceColumn = false,
+ canReferenceAggregateExpressionAliases = false
+ )
+
+ if (nameTarget.candidates.nonEmpty) {
+ nameTarget.copy(
+ isOuterReference = true,
+ candidates = nameTarget.candidates.map(wrapCandidateInOuterReference)
+ )
+ } else {
+ nameTargetFromCurrentScope
+ }
+
+ case None =>
+ nameTargetFromCurrentScope
+ }
+ }
+ }
+
+ /**
+ * Find the nearest outer scope and return it if we are in a subquery.
+ */
+ private def outer: Option[NameScope] = {
+ var outerScope: Option[NameScope] = None
+
+ val iter = stack.iterator
+ while (iter.hasNext && !outerScope.isDefined) {
+ val scope = iter.next
+
+ if (scope.isSubqueryRoot && iter.hasNext) {
+ outerScope = Some(iter.next)
+ }
+ }
+
+ outerScope
+ }
+
+ /**
+ * Wrap candidate in [[OuterReference]]. If the root is not an [[Attribute]], but an
+ * [[ExtractValue]] (struct/map/array field reference) we find the actual [[Attribute]] and wrap
+ * it in [[OuterReference]].
+ */
+ private def wrapCandidateInOuterReference(candidate: Expression): Expression = candidate match {
+ case candidate: Attribute =>
+ OuterReference(candidate)
+ case extractValue: ExtractValue =>
+ extractValue.transformUp {
+ case attribute: Attribute => OuterReference(attribute)
+ case other => other
+ }
+ case _ =>
+ candidate
+ }
+
+ /**
+ * When the scope gets the new output, we need to refresh nullabilities in its `hiddenOutput`. If
+ * an attribute is nullable in either old hidden output or new output, it must remain nullable in
+ * new hidden output as well.
+ */
+ private def updateNullabilitiesInHiddenOutput(
+ output: Seq[Attribute],
+ hiddenOutput: Seq[Attribute]) = {
+ val outputLookup = new HashMap[ExprId, Attribute](output.size)
+ output.foreach(attribute => outputLookup.put(attribute.exprId, attribute))
+
+ hiddenOutput.map {
+ case attribute if outputLookup.containsKey(attribute.exprId) =>
+ attribute.withNullability(attribute.nullable || outputLookup.get(attribute.exprId).nullable)
+ case attribute => attribute
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameTarget.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameTarget.scala
index 9e949835c413..5b5f4e444bb9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameTarget.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/NameTarget.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression}
import org.apache.spark.sql.catalyst.util.StringUtils.orderSuggestedIdentifiersBySimilarity
import org.apache.spark.sql.errors.QueryCompilationErrors
+import org.apache.spark.sql.types.Metadata
/**
* [[NameTarget]] is a result of a multipart name resolution of the
@@ -54,16 +55,22 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
* @param aliasName If the candidates size is 1 and it's type is [[ExtractValue]] (which means that
* it's a field/value/item from a recursive type), then the `aliasName` should be the name with
* which the candidate needs to be aliased. Otherwise, `aliasName` is `None`.
+ * @param aliasMetadata If the candidates were created out of expressions referenced by group by
+ * alias, store the metadata of the alias. Otherwise, `aliasMetadata` is `None`.
* @param lateralAttributeReference If the candidate is laterally referencing another column this
* field is populated with that column's attribute.
* @param output [[output]] of a [[NameScope]] that produced this [[NameTarget]]. Used to provide
* suggestions for thrown errors.
+ * @param isOuterReference A flag indicating that this [[NameTarget]] resolves to an outer
+ * reference.
*/
case class NameTarget(
candidates: Seq[Expression],
aliasName: Option[String] = None,
+ aliasMetadata: Option[Metadata] = None,
lateralAttributeReference: Option[Attribute] = None,
- output: Seq[Attribute] = Seq.empty) {
+ output: Seq[Attribute] = Seq.empty,
+ isOuterReference: Boolean = false) {
/**
* Pick a single candidate from `candidates`:
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala
new file mode 100644
index 000000000000..8755c24f9ca3
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PlanRewriter.scala
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.expressions.SubqueryExpression
+import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan}
+import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
+import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION
+
+/**
+ * Utility wrapper on top of [[RuleExecutor]], used to apply post-resolution rules on single-pass
+ * resolution result. [[SinglePassRewriter]] transforms the plan and the subqueries inside.
+ */
+class PlanRewriter(planRewriteRules: Seq[Rule[LogicalPlan]]) {
+ private val planRewriter = new RuleExecutor[LogicalPlan] {
+ override def batches: Seq[Batch] =
+ Seq(
+ Batch(
+ "Plan Rewriting",
+ Once,
+ planRewriteRules: _*
+ )
+ )
+ }
+
+ /**
+ * Rewrites the plan by first recursing into all subqueries and applying post-resolution rules on
+ * them and then applying post-resolution rules on the entire plan.
+ */
+ def rewriteWithSubqueries(plan: LogicalPlan): LogicalPlan =
+ AnalysisHelper.allowInvokingTransformsInAnalyzer {
+ val planWithRewrittenSubqueries =
+ plan.transformAllExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION)) {
+ case subqueryExpression: SubqueryExpression =>
+ val rewrittenSubqueryPlan = rewrite(subqueryExpression.plan)
+
+ subqueryExpression.withNewPlan(rewrittenSubqueryPlan)
+ }
+
+ rewrite(planWithRewrittenSubqueries)
+ }
+
+ /**
+ * Rewrites the plan __without__ recursing into the subqueries.
+ */
+ private def rewrite(plan: LogicalPlan): LogicalPlan = planRewriter.execute(plan)
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PredicateResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PredicateResolver.scala
index 1c4d8dd50113..7669fcce5329 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PredicateResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PredicateResolver.scala
@@ -43,17 +43,17 @@ class PredicateResolver(
with ResolvesExpressionChildren {
private val typeCoercionTransformations: Seq[Expression => Expression] =
- if (conf.ansiEnabled) {
- PredicateResolver.ANSI_TYPE_COERCION_TRANSFORMATIONS
- } else {
- PredicateResolver.TYPE_COERCION_TRANSFORMATIONS
- }
+ if (conf.ansiEnabled) {
+ PredicateResolver.ANSI_TYPE_COERCION_TRANSFORMATIONS
+ } else {
+ PredicateResolver.TYPE_COERCION_TRANSFORMATIONS
+ }
private val typeCoercionResolver: TypeCoercionResolver =
new TypeCoercionResolver(timezoneAwareExpressionResolver, typeCoercionTransformations)
override def resolve(unresolvedPredicate: Predicate): Expression = {
val predicateWithResolvedChildren =
- withResolvedChildren(unresolvedPredicate, expressionResolver.resolve)
+ withResolvedChildren(unresolvedPredicate, expressionResolver.resolve _)
val predicateWithTypeCoercion = typeCoercionResolver.resolve(predicateWithResolvedChildren)
val predicateWithCharTypePadding = {
ApplyCharTypePaddingHelper.singleNodePaddingForStringComparison(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala
index 8d85804a9363..576cb98bbabe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProducesUnresolvedSubtree.scala
@@ -37,6 +37,10 @@ trait ProducesUnresolvedSubtree extends ResolvesExpressionChildren {
* node. Method ensures that the downwards traversal never visits previously resolved nodes by
* tracking the limits of the traversal with a tag. Invokes a resolver callback to resolve
* children, but DOES NOT resolve the root of the subtree.
+ *
+ * If the result of the callback is the same object as the source `expression`, we don't perform
+ * the downwards traversal. This is both more optimal and a fail-safe mechanism in case we
+ * accidentally lose the [[ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY]] tag.
*/
protected def withResolvedSubtree(
expression: Expression,
@@ -45,8 +49,31 @@ trait ProducesUnresolvedSubtree extends ResolvesExpressionChildren {
child.setTagValue(ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY, ())
}
- val result = body
+ val resultExpression = body
- withResolvedChildren(result, expressionResolver)
+ if (resultExpression.eq(expression)) {
+ expression.children.foreach { child =>
+ child.unsetTagValue(ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY)
+ }
+ resultExpression
+ } else {
+ withResolvedChildren(resultExpression, expressionResolver)
+ }
+ }
+
+ /**
+ * Try to pop the tag that marks the boundary of the single-pass subtree resolution.
+ * [[ExpressionResolver]] calls this method to check if the subtree traversal needs to be stopped
+ * because lower subtree is already resolved.
+ */
+ protected def tryPopSinglePassSubtreeBoundary(unresolvedExpression: Expression): Boolean = {
+ if (unresolvedExpression
+ .getTagValue(ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY)
+ .isDefined) {
+ unresolvedExpression.unsetTagValue(ExpressionResolver.SINGLE_PASS_SUBTREE_BOUNDARY)
+ true
+ } else {
+ false
+ }
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala
index 04623eca1c54..f6af7e4f2f41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ProjectResolver.scala
@@ -17,12 +17,21 @@
package org.apache.spark.sql.catalyst.analysis.resolver
+import java.util.{HashMap, HashSet}
+
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
-import org.apache.spark.sql.catalyst.analysis.{withPosition, AnalysisErrorAt}
-import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, NamedExpression}
-import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.expressions.{
+ Alias,
+ Attribute,
+ Expression,
+ ExprId,
+ ExprUtils,
+ NamedExpression
+}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Join, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.internal.SQLConf
/**
@@ -34,14 +43,13 @@ import org.apache.spark.sql.internal.SQLConf
*/
class ProjectResolver(operatorResolver: Resolver, expressionResolver: ExpressionResolver)
extends TreeNodeResolver[Project, LogicalPlan] {
-
- private val isLcaEnabled = conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)
private val scopes = operatorResolver.getNameScopes
/**
- * * [[Project]] introduces a new scope to resolve its subtree and project list expressions.
- * * During the resolution we determine whether the output operator will be [[Aggregate]] or
- * * [[Project]] (based on the `hasAggregateExpressions` flag).
+ * [[Project]] introduces a new scope to resolve its subtree and project list expressions.
+ * During the resolution we determine whether the output operator will be [[Aggregate]] or
+ * [[Project]] (based on the `hasAggregateExpressions` flag). If the result is an [[Aggregate]]
+ * validate it using the [[ExprUtils.assertValidAggregation]].
*
* If the output operator is [[Project]] and if lateral column alias resolution is enabled, we
* construct a multi-level [[Project]], created from all lateral column aliases and their
@@ -51,10 +59,15 @@ class ProjectResolver(operatorResolver: Resolver, expressionResolver: Expression
* current scope with resolved operators output to expose new names to the parent operators.
*/
override def resolve(unresolvedProject: Project): LogicalPlan = {
- val (resolvedOperator, resolvedProjectList) = scopes.withNewScope {
+ val (resolvedOperator, resolvedProjectList) = scopes.withNewScope() {
val resolvedChild = operatorResolver.resolve(unresolvedProject.child)
+ val childReferencedAttributes = expressionResolver.getLastReferencedAttributes
val resolvedProjectList =
expressionResolver.resolveProjectList(unresolvedProject.projectList, unresolvedProject)
+
+ val resolvedChildWithMetadataColumns =
+ retainOriginalJoinOutput(resolvedChild, resolvedProjectList, childReferencedAttributes)
+
if (resolvedProjectList.hasAggregateExpressions) {
if (resolvedProjectList.hasLateralColumnAlias) {
// Disable LCA in Aggregates until fully supported.
@@ -63,32 +76,157 @@ class ProjectResolver(operatorResolver: Resolver, expressionResolver: Expression
val aggregate = Aggregate(
groupingExpressions = Seq.empty[Expression],
aggregateExpressions = resolvedProjectList.expressions,
- child = resolvedChild,
+ child = resolvedChildWithMetadataColumns,
hint = None
)
- if (resolvedProjectList.hasAttributes) {
- aggregate.failAnalysis(errorClass = "MISSING_GROUP_BY", messageParameters = Map.empty)
- }
+
+ // TODO: This validation function does a post-traversal. This is discouraged in
+ // single-pass Analyzer.
+ ExprUtils.assertValidAggregation(aggregate)
+
(aggregate, resolvedProjectList)
} else {
- val projectWithLca = if (isLcaEnabled) {
- buildProjectWithResolvedLCAs(resolvedChild, resolvedProjectList.expressions)
+ val projectWithLca = if (conf.getConf(SQLConf.LATERAL_COLUMN_ALIAS_IMPLICIT_ENABLED)) {
+ buildProjectWithResolvedLCAs(
+ resolvedChildWithMetadataColumns,
+ resolvedProjectList.expressions
+ )
} else {
- Project(resolvedProjectList.expressions, resolvedChild)
+ Project(resolvedProjectList.expressions, resolvedChildWithMetadataColumns)
}
(projectWithLca, resolvedProjectList)
}
}
- withPosition(unresolvedProject) {
- scopes.overwriteTop(
+ scopes.overwriteOutputAndExtendHiddenOutput(
+ output =
resolvedProjectList.expressions.map(namedExpression => namedExpression.toAttribute)
- )
- }
+ )
resolvedOperator
}
+ /**
+ * This method adds a [[Project]] node on top of a [[Join]], if [[Join]]'s output has been
+ * changed when metadata columns are added to [[Project]] nodes below the [[Join]]. This is
+ * necessary in order to stay compatible with fixed-point analyzer. Instead of doing this in
+ * [[JoinResolver]] we must do this here, because while resolving [[Join]] we still don't know if
+ * we should add a [[Project]] or not. For example consider the following query:
+ *
+ * {{{
+ * -- tables: nt1(k, v1), nt2(k, v2), nt3(k, v3)
+ * SELECT * FROM nt1 NATURAL JOIN nt2 JOIN nt3 ON nt2.k = nt3.k;
+ * }}}
+ *
+ * Unresolved plan will be:
+ *
+ * 'Project [*]
+ * +- 'Join Inner, ('nt2.k = 'nt3.k)
+ * :- 'Join NaturalJoin(Inner)
+ * : :- 'UnresolvedRelation [nt1], [], false
+ * : +- 'UnresolvedRelation [nt2], [], false
+ * +- 'UnresolvedRelation [nt3], [], false
+ *
+ * After resolving the inner natural join, the plan becomes:
+ *
+ * 'Project [*]
+ * +- 'Join Inner, ('nt2.k = 'nt3.k)
+ * :- Project [k#15, v1#16, v2#28, k#27]
+ * : +- Join Inner, (k#15 = k#27)
+ * : : +- SubqueryAlias nt1
+ * : : +- LocalRelation [k#15, v1#16]
+ * : +- SubqueryAlias nt2
+ * : +- LocalRelation [k#27, v2#28]
+ * +- 'UnresolvedRelation [nt3], [], false
+ *
+ * Because we are resolving a natural join, we have placed a [[Project]] node on top of it with
+ * the inner join's output. Additionally, in single-pass, we add all metadata columns as we
+ * resolve up and then prune away unnecessary columns later (more in [[PruneMetadataColumns]]).
+ * This is necessary in order to stay compatible with fixed-point's [[AddMetadataColumns]] rule,
+ * because [[AddMetadata]] columns will recognize k#27 as missing attribute needed for [[Join]]
+ * condition and will therefore add it in the below [[Project]] node. Because of this we are also
+ * adding k#27 as a metadata column to this [[Project]]. This addition of a metadata column
+ * changes the original output of the outer join (because one of the inputs has changed) and in
+ * order to stay compatible with fixed-point, we need to place another [[Project]] on top of the
+ * outer join with its original output. Now, the final plan looks like this:
+ *
+ * Project [k#15, v1#16, v2#28, k#31, v3#32]
+ * +- Project [k#15, v1#16, v2#28, k#31, v3#32]
+ * +- Join Inner, (k#27 = k#31)
+ * :- Project [k#15, v1#16, v2#28, k#27]
+ * : +- Join Inner, (k#15 = k#27)
+ * : :- SubqueryAlias nt1
+ * : : +- LocalRelation [k#15, v1#16]
+ * : +- SubqueryAlias nt2
+ * : +- LocalRelation [k#27, v2#28]
+ * +- SubqueryAlias nt3
+ * +- LocalRelation [k#31, v3#32]
+ *
+ * As can be seen, the [[Project]] node immediately on top of [[Join]] doesn't contain the
+ * metadata column k#27 that we have added. Because of this, k#27 will be pruned away later.
+ *
+ * Now consider the following query for the same input:
+ *
+ * {{{ SELECT *, nt2.k FROM nt1 NATURAL JOIN nt2 JOIN nt3 ON nt2.k = nt3.k; }}}
+ *
+ * The plan will be:
+ *
+ * Project [k#15, v1#16, v2#28, k#31, v3#32, k#27]
+ * +- Join Inner, (k#27 = k#31)
+ * :- Project [k#15, v1#16, v2#28, k#27]
+ * : +- Join Inner, (k#15 = k#27)
+ * : :- SubqueryAlias nt1
+ * : : +- LocalRelation [k#15, v1#16]
+ * : +- SubqueryAlias nt2
+ * : +- LocalRelation [k#27, v2#28]
+ * +- SubqueryAlias nt3
+ * +- LocalRelation [k#31, v3#32]
+ *
+ * In fixed-point, because we are referencing k#27 from [[Project]] node, [[AddMetadataColumns]]
+ * (which is transforming the tree top-down) will see that [[Project]] has a missing metadata
+ * column and will therefore place k#27 in the [[Project]] node below outer [[Join]]. This is
+ * important, because by [[AddMetadataColumns]] logic, we don't check whether the output of the
+ * outer [[Join]] has changed, and we only check the output change for top-most [[Project]].
+ * Because we need to stay fully compatible with fixed-point, in this case w don't place a
+ * [[Project]] on top of the outer [[Join]] even though its output has changed.
+ */
+ private def retainOriginalJoinOutput(
+ plan: LogicalPlan,
+ resolvedProjectList: ResolvedProjectList,
+ childReferencedAttributes: HashMap[ExprId, Attribute]): LogicalPlan = {
+ plan match {
+ case join: Join
+ if childHasMissingAttributesNotInProjectList(
+ resolvedProjectList.expressions,
+ childReferencedAttributes
+ ) =>
+ Project(scopes.current.output, join)
+ case other => other
+ }
+ }
+
+ /**
+ * Returns true if a child node of [[Project]] has missing attributes that can be resolved from
+ * [[NameScope.hiddenOutput]] and those attributes are not present in the project list.
+ */
+ private def childHasMissingAttributesNotInProjectList(
+ projectList: Seq[NamedExpression],
+ referencedAttributes: HashMap[ExprId, Attribute]): Boolean = {
+ val expressionIdsFromProjectList = new HashSet[ExprId](projectList.map(_.exprId).asJava)
+ val missingAttributes = new HashMap[ExprId, Attribute]
+ referencedAttributes.asScala
+ .foreach {
+ case (exprId, attribute) =>
+ if (!expressionIdsFromProjectList.contains(exprId) && attribute.isMetadataCol) {
+ missingAttributes.put(exprId, attribute)
+ }
+ }
+ val missingAttributeResolvedByHiddenOutput =
+ scopes.current.resolveMissingAttributesByHiddenOutput(missingAttributes)
+
+ missingAttributeResolvedByHiddenOutput.nonEmpty
+ }
+
/**
* Builds a multi-level [[Project]] with all lateral column aliases and their dependencies. First,
* from top scope, we acquire dependency levels of all aliases. Dependency level is defined as a
@@ -127,15 +265,15 @@ class ProjectResolver(operatorResolver: Resolver, expressionResolver: Expression
private def buildProjectWithResolvedLCAs(
resolvedChild: LogicalPlan,
originalProjectList: Seq[NamedExpression]) = {
- val aliasDependencyMap = scopes.top.lcaRegistry.getAliasDependencyLevels()
+ val aliasDependencyMap = scopes.current.lcaRegistry.getAliasDependencyLevels()
val (finalChildPlan, _) = aliasDependencyMap.asScala.foldLeft(
- (resolvedChild, scopes.top.output.map(_.asInstanceOf[NamedExpression]))
+ (resolvedChild, scopes.current.output.map(_.asInstanceOf[NamedExpression]))
) {
case ((currentPlan, currentProjectList), availableAliases) =>
val referencedAliases = new ArrayBuffer[Alias]
availableAliases.forEach(
alias =>
- if (scopes.top.lcaRegistry.isAttributeLaterallyReferenced(alias.toAttribute)) {
+ if (scopes.current.lcaRegistry.isAttributeLaterallyReferenced(alias.toAttribute)) {
referencedAliases.append(alias)
}
)
@@ -150,7 +288,7 @@ class ProjectResolver(operatorResolver: Resolver, expressionResolver: Expression
val finalProjectList = originalProjectList.map(
alias =>
- if (scopes.top.lcaRegistry.isAttributeLaterallyReferenced(alias.toAttribute)) {
+ if (scopes.current.lcaRegistry.isAttributeLaterallyReferenced(alias.toAttribute)) {
alias.toAttribute
} else {
alias
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PruneMetadataColumns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PruneMetadataColumns.scala
new file mode 100644
index 000000000000..2bbf627da8fc
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PruneMetadataColumns.scala
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.HashSet
+
+import org.apache.spark.sql.catalyst.expressions.{ExprId, NamedExpression, OuterReference}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.CurrentOrigin
+import org.apache.spark.sql.catalyst.util._
+
+/**
+ * This is a special rule for single-pass resolver that performs a single downwards traversal in
+ * order to prune away unnecessary metadata columns. This is necessary because fixed-point is
+ * looking into the operator tree in order to determine whether it is necessary to add metadata
+ * columns. In single-pass, we always add metadata columns during the main traversal and this rule
+ * performs the cleanup of those columns that are unnecessary. Important thing to note here is that
+ * by "unnecessary" columns we are not referring to the ones that are not needed in upper operators
+ * for correct result, but to the columns that have been added by single-pass resolver but are not
+ * present in the fixed-point plan.
+ */
+object PruneMetadataColumns extends Rule[LogicalPlan] {
+
+ /**
+ * Entry point for [[PruneMetadataColumns]] rule.
+ */
+ override def apply(plan: LogicalPlan): LogicalPlan =
+ pruneMetadataColumns(plan = plan, neededAttributes = new HashSet[ExprId])
+
+ /**
+ * This method recursively prunes away unnecessary metadata columns, going from top to bottom.
+ *
+ * @param plan Operator for which we are pruning columns.
+ * @param neededAttributes A set of [[ExprId]]s that is required by operators above [[plan]].
+ *
+ * We distinguish three separate cases here, based on the type of operator node:
+ * - For [[Aggregate]] nodes we only need to propagate aggregate expressions as needed
+ * attributes to the lower operators. This is because in single-pass, we are not adding metadata
+ * columns to [[Aggregate]] operators.
+ * - For [[Project]] nodes we prune away all metadata columns that are either not required by
+ * operators above or they are duplicated in the project list.
+ * - For all other operators we collect references, add them to [[neededAttributes]] and
+ * recursively call [[pruneMetadataColumns]] for children.
+ */
+ private def pruneMetadataColumns(
+ plan: LogicalPlan,
+ neededAttributes: HashSet[ExprId]): LogicalPlan = plan match {
+ case aggregate: Aggregate =>
+ withNewChildrenPrunedByNeededAttributes(
+ aggregate,
+ aggregate.aggregateExpressions
+ )
+ case project: Project =>
+ pruneMetadataColumnsInProject(project, neededAttributes)
+ case other =>
+ pruneMetadataColumnsGenerically(other, neededAttributes)
+ }
+
+ /**
+ * Prune unnecessary columns for a [[Project]] node and recursively do it for its children. While
+ * pruning we preserve all non-qualified-access-only columns as well as any columns that are
+ * needed in the operators above, but without duplicating them in the project list. This behavior
+ * is consistent with fixed-point's behavior when
+ * [[SQLConf.ONLY_NECESSARY_AND_UNIQUE_METADATA_COLUMNS]] is true. We don't support legacy
+ * behavior in single-pass.
+ *
+ * IMPORTANT NOTE: In this case we only prune away only the qualified access only columns instead
+ * of all metadata columns. This is because we can have metadata columns from sources other than
+ * hidden output and additionally, when a column is resolved from any source, its qualified only
+ * flag is removed (see more in [[AttributeSeq]]). Therefore, qualified access only columns that
+ * appear in the project list must have come from artificial appending, and we should potentially
+ * prune them.
+ */
+ private def pruneMetadataColumnsInProject(project: Project, neededAttributes: HashSet[ExprId]) = {
+ val existingExprIds = new HashSet[ExprId]
+ val newProjectList = if (!neededAttributes.isEmpty) {
+ project.projectList.collect {
+ case namedExpression: NamedExpression if !namedExpression.toAttribute.qualifiedAccessOnly =>
+ existingExprIds.add(namedExpression.exprId)
+ namedExpression
+ case namedExpression: NamedExpression
+ if namedExpression.toAttribute.qualifiedAccessOnly && neededAttributes.contains(
+ namedExpression.exprId
+ ) && !existingExprIds.contains(namedExpression.exprId) =>
+ existingExprIds.add(namedExpression.exprId)
+ namedExpression
+ }
+ } else {
+ project.projectList
+ }
+ val projectWithNewChildren =
+ withNewChildrenPrunedByNeededAttributes(project, newProjectList).asInstanceOf[Project]
+ val newProject = CurrentOrigin.withOrigin(projectWithNewChildren.origin) {
+ projectWithNewChildren.copy(projectList = newProjectList)
+ }
+ newProject.copyTagsFrom(project)
+ newProject
+ }
+
+ /**
+ * Prune unnecessary metadata column in operators that are not [[Project]] or [[Aggregate]].
+ */
+ private def pruneMetadataColumnsGenerically(
+ operator: LogicalPlan,
+ neededAttributes: HashSet[ExprId]) = {
+ operator.references.foreach(attr => neededAttributes.add(attr.exprId))
+ val newChildren = operator.children.map { child =>
+ pruneMetadataColumns(child, new HashSet(neededAttributes))
+ }
+ operator.withNewChildren(newChildren)
+ }
+
+ private def withNewChildrenPrunedByNeededAttributes(
+ plan: LogicalPlan,
+ newNeededAttributes: Seq[NamedExpression]): LogicalPlan = {
+ val neededAttributes = new HashSet[ExprId]
+ newNeededAttributes.foreach {
+ case _: OuterReference =>
+ case other: NamedExpression =>
+ other.foreach {
+ case namedExpression: NamedExpression =>
+ neededAttributes.add(namedExpression.exprId)
+ case _ =>
+ }
+ }
+ plan.withNewChildren(
+ plan.children.map(
+ child => pruneMetadataColumns(child, neededAttributes)
+ )
+ )
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PullOutNondeterministicExpressionInExpressionTree.scala
similarity index 50%
copy from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala
copy to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PullOutNondeterministicExpressionInExpressionTree.scala
index 7f3ca796a494..3272c6975075 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/PullOutNondeterministicExpressionInExpressionTree.scala
@@ -17,20 +17,28 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import org.apache.spark.sql.catalyst.expressions.NamedExpression
+import java.util.LinkedHashMap
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression}
/**
- * Structure used to return results of the resolved project list.
- * - expressions: The resolved expressions. It is resolved using the
- * `resolveExpressionTreeInOperator`.
- * - hasAggregateExpressions: True if the resolved project list contains any aggregate
- * expressions.
- * - hasAttributes: True if the resolved project list contains any attributes that are not under
- * an aggregate expression.
- * - hasLateralColumnAlias: True if the resolved project list contains any lateral column aliases.
+ * Pull out nondeterministic expressions in an expression tree and replace them with the
+ * corresponding attributes in the `nondeterministicToAttributes` map.
*/
-case class ResolvedProjectList(
- expressions: Seq[NamedExpression],
- hasAggregateExpressions: Boolean,
- hasAttributes: Boolean,
- hasLateralColumnAlias: Boolean)
+object PullOutNondeterministicExpressionInExpressionTree {
+ def apply[ExpressionType <: Expression](
+ expression: ExpressionType,
+ nondeterministicToAttributes: LinkedHashMap[Expression, NamedExpression]): ExpressionType = {
+ expression
+ .transform {
+ case childExpression =>
+ nondeterministicToAttributes.get(childExpression) match {
+ case null =>
+ childExpression
+ case namedExpression =>
+ namedExpression.toAttribute
+ }
+ }
+ .asInstanceOf[ExpressionType]
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/RelationMetadataProvider.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/RelationMetadataProvider.scala
index 3bf15c51977f..6a3d8e161f2c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/RelationMetadataProvider.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/RelationMetadataProvider.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import java.util.HashMap
-import org.apache.spark.sql.catalyst.analysis.{withPosition, RelationResolution, UnresolvedRelation}
+import org.apache.spark.sql.catalyst.analysis.{RelationResolution, UnresolvedRelation}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.connector.catalog.LookupCatalog
import org.apache.spark.util.ArrayImplicits._
@@ -85,9 +85,11 @@ trait RelationMetadataProvider extends LookupCatalog {
isStreaming = unresolvedRelation.isStreaming
)
case _ =>
- withPosition(unresolvedRelation) {
- unresolvedRelation.tableNotFound(unresolvedRelation.multipartIdentifier)
- }
+ RelationId(
+ multipartIdentifier = unresolvedRelation.multipartIdentifier,
+ options = unresolvedRelation.options,
+ isStreaming = unresolvedRelation.isStreaming
+ )
}
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolutionValidator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolutionValidator.scala
index 93034e931bb0..2e536c33ef41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolutionValidator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolutionValidator.scala
@@ -26,26 +26,10 @@ import org.apache.spark.sql.catalyst.analysis.{
SchemaBinding
}
import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.plans.logical.{
- Aggregate,
- CTERelationDef,
- CTERelationRef,
- Distinct,
- Filter,
- GlobalLimit,
- LocalLimit,
- LocalRelation,
- LogicalPlan,
- OneRowRelation,
- Project,
- SubqueryAlias,
- Union,
- View,
- WithCTE
-}
+import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.util.AUTO_GENERATED_ALIAS
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.{BooleanType, DataType, MetadataBuilder, StructType}
+import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StructType}
/**
* The [[ResolutionValidator]] performs the validation work after the logical plan tree is
@@ -55,10 +39,11 @@ import org.apache.spark.sql.types.{BooleanType, DataType, MetadataBuilder, Struc
* The validation approach is single-pass, post-order, complementary to the resolution process.
*/
class ResolutionValidator {
+ private val attributeScopeStack = new AttributeScopeStack
+
private val expressionResolutionValidator = new ExpressionResolutionValidator(this)
- private[resolver] var attributeScopeStack = new AttributeScopeStack
- private val cteRelationDefIds = new HashSet[Long]
+ def getAttributeScopeStack: AttributeScopeStack = attributeScopeStack
/**
* Validate the resolved logical `plan` - assert invariants that should never be false no
@@ -70,7 +55,11 @@ class ResolutionValidator {
validate(plan)
}
- private def validate(operator: LogicalPlan): Unit = {
+ /**
+ * Validate a specific `operator`. This is an internal entry point for the recursive validation.
+ * Also, [[ExpressionResolutionValidator]] calls it to validate [[SubqueryExpression]] plans.
+ */
+ def validate(operator: LogicalPlan): Unit = {
operator match {
case withCte: WithCTE =>
validateWith(withCte)
@@ -92,6 +81,10 @@ class ResolutionValidator {
validateGlobalLimit(globalLimit)
case localLimit: LocalLimit =>
validateLocalLimit(localLimit)
+ case offset: Offset =>
+ validateOffset(offset)
+ case tail: Tail =>
+ validateTail(tail)
case distinct: Distinct =>
validateDistinct(distinct)
case inlineTable: ResolvedInlineTable =>
@@ -100,57 +93,69 @@ class ResolutionValidator {
validateRelation(localRelation)
case oneRowRelation: OneRowRelation =>
validateRelation(oneRowRelation)
- case union: Union =>
- validateUnion(union)
+ case range: Range =>
+ validateRelation(range)
+ case setOperationLike @ (_: Union | _: SetOperation) =>
+ validateSetOperationLike(setOperationLike)
+ case sort: Sort =>
+ validateSort(sort)
+ case join: Join =>
+ validateJoin(join)
+ case repartition: Repartition =>
+ validateRepartition(repartition)
// [[LogicalRelation]], [[HiveTableRelation]] and other specific relations can't be imported
// because of a potential circular dependency, so we match a generic Catalyst
// [[MultiInstanceRelation]] instead.
case multiInstanceRelation: MultiInstanceRelation =>
validateRelation(multiInstanceRelation)
}
- ExpressionIdAssigner.assertOutputsHaveNoConflictingExpressionIds(
- operator.children.map(_.output)
- )
+
+ operator match {
+ case withCte: WithCTE =>
+ case _ =>
+ ExpressionIdAssigner.assertOutputsHaveNoConflictingExpressionIds(
+ operator.children.map(_.output)
+ )
+ }
}
private def validateWith(withCte: WithCTE): Unit = {
+ val knownCteDefIds = new HashSet[Long](withCte.cteDefs.length)
+
for (cteDef <- withCte.cteDefs) {
+ assert(
+ !knownCteDefIds.contains(cteDef.id),
+ s"Duplicate CTE definition id: ${cteDef.id}"
+ )
+
validate(cteDef)
+
+ knownCteDefIds.add(cteDef.id)
}
+
validate(withCte.plan)
}
private def validateCteRelationDef(cteRelationDef: CTERelationDef): Unit = {
validate(cteRelationDef.child)
-
- assert(
- !cteRelationDefIds.contains(cteRelationDef.id),
- s"Duplicate CTE relation def ID: $cteRelationDef"
- )
-
- cteRelationDefIds.add(cteRelationDef.id)
}
private def validateCteRelationRef(cteRelationRef: CTERelationRef): Unit = {
- assert(
- cteRelationDefIds.contains(cteRelationRef.cteId),
- s"CTE relation ref ID is not known: $cteRelationRef"
- )
-
handleOperatorOutput(cteRelationRef)
}
private def validateAggregate(aggregate: Aggregate): Unit = {
- attributeScopeStack.withNewScope {
+ attributeScopeStack.withNewScope() {
validate(aggregate.child)
expressionResolutionValidator.validateProjectList(aggregate.aggregateExpressions)
+ aggregate.groupingExpressions.foreach(expressionResolutionValidator.validate)
}
handleOperatorOutput(aggregate)
}
private def validateProject(project: Project): Unit = {
- attributeScopeStack.withNewScope {
+ attributeScopeStack.withNewScope() {
validate(project.child)
expressionResolutionValidator.validateProjectList(project.projectList)
}
@@ -207,6 +212,16 @@ class ResolutionValidator {
expressionResolutionValidator.validate(localLimit.limitExpr)
}
+ private def validateOffset(offset: Offset): Unit = {
+ validate(offset.child)
+ expressionResolutionValidator.validate(offset.offsetExpr)
+ }
+
+ private def validateTail(tail: Tail): Unit = {
+ validate(tail.child)
+ expressionResolutionValidator.validate(tail.limitExpr)
+ }
+
private def validateDistinct(distinct: Distinct): Unit = {
validate(distinct.child)
}
@@ -225,31 +240,57 @@ class ResolutionValidator {
handleOperatorOutput(relation)
}
- private def validateUnion(union: Union): Unit = {
- union.children.foreach(validate)
+ private def validateSetOperationLike(plan: LogicalPlan): Unit = {
+ plan.children.foreach(validate)
- assert(union.children.length > 1, "Union operator has to have at least 2 children")
- val firstChildOutput = union.children.head.output
- for (child <- union.children.tail) {
+ assert(
+ plan.children.length > 1,
+ s"${plan.nodeName} operator has to have at least 2 children"
+ )
+ val firstChildOutput = plan.children.head.output
+ for (child <- plan.children.tail) {
val childOutput = child.output
assert(
childOutput.length == firstChildOutput.length,
- s"Unexpected output length for Union child $child"
+ s"Unexpected output length for ${plan.nodeName} child $child"
)
- childOutput.zip(firstChildOutput).foreach {
- case (current, first) =>
- assert(
- DataType.equalsStructurally(current.dataType, first.dataType, ignoreNullability = true),
- s"Unexpected type of Union child attribute $current for $child"
- )
+ }
+
+ handleOperatorOutput(plan)
+ }
+
+ private def validateSort(sort: Sort): Unit = {
+ validate(sort.child)
+ for (sortOrder <- sort.order) {
+ expressionResolutionValidator.validate(sortOrder.child)
+ }
+ }
+
+ private def validateRepartition(repartition: Repartition): Unit = {
+ validate(repartition.child)
+ }
+
+ private def validateJoin(join: Join) = {
+ attributeScopeStack.withNewScope() {
+ attributeScopeStack.withNewScope() {
+ validate(join.left)
+ validate(join.right)
+ assert(join.left.outputSet.intersect(join.right.outputSet).isEmpty)
+ }
+
+ attributeScopeStack.overwriteCurrent(join.left.output ++ join.right.output)
+
+ join.condition match {
+ case Some(condition) => expressionResolutionValidator.validate(condition)
+ case None =>
}
}
- handleOperatorOutput(union)
+ handleOperatorOutput(join)
}
private def handleOperatorOutput(operator: LogicalPlan): Unit = {
- attributeScopeStack.overwriteTop(operator.output)
+ attributeScopeStack.overwriteCurrent(operator.output)
operator.output.foreach(attribute => {
assert(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedAggregateExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedAggregateExpressions.scala
new file mode 100644
index 000000000000..9a3eba5da646
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedAggregateExpressions.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.HashSet
+
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+
+/**
+ * [[ResolvedAggregateExpressions]] is used by the
+ * [[ExpressionResolver.resolveAggregateExpressions]] to return resolution results.
+ * - expressions: The resolved expressions. They are resolved using the
+ * `resolveExpressionTreeInOperator`.
+ * - resolvedExpressionsWithoutAggregates: List of resolved aggregate expressions that don't have
+ * [[AggregateExpression]]s in their subtrees.
+ * - hasAttributeOutsideOfAggregateExpressions: True if `expressions` list contains any attributes
+ * that are not under an [[AggregateExpression]].
+ * - hasStar: True if there is a star (`*`) in aggregate expressions list
+ * - expressionIndexesWithAggregateFunctions: Indices of expressions in aggregate expressions list
+ * that have aggregate functions in their subtrees.
+ */
+case class ResolvedAggregateExpressions(
+ expressions: Seq[NamedExpression],
+ resolvedExpressionsWithoutAggregates: Seq[NamedExpression],
+ hasAttributeOutsideOfAggregateExpressions: Boolean,
+ hasStar: Boolean,
+ expressionIndexesWithAggregateFunctions: HashSet[Int]
+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala
index 7f3ca796a494..3cb1dc0d454c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedProjectList.scala
@@ -25,12 +25,9 @@ import org.apache.spark.sql.catalyst.expressions.NamedExpression
* `resolveExpressionTreeInOperator`.
* - hasAggregateExpressions: True if the resolved project list contains any aggregate
* expressions.
- * - hasAttributes: True if the resolved project list contains any attributes that are not under
- * an aggregate expression.
* - hasLateralColumnAlias: True if the resolved project list contains any lateral column aliases.
*/
case class ResolvedProjectList(
expressions: Seq[NamedExpression],
hasAggregateExpressions: Boolean,
- hasAttributes: Boolean,
hasLateralColumnAlias: Boolean)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedSubqueryExpressionPlan.scala
similarity index 53%
copy from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala
copy to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedSubqueryExpressionPlan.scala
index d3e93c82dfa2..30e560b7afdd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/AnalyzerBridgeState.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvedSubqueryExpressionPlan.scala
@@ -17,24 +17,21 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import java.util.HashMap
-
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
/**
- * The [[AnalyzerBridgeState]] is a state passed from legacy [[Analyzer]] to the single-pass
- * [[Resolver]].
+ * The result of [[SubqueryExpression.plan]] resolution. This is used internally in
+ * [[SubqueryExpressionResolver]].
*
- * @param relationsWithResolvedMetadata A map from [[UnresolvedRelation]] to the relations with
- * resolved metadata. It allows us to reuse the relation metadata and avoid duplicate
- * catalog/table lookups in dual-run mode (when
- * [[ANALYZER_SINGLE_PASS_RESOLVER_RELATION_BRIDGING_ENABLED]] is true).
+ * @param plan The resolved plan of the subquery.
+ * @param output Plan output. We don't use [[LogicalPlan.output]] in the single-pass Analyzer,
+ * because this method is often recursive.
+ * @param outerExpressions The outer expressions that are references in the plan. [[OuterReference]]
+ * wrapper is stripped away. These can be either actual leaf [[AttributeReference]]s or
+ * [[AggregateExpression]]s with outer references inside.
*/
-case class AnalyzerBridgeState(
- relationsWithResolvedMetadata: AnalyzerBridgeState.RelationsWithResolvedMetadata =
- new AnalyzerBridgeState.RelationsWithResolvedMetadata)
-
-object AnalyzerBridgeState {
- type RelationsWithResolvedMetadata = HashMap[UnresolvedRelation, LogicalPlan]
-}
+case class ResolvedSubqueryExpressionPlan(
+ plan: LogicalPlan,
+ output: Seq[Attribute],
+ outerExpressions: Seq[Expression])
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala
index 511f36b4a719..d82025b29509 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import scala.collection.mutable.ArrayBuffer
-
import org.apache.spark.SparkException
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.EvaluateUnresolvedInlineTable
@@ -33,36 +31,12 @@ import org.apache.spark.sql.catalyst.analysis.{
UnresolvedRelation,
UnresolvedSubqueryColumnAliases
}
-import org.apache.spark.sql.catalyst.expressions.{
- Alias,
- Attribute,
- AttributeSet,
- Expression,
- NamedExpression
-}
-import org.apache.spark.sql.catalyst.plans.logical.{
- AnalysisHelper,
- CTERelationDef,
- CTERelationRef,
- Distinct,
- Filter,
- GlobalLimit,
- LeafNode,
- LocalLimit,
- LocalRelation,
- LogicalPlan,
- OneRowRelation,
- Project,
- SubqueryAlias,
- Union,
- UnresolvedWith,
- View,
- WithCTE
-}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Expression}
+import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
+import org.apache.spark.sql.catalyst.trees.TreeNodeTag
import org.apache.spark.sql.connector.catalog.CatalogManager
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.BooleanType
/**
* The Resolver implements a single-pass bottom-up analysis algorithm in the Catalyst.
@@ -90,18 +64,24 @@ class Resolver(
override val extensions: Seq[ResolverExtension] = Seq.empty,
metadataResolverExtensions: Seq[ResolverExtension] = Seq.empty)
extends LogicalPlanResolver
- with ResolvesOperatorChildren
with DelegatesResolutionToExtensions {
private val scopes = new NameScopeStack
private val cteRegistry = new CteRegistry
+ private val subqueryRegistry = new SubqueryRegistry
private val planLogger = new PlanLogger
+ private val identifierAndCteSubstitutor = new IdentifierAndCteSubstitutor
private val relationResolution = Resolver.createRelationResolution(catalogManager)
private val functionResolution = new FunctionResolution(catalogManager, relationResolution)
private val expressionResolver = new ExpressionResolver(this, functionResolution, planLogger)
+ private val aggregateResolver = new AggregateResolver(this, expressionResolver)
private val expressionIdAssigner = expressionResolver.getExpressionIdAssigner
private val projectResolver = new ProjectResolver(this, expressionResolver)
private val viewResolver = new ViewResolver(resolver = this, catalogManager = catalogManager)
- private val unionResolver = new UnionResolver(this, expressionResolver)
+ private val setOperationLikeResolver =
+ new SetOperationLikeResolver(this, expressionResolver)
+ private val filterResolver = new FilterResolver(this, expressionResolver)
+ private val sortResolver = new SortResolver(this, expressionResolver)
+ private val joinResolver = new JoinResolver(this, expressionResolver)
/**
* [[relationMetadataProvider]] is used to resolve metadata for relations. It's initialized with
@@ -121,21 +101,38 @@ class Resolver(
metadataResolverExtensions
)
+ /**
+ * Get [[NameScopeStack]] bound to the used [[Resolver]].
+ */
+ def getNameScopes: NameScopeStack = scopes
+
/**
* Get the [[CteRegistry]] which is a single instance per query resolution.
*/
- def getCteRegistry: CteRegistry = {
- cteRegistry
- }
+ def getCteRegistry: CteRegistry = cteRegistry
/**
- * This method is an analysis entry point. It resolves the metadata and invokes [[resolve]],
- * which does most of the analysis work.
+ * Get the [[SubqueryRegistry]] which is a single instance per query resolution.
+ */
+ def getSubqueryRegistry: SubqueryRegistry = subqueryRegistry
+
+ /**
+ * This method is a top-level analysis entry point:
+ * 1. Substitute IDENTIFIERs and CTEs in the `unresolvedPlan` using
+ * [[IdentifierAndCteSubstitutor]];
+ * 2. Resolve the metadata for the plan using [[MetadataResolver]];
+ * 3. Resolve the plan using [[resolve]].
+ *
+ * This method is called for the top-level query and each unresolved [[View]].
*/
def lookupMetadataAndResolve(
unresolvedPlan: LogicalPlan,
analyzerBridgeState: Option[AnalyzerBridgeState] = None): LogicalPlan = {
- planLogger.logPlanResolutionEvent(unresolvedPlan, "Lookup metadata and resolve")
+ planLogger.logPlanResolutionEvent(unresolvedPlan, "IDENTIFIER and CTE substitution")
+
+ val planAfterSubstitution = identifierAndCteSubstitutor.substitutePlan(unresolvedPlan)
+
+ planLogger.logPlanResolutionEvent(planAfterSubstitution, "Metadata lookup")
relationMetadataProvider = analyzerBridgeState match {
case Some(analyzerBridgeState) =>
@@ -148,9 +145,13 @@ class Resolver(
relationMetadataProvider
}
- relationMetadataProvider.resolve(unresolvedPlan)
+ planLogger.logPlanResolutionEvent(planAfterSubstitution, "Main resolution")
+
+ relationMetadataProvider.resolve(planAfterSubstitution)
+
+ planAfterSubstitution.setTagValue(Resolver.TOP_LEVEL_OPERATOR, ())
- resolve(unresolvedPlan)
+ resolve(planAfterSubstitution)
}
/**
@@ -165,18 +166,24 @@ class Resolver(
* [[resolve]] will be called recursively during the unresolved plan traversal eventually
* producing a fully resolved plan or a descriptive error message.
*/
- override def resolve(unresolvedPlan: LogicalPlan): LogicalPlan =
+ override def resolve(unresolvedPlan: LogicalPlan): LogicalPlan = {
withOrigin(unresolvedPlan.origin) {
planLogger.logPlanResolutionEvent(unresolvedPlan, "Unresolved plan")
val resolvedPlan =
unresolvedPlan match {
+ case unresolvedJoin: Join =>
+ joinResolver.resolve(unresolvedJoin)
case unresolvedWith: UnresolvedWith =>
resolveWith(unresolvedWith)
+ case withCte: WithCTE =>
+ handleResolvedWithCte(withCte)
case unresolvedProject: Project =>
projectResolver.resolve(unresolvedProject)
+ case unresolvedAggregate: Aggregate =>
+ aggregateResolver.resolve(unresolvedAggregate)
case unresolvedFilter: Filter =>
- resolveFilter(unresolvedFilter)
+ filterResolver.resolve(unresolvedFilter)
case unresolvedSubqueryColumnAliases: UnresolvedSubqueryColumnAliases =>
resolveSubqueryColumnAliases(unresolvedSubqueryColumnAliases)
case unresolvedSubqueryAlias: SubqueryAlias =>
@@ -187,16 +194,26 @@ class Resolver(
resolveGlobalLimit(unresolvedGlobalLimit)
case unresolvedLocalLimit: LocalLimit =>
resolveLocalLimit(unresolvedLocalLimit)
+ case unresolvedOffset: Offset =>
+ resolveOffset(unresolvedOffset)
+ case unresolvedTail: Tail =>
+ resolveTail(unresolvedTail)
case unresolvedDistinct: Distinct =>
resolveDistinct(unresolvedDistinct)
case unresolvedRelation: UnresolvedRelation =>
resolveRelation(unresolvedRelation)
- case unresolvedCteRelationDef: CTERelationDef =>
- resolveCteRelationDef(unresolvedCteRelationDef)
+ case unresolvedCteRelationRef: UnresolvedCteRelationRef =>
+ resolveCteRelationRef(unresolvedCteRelationRef)
+ case cteRelationDef: CTERelationDef =>
+ handleResolvedCteRelationDef(cteRelationDef)
+ case cteRelationRef: CTERelationRef =>
+ handleLeafOperator(cteRelationRef)
case unresolvedInlineTable: UnresolvedInlineTable =>
resolveInlineTable(unresolvedInlineTable)
- case unresolvedUnion: Union =>
- unionResolver.resolve(unresolvedUnion)
+ case unresolvedSetOperationLike @ (_: Union | _: SetOperation) =>
+ setOperationLikeResolver.resolve(unresolvedSetOperationLike)
+ case unresolvedSort: Sort =>
+ sortResolver.resolve(unresolvedSort)
// See the reason why we have to match both [[LocalRelation]] and [[ResolvedInlineTable]]
// in the [[resolveInlineTable]] scaladoc
case resolvedInlineTable: ResolvedInlineTable =>
@@ -205,34 +222,25 @@ class Resolver(
handleLeafOperator(localRelation)
case unresolvedOneRowRelation: OneRowRelation =>
handleLeafOperator(unresolvedOneRowRelation)
+ case unresolvedRange: Range =>
+ handleLeafOperator(unresolvedRange)
case _ =>
tryDelegateResolutionToExtension(unresolvedPlan).getOrElse {
handleUnmatchedOperator(unresolvedPlan)
}
}
- if (resolvedPlan.children.nonEmpty) {
- val missingInput = resolvedPlan.missingInput
- if (missingInput.nonEmpty) {
- withPosition(unresolvedPlan) {
- throwMissingAttributesError(resolvedPlan, missingInput)
- }
- }
- }
-
- if (!resolvedPlan.resolved) {
- throwSinglePassFailedToResolveOperator(resolvedPlan)
+ withPosition(unresolvedPlan) {
+ validateResolvedOperatorGenerically(resolvedPlan)
}
planLogger.logPlanResolution(unresolvedPlan, resolvedPlan)
- preservePlanIdTag(unresolvedPlan, resolvedPlan)
- }
+ resolvedPlan.copyTagsFrom(unresolvedPlan)
- /**
- * Get [[NameScopeStack]] bound to the used [[Resolver]].
- */
- def getNameScopes: NameScopeStack = scopes
+ resolvedPlan
+ }
+ }
/**
* [[UnresolvedWith]] contains a list of unresolved CTE definitions, which are represented by
@@ -247,19 +255,13 @@ class Resolver(
* See [[CteScope]] scaladoc for all the details on how CTEs are resolved.
*/
private def resolveWith(unresolvedWith: UnresolvedWith): LogicalPlan = {
- val childOutputs = new ArrayBuffer[Seq[Attribute]]
-
for (cteRelation <- unresolvedWith.cteRelations) {
val (cteName, ctePlan) = cteRelation
- val resolvedCtePlan = scopes.withNewScope {
+ val resolvedCtePlan = scopes.withNewScope() {
expressionIdAssigner.withNewMapping() {
cteRegistry.withNewScope() {
- val resolvedCtePlan = resolve(ctePlan)
-
- childOutputs.append(scopes.top.output)
-
- resolvedCtePlan
+ resolve(ctePlan)
}
}
}
@@ -271,35 +273,31 @@ class Resolver(
resolve(unresolvedWith.child)
}
- childOutputs.append(scopes.top.output)
-
- ExpressionIdAssigner.assertOutputsHaveNoConflictingExpressionIds(childOutputs.toSeq)
-
- if (cteRegistry.currentScope.isRoot) {
- WithCTE(resolvedChild, cteRegistry.currentScope.getKnownCtes)
- } else {
- resolvedChild
- }
+ cteRegistry.currentScope.tryPutWithCTE(
+ unresolvedOperator = unresolvedWith,
+ resolvedOperator = resolvedChild
+ )
}
/**
- * [[Filter]] has a single child and a single condition and we resolve them in this respective
- * order.
+ * We may meet resolved [[WithCTE]] while traversing partially resolved trees in DataFrame
+ * programs. In that case we simply recurse into the CTE definitions and the main plan under
+ * new scopes and mappings.
*/
- private def resolveFilter(unresolvedFilter: Filter): LogicalPlan = {
- val resolvedChild = resolve(unresolvedFilter.child)
- val resolvedCondition =
- expressionResolver
- .resolveExpressionTreeInOperator(unresolvedFilter.condition, unresolvedFilter)
-
- val resolvedFilter = Filter(resolvedCondition, resolvedChild)
- if (resolvedFilter.condition.dataType != BooleanType) {
- withPosition(unresolvedFilter) {
- throwDatatypeMismatchFilterNotBoolean(resolvedFilter)
+ private def handleResolvedWithCte(withCte: WithCTE): LogicalPlan = {
+ val resolvedCteDefs = withCte.cteDefs.map { cteDef =>
+ scopes.withNewScope() {
+ expressionIdAssigner.withNewMapping() {
+ cteRegistry.withNewScope() {
+ resolve(cteDef).asInstanceOf[CTERelationDef]
+ }
+ }
}
}
- resolvedFilter
+ val resolvedPlan = resolve(withCte.plan)
+
+ WithCTE(plan = resolvedPlan, cteDefs = resolvedCteDefs)
}
/**
@@ -315,21 +313,20 @@ class Resolver(
unresolvedSubqueryColumnAliases: UnresolvedSubqueryColumnAliases): LogicalPlan = {
val resolvedChild = resolve(unresolvedSubqueryColumnAliases.child)
- if (unresolvedSubqueryColumnAliases.outputColumnNames.size != scopes.top.output.size) {
- withPosition(unresolvedSubqueryColumnAliases) {
- throw QueryCompilationErrors.aliasNumberNotMatchColumnNumberError(
- unresolvedSubqueryColumnAliases.outputColumnNames.size,
- scopes.top.output.size,
- unresolvedSubqueryColumnAliases
- )
- }
+ if (unresolvedSubqueryColumnAliases.outputColumnNames.size != scopes.current.output.size) {
+ throw QueryCompilationErrors.aliasNumberNotMatchColumnNumberError(
+ unresolvedSubqueryColumnAliases.outputColumnNames.size,
+ scopes.current.output.size,
+ unresolvedSubqueryColumnAliases
+ )
}
- val projectList = scopes.top.output.zip(unresolvedSubqueryColumnAliases.outputColumnNames).map {
- case (attr, columnName) => expressionIdAssigner.mapExpression(Alias(attr, columnName)())
- }
+ val projectList =
+ scopes.current.output.zip(unresolvedSubqueryColumnAliases.outputColumnNames).map {
+ case (attr, columnName) => expressionIdAssigner.mapExpression(Alias(attr, columnName)())
+ }
- overwriteTopScope(unresolvedSubqueryColumnAliases, projectList.map(_.toAttribute))
+ scopes.overwriteCurrent(output = Some(projectList.map(_.toAttribute)))
Project(projectList = projectList, child = resolvedChild)
}
@@ -338,16 +335,30 @@ class Resolver(
* [[SubqueryAlias]] has a single child and an identifier. We need to resolve the child and update
* the scope with the output, since upper expressions can reference [[SubqueryAlias]]es output by
* its identifier.
+ *
+ * Hidden output is reset when [[SubqueryAlias]] is reached during tree traversal. This has to be
+ * done because upper SQL projection (or DataFrame) cannot look down into the previous
+ * [[SubqueryAlias]]'s hidden output.
+ * Examples (both will throw `UNRESOLVED_COLUMN` exception):
+ *
+ * 1. SQL
+ * {{{
+ * -- Hidden output will be reset at [[SubqueryAlias]] and therefore both `output` and
+ * -- `hiddenOutput` will be [`col2`]. Because of that, `UNRESOLVED_COLUMN` is thrown
+ * -- when resolving `col1`
+ * SELECT col1 FROM (SELECT col2 FROM VALUES (1, 2));
+ * }}}
+ *
+ * 2. DataFrame
+ * {{{ spark.sql("SELECT * FROM VALUES (1, 2)").select("col1").as("q1").select("col2"); }}}
*/
private def resolveSubqueryAlias(unresolvedSubqueryAlias: SubqueryAlias): LogicalPlan = {
val resolvedSubqueryAlias =
unresolvedSubqueryAlias.copy(child = resolve(unresolvedSubqueryAlias.child))
val qualifier = resolvedSubqueryAlias.identifier.qualifier :+ resolvedSubqueryAlias.alias
- overwriteTopScope(
- unresolvedSubqueryAlias,
- scopes.top.output.map(attribute => attribute.withQualifier(qualifier))
- )
+ val output = scopes.current.output.map(attribute => attribute.withQualifier(qualifier))
+ scopes.overwriteCurrent(output = Some(output), hiddenOutput = Some(output))
resolvedSubqueryAlias
}
@@ -359,12 +370,10 @@ class Resolver(
private def resolveGlobalLimit(unresolvedGlobalLimit: GlobalLimit): LogicalPlan = {
val resolvedChild = resolve(unresolvedGlobalLimit.child)
- val resolvedLimitExpr = withPosition(unresolvedGlobalLimit) {
- expressionResolver.resolveLimitExpression(
- unresolvedGlobalLimit.limitExpr,
- unresolvedGlobalLimit
- )
- }
+ val resolvedLimitExpr = expressionResolver.resolveLimitLikeExpression(
+ unresolvedGlobalLimit.limitExpr,
+ unresolvedGlobalLimit.copy(child = resolvedChild)
+ )
GlobalLimit(resolvedLimitExpr, resolvedChild)
}
@@ -376,21 +385,53 @@ class Resolver(
private def resolveLocalLimit(unresolvedLocalLimit: LocalLimit): LogicalPlan = {
val resolvedChild = resolve(unresolvedLocalLimit.child)
- val resolvedLimitExpr = withPosition(unresolvedLocalLimit) {
- expressionResolver.resolveLimitExpression(
- unresolvedLocalLimit.limitExpr,
- unresolvedLocalLimit
- )
- }
+ val resolvedLimitExpr = expressionResolver.resolveLimitLikeExpression(
+ unresolvedLocalLimit.limitExpr,
+ unresolvedLocalLimit.copy(child = resolvedChild)
+ )
LocalLimit(resolvedLimitExpr, resolvedChild)
}
+ /**
+ * Resolve [[Offset]]. We have to resolve its child and resolve and validate its offset
+ * expression.
+ */
+ private def resolveOffset(unresolvedOffset: Offset): LogicalPlan = {
+ val resolvedChild = resolve(unresolvedOffset.child)
+
+ val resolvedOffsetExpr = expressionResolver.resolveLimitLikeExpression(
+ unresolvedOffset.offsetExpr,
+ unresolvedOffset.copy(child = resolvedChild)
+ )
+
+ Offset(resolvedOffsetExpr, resolvedChild)
+ }
+
+ /**
+ * Resolve [[Tail]]. We have to resolve its child and resolve and validate its limit
+ * expression.
+ */
+ private def resolveTail(unresolvedTail: Tail): LogicalPlan = {
+ val resolvedChild = resolve(unresolvedTail.child)
+
+ val resolvedTailExpr = expressionResolver.resolveLimitLikeExpression(
+ unresolvedTail.limitExpr,
+ unresolvedTail.copy(child = resolvedChild)
+ )
+
+ Tail(resolvedTailExpr, resolvedChild)
+ }
+
/**
* [[Distinct]] operator doesn't require any special resolution.
+ *
+ * Hidden output is reset when [[Distinct]] is reached during tree traversal.
*/
private def resolveDistinct(unresolvedDistinct: Distinct): LogicalPlan = {
- withResolvedChildren(unresolvedDistinct, resolve)
+ val resolvedDistinct = unresolvedDistinct.copy(child = resolve(unresolvedDistinct.child))
+ scopes.overwriteCurrent(hiddenOutput = Some(scopes.current.output))
+ resolvedDistinct
}
/**
@@ -400,50 +441,53 @@ class Resolver(
* - Resolve it further, usually using extensions, like [[DataSourceResolver]]
*/
private def resolveRelation(unresolvedRelation: UnresolvedRelation): LogicalPlan = {
- withPosition(unresolvedRelation) {
- viewResolver.withSourceUnresolvedRelation(unresolvedRelation) {
- val maybeResolvedRelation = cteRegistry.resolveCteName(unresolvedRelation.name).orElse {
- relationMetadataProvider.getRelationWithResolvedMetadata(unresolvedRelation)
- }
-
- val resolvedRelation = maybeResolvedRelation match {
- case Some(cteRelationDef: CTERelationDef) =>
- planLogger.logPlanResolutionEvent(cteRelationDef, "CTE definition resolved")
-
- SubqueryAlias(identifier = unresolvedRelation.name, child = cteRelationDef)
- case Some(relationsWithResolvedMetadata) =>
- planLogger.logPlanResolutionEvent(
- relationsWithResolvedMetadata,
- "Relation metadata retrieved"
- )
-
- relationsWithResolvedMetadata
- case None =>
- unresolvedRelation.tableNotFound(unresolvedRelation.multipartIdentifier)
- }
-
- resolve(resolvedRelation)
+ viewResolver.withSourceUnresolvedRelation(unresolvedRelation) {
+ val maybeResolvedRelation =
+ relationMetadataProvider.getRelationWithResolvedMetadata(unresolvedRelation)
+
+ val resolvedRelation = maybeResolvedRelation match {
+ case Some(relationsWithResolvedMetadata) =>
+ planLogger.logPlanResolutionEvent(
+ relationsWithResolvedMetadata,
+ "Relation metadata retrieved"
+ )
+
+ relationsWithResolvedMetadata
+ case None =>
+ unresolvedRelation.tableNotFound(unresolvedRelation.multipartIdentifier)
}
+
+ resolve(resolvedRelation)
}
}
/**
- * Resolve [[CTERelationDef]] by replacing it with [[CTERelationRef]] with the same ID so that
- * the Optimizer can make a decision whether to inline the definition or not.
- *
- * [[CTERelationDef.statsOpt]] is filled by the Optimizer.
+ * Resolve the [[UnresolvedCteRelationRef]] which was previously introduved by the
+ * [[IdentifierAndCteSubstitutor]].
*/
- private def resolveCteRelationDef(unresolvedCteRelationDef: CTERelationDef): LogicalPlan = {
- val cteRelationRef = CTERelationRef(
- cteId = unresolvedCteRelationDef.id,
- _resolved = true,
- isStreaming = unresolvedCteRelationDef.isStreaming,
- output = unresolvedCteRelationDef.output,
- recursive = false,
- maxRows = unresolvedCteRelationDef.maxRows
- )
+ private def resolveCteRelationRef(
+ unresolvedCteRelationRef: UnresolvedCteRelationRef): LogicalPlan = {
+ val cteRelationRef = cteRegistry.resolveCteName(unresolvedCteRelationRef.name) match {
+ case Some(cteRelationDef) =>
+ planLogger.logPlanResolutionEvent(cteRelationDef, "CTE definition resolved")
+
+ createCteRelationRef(
+ name = unresolvedCteRelationRef.name,
+ cteRelationDef = cteRelationDef
+ )
+ case None =>
+ unresolvedCteRelationRef.tableNotFound(Seq(unresolvedCteRelationRef.name))
+ }
+
+ resolve(cteRelationRef)
+ }
- handleLeafOperator(cteRelationRef)
+ /**
+ * We may meet resolved [[CTERelationRef]] while traversing partially resolved trees in DataFrame
+ * programs. In that case we simply recurse into the child plan.
+ */
+ private def handleResolvedCteRelationDef(cteRelationDef: CTERelationDef): LogicalPlan = {
+ cteRelationDef.copy(child = resolve(cteRelationDef.child))
}
/**
@@ -470,23 +514,7 @@ class Resolver(
val resolvedRelation = EvaluateUnresolvedInlineTable
.evaluateUnresolvedInlineTable(withResolvedExpressions)
- withPosition(unresolvedInlineTable) {
- resolve(resolvedRelation)
- }
- }
-
- /**
- * Preserve `PLAN_ID_TAG` which is used for DataFrame column resolution in Spark Connect.
- */
- private def preservePlanIdTag(
- unresolvedOperator: LogicalPlan,
- resolvedOperator: LogicalPlan): LogicalPlan = {
- unresolvedOperator.getTagValue(LogicalPlan.PLAN_ID_TAG) match {
- case Some(planIdTag) =>
- resolvedOperator.setTagValue(LogicalPlan.PLAN_ID_TAG, planIdTag)
- case None =>
- }
- resolvedOperator
+ resolve(resolvedRelation)
}
private def tryDelegateResolutionToExtension(
@@ -507,57 +535,70 @@ class Resolver(
* `leafOperator`'s output attribute IDs. We don't reassign expression IDs in the leftmost
* branch, see [[ExpressionIdAssigner]] class doc for more details.
* [[CTERelationRef]]'s output can always be reassigned.
- * - Overwrite the current [[NameScope]] with remapped output attributes. It's OK to call
+ * - Overwrite the current [[NameScope]] with remapped output attributes (both
+ * [[NameScope.output]] and [[NameScope.hiddenOutput]] are updated). It's OK to call
* `output` on a [[LeafNode]], because it's not recursive (this call fits the single-pass
* framework).
*/
private def handleLeafOperator(leafOperator: LeafNode): LogicalPlan = {
val leafOperatorWithAssignedExpressionIds = leafOperator match {
- case leafOperator
- if expressionIdAssigner.isLeftmostBranch && !leafOperator.isInstanceOf[CTERelationRef] =>
- expressionIdAssigner.createMapping(newOutput = leafOperator.output)
+ case leafOperator if expressionIdAssigner.shouldPreserveLeafOperatorIds(leafOperator) =>
+ expressionIdAssigner.createMappingForLeafOperator(newOperator = leafOperator)
+
leafOperator
- /**
- * [[InMemoryRelation.statsOfPlanToCache]] is mutable and does not get copied during normal
- * [[transformExpressionsUp]]. The easiest way to correctly copy it is via [[newInstance]]
- * call.
- *
- * We match [[MultiInstanceRelation]] to avoid a cyclic import between [[catalyst]] and
- * [[execution]].
- */
- case originalRelation: MultiInstanceRelation =>
- val newRelation = originalRelation.newInstance()
-
- expressionIdAssigner.createMapping(
- newOutput = newRelation.output,
- oldOutput = Some(originalRelation.output)
- )
+ case originalLeafOperator =>
+ val newLeafOperator = originalLeafOperator match {
- newRelation
- case _ =>
- expressionIdAssigner.createMapping()
-
- AnalysisHelper.allowInvokingTransformsInAnalyzer {
- leafOperator.transformExpressionsUp {
- case expression: NamedExpression =>
- val newExpression = expressionIdAssigner.mapExpression(expression)
- if (newExpression.eq(expression)) {
- throw SparkException.internalError(
- s"Leaf operator expression ID was not reassigned. Expression: $expression, " +
- s"leaf operator: $leafOperator"
- )
+ /**
+ * [[InMemoryRelation.statsOfPlanToCache]] is mutable and does not get copied during
+ * [[transformExpressionsUp]]. The easiest way to correctly copy it is via
+ * [[newInstance]] call.
+ *
+ * We match [[MultiInstanceRelation]] to avoid a cyclic import between [[catalyst]] and
+ * [[execution]].
+ */
+ case originalRelation: MultiInstanceRelation =>
+ originalRelation.newInstance().asInstanceOf[LeafNode]
+
+ case _ =>
+ AnalysisHelper
+ .allowInvokingTransformsInAnalyzer {
+ leafOperator.transformExpressions {
+ case attribute: Attribute => attribute.newInstance()
+ }
}
- newExpression
- }
+ .asInstanceOf[LeafNode]
}
+
+ expressionIdAssigner.createMappingForLeafOperator(
+ newOperator = newLeafOperator,
+ oldOperator = Some(originalLeafOperator)
+ )
+
+ newLeafOperator
}
- overwriteTopScope(leafOperator, leafOperatorWithAssignedExpressionIds.output)
+ val output = leafOperatorWithAssignedExpressionIds.output
+ scopes.overwriteCurrent(output = Some(output), hiddenOutput = Some(output))
leafOperatorWithAssignedExpressionIds
}
+ private def createCteRelationRef(name: String, cteRelationDef: CTERelationDef): LogicalPlan = {
+ SubqueryAlias(
+ identifier = name,
+ child = CTERelationRef(
+ cteId = cteRelationDef.id,
+ _resolved = true,
+ isStreaming = cteRelationDef.isStreaming,
+ output = cteRelationDef.output,
+ recursive = false,
+ maxRows = cteRelationDef.maxRows
+ )
+ )
+ }
+
/**
* Check if the unresolved operator is explicitly unsupported and throw
* [[ExplicitlyUnsupportedResolverFeature]] in that case. Otherwise, throw
@@ -578,15 +619,23 @@ class Resolver(
.withPosition(unresolvedOperator.origin)
}
- private def throwDatatypeMismatchFilterNotBoolean(filter: Filter): Nothing =
- throw new AnalysisException(
- errorClass = "DATATYPE_MISMATCH.FILTER_NOT_BOOLEAN",
- messageParameters = Map(
- "sqlExpr" -> makeCommaSeparatedExpressionString(filter.expressions),
- "filter" -> toSQLExpr(filter.condition),
- "type" -> toSQLType(filter.condition.dataType)
- )
- )
+ private def validateResolvedOperatorGenerically(resolvedOperator: LogicalPlan): Unit = {
+ if (!resolvedOperator.resolved) {
+ throwSinglePassFailedToResolveOperator(resolvedOperator)
+ }
+
+ if (resolvedOperator.children.nonEmpty) {
+ val missingInput = resolvedOperator.missingInput
+ if (missingInput.nonEmpty) {
+ throwMissingAttributesError(resolvedOperator, missingInput)
+ }
+ }
+
+ val invalidExpressions = expressionResolver.getLastInvalidExpressionsInTheContextOfOperator
+ if (invalidExpressions.nonEmpty) {
+ throwUnsupportedExprForOperator(invalidExpressions)
+ }
+ }
private def throwMissingAttributesError(
operator: LogicalPlan,
@@ -631,21 +680,27 @@ class Resolver(
summary = operator.origin.context.summary()
)
- private def makeCommaSeparatedExpressionString(expressions: Seq[Expression]): String = {
- expressions.map(toSQLExpr).mkString(", ")
+ private def throwUnsupportedExprForOperator(invalidExpressions: Seq[Expression]): Nothing = {
+ throw new AnalysisException(
+ errorClass = "UNSUPPORTED_EXPR_FOR_OPERATOR",
+ messageParameters = Map(
+ "invalidExprSqls" -> makeCommaSeparatedExpressionString(invalidExpressions)
+ )
+ )
}
- private def overwriteTopScope(
- sourceUnresolvedOperator: LogicalPlan,
- output: Seq[Attribute]): Unit = {
- withPosition(sourceUnresolvedOperator) {
- scopes.overwriteTop(output)
- }
+ private def makeCommaSeparatedExpressionString(expressions: Seq[Expression]): String = {
+ expressions.map(toSQLExpr).mkString(", ")
}
}
object Resolver {
+ /**
+ * Marks the operator as the top-most operator in a query or a view.
+ */
+ val TOP_LEVEL_OPERATOR = TreeNodeTag[Unit]("top_level_operator")
+
/**
* Create a new instance of the [[RelationResolution]].
*/
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala
index 833f50a5203d..238fed31741b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala
@@ -19,7 +19,11 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import java.util.Locale
-import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, SqlScriptingLocalVariableManager}
+import org.apache.spark.sql.catalyst.{
+ FunctionIdentifier,
+ SQLConfHelper,
+ SqlScriptingLocalVariableManager
+}
import org.apache.spark.sql.catalyst.analysis.{
FunctionRegistry,
GetViewColumnByNameAndOrdinal,
@@ -32,34 +36,10 @@ import org.apache.spark.sql.catalyst.analysis.{
UnresolvedStar,
UnresolvedSubqueryColumnAliases
}
-import org.apache.spark.sql.catalyst.expressions.{
- Alias,
- AttributeReference,
- BinaryArithmetic,
- Cast,
- ConditionalExpression,
- CreateNamedStruct,
- Expression,
- Literal,
- Predicate,
- SubqueryExpression,
- UpCast
-}
-import org.apache.spark.sql.catalyst.plans.logical.{
- Distinct,
- Filter,
- GlobalLimit,
- LocalLimit,
- LocalRelation,
- LogicalPlan,
- OneRowRelation,
- Project,
- SubqueryAlias,
- Union,
- UnresolvedWith,
- View
-}
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.connector.catalog.CatalogManager
+import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
import org.apache.spark.sql.internal.SQLConf.HiveCaseSensitiveInferenceMode
@@ -75,46 +55,93 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
/**
* Check the top level operator of the parsed operator.
*/
- def apply(operator: LogicalPlan): Boolean =
- checkConfValues() && checkVariables() && checkOperator(operator)
+ def apply(operator: LogicalPlan): Boolean = {
+ val unsupportedConf = detectUnsupportedConf()
+ if (unsupportedConf.isDefined) {
+ tryThrowUnsupportedSinglePassAnalyzerFeature(s"configuration: ${unsupportedConf.get}")
+ }
+
+ val areTempVariablesSupported = checkTempVariables()
+ if (!areTempVariablesSupported) {
+ tryThrowUnsupportedSinglePassAnalyzerFeature("temp variables")
+ }
+
+ val areScriptingVariablesSupported = checkScriptingVariables()
+ if (!areScriptingVariablesSupported) {
+ tryThrowUnsupportedSinglePassAnalyzerFeature("scripting variables")
+ }
+
+ !unsupportedConf.isDefined &&
+ areTempVariablesSupported &&
+ areScriptingVariablesSupported &&
+ checkOperator(operator)
+ }
/**
* Check if all the operators are supported. For implemented ones, recursively check
* their children. For unimplemented ones, return false.
*/
- private def checkOperator(operator: LogicalPlan): Boolean = operator match {
- case unresolvedWith: UnresolvedWith =>
- checkUnresolvedWith(unresolvedWith)
- case project: Project =>
- checkProject(project)
- case filter: Filter =>
- checkFilter(filter)
- case unresolvedSubqueryColumnAliases: UnresolvedSubqueryColumnAliases =>
- checkUnresolvedSubqueryColumnAliases(unresolvedSubqueryColumnAliases)
- case subqueryAlias: SubqueryAlias =>
- checkSubqueryAlias(subqueryAlias)
- case globalLimit: GlobalLimit =>
- checkGlobalLimit(globalLimit)
- case localLimit: LocalLimit =>
- checkLocalLimit(localLimit)
- case distinct: Distinct =>
- checkDistinct(distinct)
- case view: View =>
- checkView(view)
- case unresolvedRelation: UnresolvedRelation =>
- checkUnresolvedRelation(unresolvedRelation)
- case unresolvedInlineTable: UnresolvedInlineTable =>
- checkUnresolvedInlineTable(unresolvedInlineTable)
- case resolvedInlineTable: ResolvedInlineTable =>
- checkResolvedInlineTable(resolvedInlineTable)
- case localRelation: LocalRelation =>
- checkLocalRelation(localRelation)
- case oneRowRelation: OneRowRelation =>
- checkOneRowRelation(oneRowRelation)
- case union: Union =>
- checkUnion(union)
- case _ =>
- false
+ private def checkOperator(operator: LogicalPlan): Boolean = {
+ val isSupported = operator match {
+ case unresolvedWith: UnresolvedWith =>
+ checkUnresolvedWith(unresolvedWith)
+ case withCte: WithCTE =>
+ checkWithCte(withCte)
+ case project: Project =>
+ checkProject(project)
+ case aggregate: Aggregate =>
+ checkAggregate(aggregate)
+ case filter: Filter =>
+ checkFilter(filter)
+ case join: Join =>
+ checkJoin(join)
+ case unresolvedSubqueryColumnAliases: UnresolvedSubqueryColumnAliases =>
+ checkUnresolvedSubqueryColumnAliases(unresolvedSubqueryColumnAliases)
+ case subqueryAlias: SubqueryAlias =>
+ checkSubqueryAlias(subqueryAlias)
+ case globalLimit: GlobalLimit =>
+ checkGlobalLimit(globalLimit)
+ case localLimit: LocalLimit =>
+ checkLocalLimit(localLimit)
+ case offset: Offset =>
+ checkOffset(offset)
+ case tail: Tail =>
+ checkTail(tail)
+ case distinct: Distinct =>
+ checkDistinct(distinct)
+ case view: View =>
+ checkView(view)
+ case unresolvedRelation: UnresolvedRelation =>
+ checkUnresolvedRelation(unresolvedRelation)
+ case unresolvedInlineTable: UnresolvedInlineTable =>
+ checkUnresolvedInlineTable(unresolvedInlineTable)
+ case resolvedInlineTable: ResolvedInlineTable =>
+ checkResolvedInlineTable(resolvedInlineTable)
+ case localRelation: LocalRelation =>
+ checkLocalRelation(localRelation)
+ case range: Range =>
+ checkRange(range)
+ case oneRowRelation: OneRowRelation =>
+ checkOneRowRelation(oneRowRelation)
+ case cteRelationDef: CTERelationDef =>
+ checkCteRelationDef(cteRelationDef)
+ case cteRelationRef: CTERelationRef =>
+ checkCteRelationRef(cteRelationRef)
+ case union: Union =>
+ checkUnion(union)
+ case setOperation: SetOperation =>
+ checkSetOperation(setOperation)
+ case sort: Sort =>
+ checkSort(sort)
+ case _ =>
+ false
+ }
+
+ if (!isSupported) {
+ tryThrowUnsupportedSinglePassAnalyzerFeature(operator)
+ }
+
+ isSupported
}
/**
@@ -122,11 +149,9 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
* For LeafNode types, we return true or false. For other ones, check their children.
*/
private def checkExpression(expression: Expression): Boolean = {
- expression match {
+ val isSupported = expression match {
case alias: Alias =>
checkAlias(alias)
- case unresolvedBinaryArithmetic: BinaryArithmetic =>
- checkUnresolvedBinaryArithmetic(unresolvedBinaryArithmetic)
case unresolvedConditionalExpression: ConditionalExpression =>
checkUnresolvedConditionalExpression(unresolvedConditionalExpression)
case unresolvedCast: Cast =>
@@ -139,10 +164,16 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
checkUnresolvedAlias(unresolvedAlias)
case unresolvedAttribute: UnresolvedAttribute =>
checkUnresolvedAttribute(unresolvedAttribute)
- case unresolvedPredicate: Predicate =>
- checkUnresolvedPredicate(unresolvedPredicate)
case literal: Literal =>
checkLiteral(literal)
+ case unresolvedPredicate: Predicate =>
+ checkUnresolvedPredicate(unresolvedPredicate)
+ case scalarSubquery: ScalarSubquery =>
+ checkScalarSubquery(scalarSubquery)
+ case listQuery: ListQuery =>
+ checkListQuery(listQuery)
+ case outerReference: OuterReference =>
+ checkOuterReference(outerReference)
case attributeReference: AttributeReference =>
checkAttributeReference(attributeReference)
case createNamedStruct: CreateNamedStruct =>
@@ -151,9 +182,17 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
checkUnresolvedFunction(unresolvedFunction)
case getViewColumnByNameAndOrdinal: GetViewColumnByNameAndOrdinal =>
checkGetViewColumnBynameAndOrdinal(getViewColumnByNameAndOrdinal)
+ case expression if isGenerallySupportedExpression(expression) =>
+ expression.children.forall(checkExpression)
case _ =>
false
}
+
+ if (!isSupported) {
+ tryThrowUnsupportedSinglePassAnalyzerFeature(expression)
+ }
+
+ isSupported
}
private def checkUnresolvedWith(unresolvedWith: UnresolvedWith) = {
@@ -163,10 +202,29 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
} && checkOperator(unresolvedWith.child)
}
+ private def checkWithCte(withCte: WithCTE) = {
+ withCte.children.forall(checkOperator)
+ }
+
private def checkProject(project: Project) = {
checkOperator(project.child) && project.projectList.forall(checkExpression)
}
+ private def checkAggregate(aggregate: Aggregate) = {
+ checkOperator(aggregate.child) &&
+ aggregate.groupingExpressions.forall(checkExpression) &&
+ aggregate.aggregateExpressions.forall(checkExpression)
+ }
+
+ private def checkJoin(join: Join) = {
+ checkOperator(join.left) && checkOperator(join.right) && {
+ join.condition match {
+ case Some(condition) => checkExpression(condition)
+ case None => true
+ }
+ }
+ }
+
private def checkFilter(unresolvedFilter: Filter) =
checkOperator(unresolvedFilter.child) && checkExpression(unresolvedFilter.condition)
@@ -183,6 +241,12 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
private def checkLocalLimit(localLimit: LocalLimit) =
checkOperator(localLimit.child) && checkExpression(localLimit.limitExpr)
+ private def checkOffset(offset: Offset) =
+ checkOperator(offset.child) && checkExpression(offset.offsetExpr)
+
+ private def checkTail(tail: Tail) =
+ checkOperator(tail.child) && checkExpression(tail.limitExpr)
+
private def checkDistinct(distinct: Distinct) =
checkOperator(distinct.child)
@@ -201,16 +265,29 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
private def checkLocalRelation(localRelation: LocalRelation) =
localRelation.output.forall(checkExpression)
+ private def checkRange(range: Range) = true
+
private def checkUnion(union: Union) =
!union.byName && !union.allowMissingCol && union.children.forall(checkOperator)
+ private def checkSetOperation(setOperation: SetOperation) =
+ setOperation.children.forall(checkOperator)
+
+ private def checkSort(sort: Sort) = {
+ checkOperator(sort.child) && sort.order.forall(
+ sortOrder => checkExpression(sortOrder.child)
+ )
+ }
+
private def checkOneRowRelation(oneRowRelation: OneRowRelation) = true
- private def checkAlias(alias: Alias) = checkExpression(alias.child)
+ private def checkCteRelationDef(cteRelationDef: CTERelationDef) = {
+ checkOperator(cteRelationDef.child)
+ }
- private def checkUnresolvedBinaryArithmetic(unresolvedBinaryArithmetic: BinaryArithmetic) =
- checkExpression(unresolvedBinaryArithmetic.left) &&
- checkExpression(unresolvedBinaryArithmetic.right)
+ private def checkCteRelationRef(cteRelationRef: CTERelationRef) = true
+
+ private def checkAlias(alias: Alias) = checkExpression(alias.child)
private def checkUnresolvedConditionalExpression(
unresolvedConditionalExpression: ConditionalExpression) =
@@ -229,12 +306,13 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
!ResolverGuard.UNSUPPORTED_ATTRIBUTE_NAMES.contains(unresolvedAttribute.nameParts.head) &&
!unresolvedAttribute.getTagValue(LogicalPlan.PLAN_ID_TAG).isDefined
- private def checkUnresolvedPredicate(unresolvedPredicate: Predicate) = {
- unresolvedPredicate match {
- case _: SubqueryExpression => false
- case other =>
- other.children.forall(checkExpression)
- }
+ private def checkUnresolvedPredicate(unresolvedPredicate: Predicate) = unresolvedPredicate match {
+ case inSubquery: InSubquery =>
+ checkInSubquery(inSubquery)
+ case exists: Exists =>
+ checkExists(exists)
+ case _ =>
+ unresolvedPredicate.children.forall(checkExpression)
}
private def checkAttributeReference(attributeReference: AttributeReference) = true
@@ -253,22 +331,128 @@ class ResolverGuard(catalogManager: CatalogManager) extends SQLConfHelper {
private def checkLiteral(literal: Literal) = true
+ private def checkScalarSubquery(scalarSubquery: ScalarSubquery) =
+ checkOperator(scalarSubquery.plan)
+
+ private def checkInSubquery(inSubquery: InSubquery) =
+ inSubquery.values.forall(checkExpression) && checkExpression(inSubquery.query)
+
+ private def checkListQuery(listQuery: ListQuery) = checkOperator(listQuery.plan)
+
+ private def checkExists(exists: Exists) = checkOperator(exists.plan)
+
+ private def checkOuterReference(outerReference: OuterReference) =
+ checkExpression(outerReference.e)
+
private def checkGetViewColumnBynameAndOrdinal(
getViewColumnByNameAndOrdinal: GetViewColumnByNameAndOrdinal) = true
- private def checkConfValues() =
- // Case sensitive analysis is not supported.
- !conf.caseSensitiveAnalysis &&
- // Case-sensitive inference is not supported for Hive table schema.
- conf.caseSensitiveInferenceMode == HiveCaseSensitiveInferenceMode.NEVER_INFER &&
- // Legacy CTE resolution modes are not supported.
- !conf.getConf(SQLConf.LEGACY_INLINE_CTE_IN_COMMANDS) &&
- LegacyBehaviorPolicy.withName(conf.getConf(SQLConf.LEGACY_CTE_PRECEDENCE_POLICY)) ==
- LegacyBehaviorPolicy.CORRECTED
-
- private def checkVariables() =
- catalogManager.tempVariableManager.isEmpty &&
- SqlScriptingLocalVariableManager.get().forall(_.isEmpty)
+ /**
+ * Most of the expressions come from resolving the [[UnresolvedFunction]], but here we have some
+ * popular expressions allowlist for two reasons:
+ * 1. Some of them are allocated in the Parser;
+ * 2. To allow the resolution of resolved DataFrame subtrees.
+ */
+ private def isGenerallySupportedExpression(expression: Expression): Boolean = {
+ expression match {
+ // Math
+ case _: UnaryMinus | _: BinaryArithmetic | _: LeafMathExpression | _: UnaryMathExpression |
+ _: UnaryLogExpression | _: BinaryMathExpression | _: BitShiftOperation | _: RoundCeil |
+ _: Conv | _: RoundBase | _: Factorial | _: Bin | _: Hex | _: Unhex | _: WidthBucket =>
+ true
+ // Strings
+ case _: Collate | _: Collation | _: ResolvedCollation | _: UnresolvedCollation | _: Concat |
+ _: Mask | _: ConcatWs | _: Elt | _: Upper | _: Lower | _: BinaryPredicate |
+ _: StringPredicate | _: IsValidUTF8 | _: MakeValidUTF8 | _: ValidateUTF8 |
+ _: TryValidateUTF8 | _: StringReplace | _: Overlay | _: StringTranslate | _: FindInSet |
+ _: String2TrimExpression | _: StringTrimBoth | _: StringInstr | _: SubstringIndex |
+ _: StringLocate | _: StringLPad | _: BinaryPad | _: StringRPad | _: FormatString |
+ _: InitCap | _: StringRepeat | _: StringSpace | _: Substring | _: Right | _: Left |
+ _: Length | _: BitLength | _: OctetLength | _: Levenshtein | _: SoundEx | _: Ascii |
+ _: Chr | _: Base64 | _: UnBase64 | _: Decode | _: StringDecode | _: Encode | _: ToBinary |
+ _: FormatNumber | _: Sentences | _: StringSplitSQL | _: SplitPart | _: Empty2Null |
+ _: Luhncheck =>
+ true
+ // Datetime
+ case _: TimeZoneAwareExpression =>
+ true
+ // Decimal
+ case _: UnscaledValue | _: MakeDecimal | _: CheckOverflow | _: CheckOverflowInSum |
+ _: DecimalAddNoOverflowCheck |
+ _: DecimalDivideWithOverflowCheck =>
+ true
+ // Interval
+ case _: ExtractIntervalPart[_] | _: IntervalNumOperation | _: MultiplyInterval |
+ _: DivideInterval | _: TryMakeInterval | _: MakeInterval | _: MakeDTInterval |
+ _: MakeYMInterval | _: MultiplyYMInterval | _: MultiplyDTInterval | _: DivideYMInterval |
+ _: DivideDTInterval =>
+ true
+ // Number format
+ case _: ToNumber | _: TryToNumber | _: ToCharacter =>
+ true
+ // Random
+ case _: Rand | _: Randn | _: Uniform | _: RandStr =>
+ true
+ // Regexp
+ case _: Like | _: ILike | _: LikeAll | _: NotLikeAll | _: LikeAny | _: NotLikeAny | _: RLike |
+ _: StringSplit | _: RegExpReplace | _: RegExpExtract | _: RegExpExtractAll |
+ _: RegExpCount | _: RegExpSubStr | _: RegExpInStr =>
+ true
+ // JSON
+ case _: JsonToStructs | _: StructsToJson |
+ _: SchemaOfJson | _: JsonObjectKeys | _: LengthOfJsonArray =>
+ true
+ // CSV
+ case _: SchemaOfCsv | _: StructsToCsv | _: CsvToStructs =>
+ true
+ // URL
+ case _: TryParseUrl | _: ParseUrl | _: UrlEncode | _: UrlDecode | _: TryUrlDecode =>
+ true
+ // XML
+ case _: XmlToStructs | _: SchemaOfXml | _: StructsToXml =>
+ true
+ // Misc
+ case _: TaggingExpression =>
+ true
+ case _ =>
+ false
+ }
+ }
+
+ private def detectUnsupportedConf(): Option[String] = {
+ if (conf.caseSensitiveAnalysis) {
+ Some("caseSensitiveAnalysis")
+ } else if (conf.caseSensitiveInferenceMode != HiveCaseSensitiveInferenceMode.NEVER_INFER) {
+ Some("hiveCaseSensitiveInferenceMode")
+ } else if (conf.getConf(SQLConf.LEGACY_INLINE_CTE_IN_COMMANDS)) {
+ Some("legacyInlineCTEInCommands")
+ } else if (LegacyBehaviorPolicy.withName(conf.getConf(SQLConf.LEGACY_CTE_PRECEDENCE_POLICY)) !=
+ LegacyBehaviorPolicy.CORRECTED) {
+ Some("legacyCTEPrecedencePolicy")
+ } else {
+ None
+ }
+ }
+
+ private def checkTempVariables() =
+ catalogManager.tempVariableManager.isEmpty
+
+ private def checkScriptingVariables() =
+ SqlScriptingLocalVariableManager.get().forall(_.isEmpty)
+
+ private def tryThrowUnsupportedSinglePassAnalyzerFeature(operator: LogicalPlan): Unit = {
+ tryThrowUnsupportedSinglePassAnalyzerFeature(s"${operator.getClass} operator resolution")
+ }
+
+ private def tryThrowUnsupportedSinglePassAnalyzerFeature(expression: Expression): Unit = {
+ tryThrowUnsupportedSinglePassAnalyzerFeature(s"${expression.getClass} expression resolution")
+ }
+
+ private def tryThrowUnsupportedSinglePassAnalyzerFeature(feature: String): Unit = {
+ if (conf.getConf(SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_THROW_FROM_RESOLVER_GUARD)) {
+ throw QueryCompilationErrors.unsupportedSinglePassAnalyzerFeature(feature)
+ }
+ }
}
object ResolverGuard {
@@ -276,16 +460,12 @@ object ResolverGuard {
private val UNSUPPORTED_ATTRIBUTE_NAMES = {
val map = new IdentifierMap[Unit]()
- /**
- * Some SQL functions can be called without the braces and thus they are found in the
- * parsed operator as UnresolvedAttributes. This list contains the names of those functions
- * so we can reject them. Find more information in [[ColumnResolutionHelper.literalFunctions]].
- */
- map += ("current_date", ())
- map += ("current_timestamp", ())
+ // Not supported until we support their ''real'' function counterparts.
map += ("current_user", ())
map += ("user", ())
map += ("session_user", ())
+
+ // Not supported until we support GroupingSets/Cube/Rollup.
map += ("grouping__id", ())
/**
@@ -319,7 +499,6 @@ object ResolverGuard {
// Functions that are not resolved properly.
map += ("collate", ())
map += ("json_tuple", ())
- map += ("schema_of_unstructured_agg", ())
// Functions that produce wrong schemas/plans because of alias assignment.
map += ("from_json", ())
map += ("schema_of_json", ())
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverMetricTracker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverMetricTracker.scala
new file mode 100644
index 000000000000..4d6f4bfdc393
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverMetricTracker.scala
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.QueryPlanningTracker
+import org.apache.spark.sql.catalyst.rules.QueryExecutionMetering
+
+/**
+ * Trait for tracking and logging timing metrics for single-pass resolver.
+ */
+trait ResolverMetricTracker {
+
+ /**
+ * Log timing metrics for single-pass analyzer. In order to utilize existing logging
+ * infrastructure, we are going to log single-pass metrics as if single pass resolver was a
+ * standalone Catalyst rule. We log every run of single-pass resolver as effective.
+ */
+ protected def recordMetrics[R](tracker: QueryPlanningTracker)(body: => R): R =
+ QueryPlanningTracker.withTracker(tracker) {
+ val startTime = System.nanoTime()
+
+ val result = body
+
+ val runTime = System.nanoTime() - startTime
+
+ collectQueryExecutionMetrics(runTime)
+ tracker.recordRuleInvocation(
+ rule = ResolverMetricTracker.SINGLE_PASS_RESOLVER_METRIC_LOGGING_ALIAS,
+ timeNs = runTime,
+ effective = true
+ )
+
+ result
+ }
+
+ private def collectQueryExecutionMetrics(runTime: Long) = {
+ val queryExecutionMetrics = QueryExecutionMetering.INSTANCE
+
+ queryExecutionMetrics.incNumEffectiveExecution(
+ ResolverMetricTracker.SINGLE_PASS_RESOLVER_METRIC_LOGGING_ALIAS
+ )
+ queryExecutionMetrics.incTimeEffectiveExecutionBy(
+ ResolverMetricTracker.SINGLE_PASS_RESOLVER_METRIC_LOGGING_ALIAS,
+ runTime
+ )
+ queryExecutionMetrics.incNumExecution(
+ ResolverMetricTracker.SINGLE_PASS_RESOLVER_METRIC_LOGGING_ALIAS
+ )
+ queryExecutionMetrics.incExecutionTimeBy(
+ ResolverMetricTracker.SINGLE_PASS_RESOLVER_METRIC_LOGGING_ALIAS,
+ runTime
+ )
+ }
+}
+
+object ResolverMetricTracker {
+
+ /**
+ * Name under which single-pass resolver metrics will show up.
+ */
+ val SINGLE_PASS_RESOLVER_METRIC_LOGGING_ALIAS = "SinglePassResolver"
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverRunner.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverRunner.scala
index 99f7409d31fa..16f3a9f1444f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverRunner.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverRunner.scala
@@ -17,58 +17,65 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import org.apache.spark.sql.catalyst.SQLConfHelper
+import org.apache.spark.sql.catalyst.{QueryPlanningTracker, SQLConfHelper}
import org.apache.spark.sql.catalyst.analysis.{AnalysisContext, CleanupAliases}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.internal.SQLConf
/**
* Wrapper class for [[Resolver]] and single-pass resolution. This class encapsulates single-pass
- * resolution and post-processing of resolved plan. This post-processing is necessary in order to
+ * resolution, rewriting and validation of resolved plan. The plan rewrite is necessary in order to
* either fully resolve the plan or stay compatible with the fixed-point analyzer.
*/
class ResolverRunner(
resolver: Resolver,
extendedResolutionChecks: Seq[LogicalPlan => Unit] = Seq.empty
-) extends SQLConfHelper {
+) extends ResolverMetricTracker
+ with SQLConfHelper {
- private val resolutionPostProcessingExecutor = new RuleExecutor[LogicalPlan] {
- override def batches: Seq[Batch] = Seq(
- Batch("Post-process", Once, CleanupAliases)
- )
- }
+ /**
+ * Sequence of post-resolution rules that should be applied on the result of single-pass
+ * resolution.
+ */
+ private val planRewriteRules: Seq[Rule[LogicalPlan]] = Seq(
+ PruneMetadataColumns,
+ CleanupAliases
+ )
/**
- * Entry point for the resolver. This method performs following 3 steps:
- * - Resolves the plan in a bottom-up, single-pass manner.
- * - Validates the result of single-pass resolution.
- * - Applies necessary post-processing rules.
+ * `planRewriter` is used to rewrite the plan and the subqueries inside by applying
+ * `planRewriteRules`.
*/
- def resolve(
- plan: LogicalPlan,
- analyzerBridgeState: Option[AnalyzerBridgeState] = None): LogicalPlan = {
- AnalysisContext.withNewAnalysisContext {
- val resolvedPlan = resolver.lookupMetadataAndResolve(plan, analyzerBridgeState)
- if (conf.getConf(SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_VALIDATION_ENABLED)) {
- val validator = new ResolutionValidator
- validator.validatePlan(resolvedPlan)
- }
- finishResolution(resolvedPlan)
- }
- }
+ private val planRewriter = new PlanRewriter(planRewriteRules)
/**
- * This method performs necessary post-processing rules that aren't suitable for single-pass
- * resolver. We apply these rules after the single-pass has finished resolution to stay
- * compatible with fixed-point analyzer.
+ * Entry point for the resolver. This method performs following 4 steps:
+ * - Resolves the plan in a bottom-up using [[Resolver]], single-pass manner.
+ * - Rewrites the plan using rules configured in the [[planRewriter]].
+ * - Validates the final result internally using [[ResolutionValidator]].
+ * - Validates the final result using [[extendedResolutionChecks]].
*/
- private def finishResolution(plan: LogicalPlan): LogicalPlan = {
- val planWithPostProcessing = resolutionPostProcessingExecutor.execute(plan)
+ def resolve(
+ plan: LogicalPlan,
+ analyzerBridgeState: Option[AnalyzerBridgeState] = None,
+ tracker: QueryPlanningTracker = new QueryPlanningTracker): LogicalPlan =
+ recordMetrics(tracker) {
+ AnalysisContext.withNewAnalysisContext {
+ val resolvedPlan = resolver.lookupMetadataAndResolve(plan, analyzerBridgeState)
+
+ val rewrittenPlan = planRewriter.rewriteWithSubqueries(resolvedPlan)
- for (rule <- extendedResolutionChecks) {
- rule(planWithPostProcessing)
+ if (conf.getConf(SQLConf.ANALYZER_SINGLE_PASS_RESOLVER_VALIDATION_ENABLED)) {
+ val validator = new ResolutionValidator
+ validator.validatePlan(rewrittenPlan)
+ }
+
+ for (rule <- extendedResolutionChecks) {
+ rule(rewrittenPlan)
+ }
+
+ rewrittenPlan
+ }
}
- planWithPostProcessing
- }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
index c170941ce534..287164b80bbc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
@@ -17,17 +17,96 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import org.apache.spark.sql.catalyst.expressions.Expression
+import scala.collection.mutable
+
+import org.apache.spark.sql.catalyst.expressions.{
+ BinaryExpression,
+ Expression,
+ QuaternaryExpression,
+ TernaryExpression,
+ UnaryExpression
+}
trait ResolvesExpressionChildren {
+ /**
+ * Resolves [[UnaryExpression]] children and returns its copy with children resolved.
+ */
+ protected def withResolvedChildren(
+ unresolvedExpression: UnaryExpression,
+ resolveChild: Expression => Expression): Expression = {
+ val newChildren = Seq(resolveChild(unresolvedExpression.child))
+ unresolvedExpression.withNewChildren(newChildren)
+ }
+
+ /**
+ * Resolves [[BinaryExpression]] children and returns its copy with children resolved.
+ */
+ protected def withResolvedChildren(
+ unresolvedExpression: BinaryExpression,
+ resolveChild: Expression => Expression): Expression = {
+ val newChildren =
+ Seq(resolveChild(unresolvedExpression.left), resolveChild(unresolvedExpression.right))
+ unresolvedExpression.withNewChildren(newChildren)
+ }
+
+ /**
+ * Resolves [[TernaryExpression]] children and returns its copy with children resolved.
+ */
+ protected def withResolvedChildren(
+ unresolvedExpression: TernaryExpression,
+ resolveChild: Expression => Expression): Expression = {
+ val newChildren = Seq(
+ resolveChild(unresolvedExpression.first),
+ resolveChild(unresolvedExpression.second),
+ resolveChild(unresolvedExpression.third)
+ )
+ unresolvedExpression.withNewChildren(newChildren)
+ }
+
+ /**
+ * Resolves [[QuaternaryExpression]] children and returns its copy with children resolved.
+ */
+ protected def withResolvedChildren(
+ unresolvedExpression: QuaternaryExpression,
+ resolveChild: Expression => Expression): Expression = {
+ val newChildren = Seq(
+ resolveChild(unresolvedExpression.first),
+ resolveChild(unresolvedExpression.second),
+ resolveChild(unresolvedExpression.third),
+ resolveChild(unresolvedExpression.fourth)
+ )
+ unresolvedExpression.withNewChildren(newChildren)
+ }
+
/**
* Resolves generic [[Expression]] children and returns its copy with children resolved.
*/
- protected def withResolvedChildren[ExpressionType <: Expression](
- unresolvedExpression: ExpressionType,
- resolveChild: Expression => Expression): ExpressionType = {
- val newChildren = unresolvedExpression.children.map(resolveChild(_))
- unresolvedExpression.withNewChildren(newChildren).asInstanceOf[ExpressionType]
+ protected def withResolvedChildren(
+ unresolvedExpression: Expression,
+ resolveChild: Expression => Expression): Expression = unresolvedExpression match {
+ case unaryExpression: UnaryExpression =>
+ withResolvedChildren(unaryExpression, resolveChild)
+ case binaryExpression: BinaryExpression =>
+ withResolvedChildren(binaryExpression, resolveChild)
+ case ternaryExpression: TernaryExpression =>
+ withResolvedChildren(ternaryExpression, resolveChild)
+ case quaternaryExpression: QuaternaryExpression =>
+ withResolvedChildren(quaternaryExpression, resolveChild)
+ case _ =>
+ withResolvedChildrenImpl(unresolvedExpression, resolveChild)
+ }
+
+ private def withResolvedChildrenImpl(
+ unresolvedExpression: Expression,
+ resolveChild: Expression => Expression): Expression = {
+ val newChildren = new mutable.ArrayBuffer[Expression](unresolvedExpression.children.size)
+
+ val childrenIterator = unresolvedExpression.children.iterator
+ while (childrenIterator.hasNext) {
+ newChildren += resolveChild(childrenIterator.next())
+ }
+
+ unresolvedExpression.withNewChildren(newChildren.toSeq)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesNameByHiddenOutput.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesNameByHiddenOutput.scala
new file mode 100644
index 000000000000..3d9f964a7b89
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesNameByHiddenOutput.scala
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Project}
+import org.apache.spark.sql.catalyst.util._
+
+/**
+ * [[ResolvesNameByHiddenOutput]] is used by resolvers for operators that are able to resolve
+ * attributes in its expression tree from hidden output or that can reference expressions not
+ * present in child's output. Update child operator's output list and place a [[Project]] node on
+ * top of original operator node with the original output of an operator's child.
+ *
+ * For example, in a following query:
+ *
+ * {{{
+ * SELECT
+ * t1.key
+ * FROM
+ * t1 FULL OUTER JOIN t2 USING (key)
+ * WHERE
+ * t1.key NOT LIKE 'bb.%';
+ * }}}
+ *
+ * Plan without adding missing attributes would be:
+ *
+ * {{{
+ * +- Project [key#1]
+ * +- Filter NOT key#1 LIKE bb.%
+ * +- Project [coalesce(key#1, key#2) AS key#3, __key#1__, __key#2__]
+ * +- Join FullOuter, (key#1 = key#2)
+ * :- SubqueryAlias t1
+ * : +- Relation t1[key#1]
+ * +- SubqueryAlias t2
+ * +- Relation t2[key#2]
+ * }}}
+ *
+ * NOTE: __#key1__ and __key#2__ at the end of inner [[Project]] are metadata columns from the
+ * full outer join. Even though they are present in the [[Project]] in single-pass, fixed-point
+ * adds these columns after resolving missing input, so duplication of some columns is possible.
+ * In order to stay fully compatible between single-pass and fixed-point, we add both missing
+ * attributes and these metadata columns. We mimic fixed-point behavior by putting metadata
+ * columns in [[NameScope.hiddenOutput]] instead of [[NameScope.output]].
+ *
+ * In the plan above, [[Filter]] requires key#1 in its condition, but key#1 is __not__ available
+ * in the below [[Project]]'s output, even though key#1 is available in [[Join]]'s hidden output.
+ * Because of that, we need to place key#1 in the project list, after original project list
+ * expressions, but before metadata columns (to remain compatible with fixed-point). In order to
+ * preserve initial output of [[Filter]], we place a [[Project]] node on top of this [[Filter]],
+ * whose project list is the original output of the [[Project]] __below__ [[Filter]] (in this
+ * case - key#3 and metadata columns key#1 and key#2).
+ *
+ * Therefore, the plan becomes:
+ *
+ * {{{
+ * +- Project [key#1]
+ * +- Project [key#3, key#1, key#2]
+ * +- Filter NOT key#1 LIKE bb.%
+ * +- Project [coalesce(key#1, key#2) AS key#3, key#1, key#1, key#2]
+ * +- Join FullOuter, (key#1 = key#2)
+ * :- SubqueryAlias t1
+ * : +- Relation t1[key#1]
+ * +- SubqueryAlias t2
+ * +- Relation t2[key#2]
+ * }}}
+ *
+ * Query below exhibits similar behavior when [[Sort]] operator resolves an attribute using hidden
+ * output:
+ *
+ * {{{ SELECT col1 FROM VALUES (1, 2) ORDER BY col2; }}}
+ *
+ * Unresolved plan would be:
+ *
+ * {{{
+ * Sort [col2 ASC NULLS FIRST], true
+ * +- Project [col1]
+ * +- LocalRelation [col1, col2]
+ * }}}
+ *
+ * As it can be seen, attribute `col2` used in [[Sort]] can't be resolved using the [[Project]]
+ * output (which is [`col1`]), so it has to be resolved using the hidden output (which is
+ * propagated from [[LocalRelation]] and is [`col1`, `col2`]). As it's been shown in the previous
+ * example, `col2` has to be added to [[Project]] list and a [[Project]] with original output of
+ * the [[Project]] below [[Sort]] is added as a top node. Because of that, analyzed plan is:
+ *
+ * {{{
+ * Project [col1]
+ * +- Sort [col2 ASC NULLS FIRST], true
+ * +- Project [col1, col2]
+ * +- LocalRelation [col1, col2]
+ * }}}
+ *
+ * Another example is when [[Sort]] order expression is an [[AggregateExpression]] which is not
+ * present in the [[Aggregate.aggregateExpressions]]:
+ *
+ * {{{
+ * SELECT col1 FROM VALUES (1) GROUP BY col1 ORDER BY sum(col1);
+ * }}}
+ *
+ * In this example `sum(col1)` should be added to child's output and a [[Project]] node should be
+ * added on top of the [[Sort]] node to preserve the original output of the [[Aggregate]] node:
+ *
+ * Project [col1]
+ * +- Sort [sum(col1)#... ASC NULLS FIRST], true
+ * +- Aggregate [col1], [col1, sum(col1) AS sum(col1)#...]
+ * +- LocalRelation [col1]
+ */
+trait ResolvesNameByHiddenOutput {
+ protected val scopes: NameScopeStack
+
+ /**
+ * If the child of an operator is a [[Project]] or an [[Aggregate]] and that operator has missing
+ * expressions, insert the missing expressions in the output list of the operator.
+ * In order to stay compatible with fixed-point, missing expressions are inserted after the
+ * original output list, but before any qualified access only columns that have been added as
+ * part of resolution from hidden output.
+ */
+ def insertMissingExpressions(
+ operator: LogicalPlan,
+ missingExpressions: Seq[NamedExpression]): LogicalPlan =
+ operator match {
+ case operator @ (_: Project | _: Aggregate) if missingExpressions.nonEmpty =>
+ expandOperatorsOutputList(operator, missingExpressions)
+ case other => other
+ }
+
+ private def expandOperatorsOutputList(
+ operator: LogicalPlan,
+ missingExpressions: Seq[NamedExpression]): LogicalPlan = {
+ val (metadataCols, nonMetadataCols) = operator match {
+ case project: Project =>
+ project.projectList.partition(_.toAttribute.qualifiedAccessOnly)
+ case aggregate: Aggregate =>
+ aggregate.aggregateExpressions.partition(_.toAttribute.qualifiedAccessOnly)
+ }
+
+ val newOutputList = nonMetadataCols ++ missingExpressions ++ metadataCols
+ val newOperator = operator match {
+ case aggregate: Aggregate =>
+ aggregate.copy(aggregateExpressions = newOutputList)
+ case project: Project =>
+ project.copy(projectList = newOutputList)
+ }
+
+ newOperator
+ }
+
+ /**
+ * If [[missingExpressions]] is not empty, output of an operator has been changed by
+ * [[insertMissingExpressions]]. Therefore, we need to restore the original output, by placing a
+ * [[Project]] on top of an original node, with original's node output. Additionally, we append
+ * all qualified access only columns from hidden output, because they may be needed in upper
+ * operators (if not, they will be pruned away in [[PruneMetadataColumns]]).
+ */
+ def retainOriginalOutput(
+ operator: LogicalPlan,
+ missingExpressions: Seq[NamedExpression]): LogicalPlan = {
+ if (missingExpressions.isEmpty) {
+ operator
+ } else {
+ val project = Project(
+ scopes.current.output.map(_.asInstanceOf[NamedExpression]) ++ scopes.current.hiddenOutput
+ .filter(_.qualifiedAccessOnly),
+ operator
+ )
+ scopes.overwriteCurrent(output = Some(project.projectList.map(_.toAttribute)))
+ project
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SemanticComparator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SemanticComparator.scala
new file mode 100644
index 000000000000..6ec34cf8511e
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SemanticComparator.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.{ArrayList, HashMap}
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+ * [[SemanticComparator]] is a tool to compare expressions semantically to a predefined sequence
+ * of `targetExpressions`. Semantic comparison is based on [[QueryPlan.canonicalized]] - for
+ * example, `col1 + 1 + col2` is semantically equal to `1 + col2 + col1`. To speed up slow tree
+ * traversals and expression node field comparisons, we cache the semantic hashes (which is
+ * simply a hash of a canonicalized subtree) and use them for O(1) indexing. If the hashes don't
+ * match, we perform an early return. Otherwise, we invoke the heavy [[Expression.semanticEquals]]
+ * method to make sure that expression trees are indeed identical.
+ */
+class SemanticComparator(targetExpressions: Seq[Expression]) {
+ private val targetExpressionsBySemanticHash =
+ new HashMap[Int, ArrayList[Expression]](targetExpressions.size)
+
+ for (targetExpression <- targetExpressions) {
+ targetExpressionsBySemanticHash
+ .computeIfAbsent(targetExpression.semanticHash(), _ => new ArrayList[Expression])
+ .add(targetExpression)
+ }
+
+ /**
+ * Returns the first expression in `targetExpressions` that is semantically equal to the given
+ * `expression`. If no such expression is found, returns `None`.
+ */
+ def collectFirst(expression: Expression): Option[Expression] = {
+ targetExpressionsBySemanticHash.get(expression.semanticHash()) match {
+ case null =>
+ None
+ case targetExpressions =>
+ val iter = targetExpressions.iterator
+ var matchedExpression: Option[Expression] = None
+ while (iter.hasNext && matchedExpression.isEmpty) {
+ val element = iter.next
+ if (element.semanticEquals(expression)) {
+ matchedExpression = Some(element)
+ }
+ }
+ matchedExpression
+ }
+ }
+
+ /**
+ * Use the previously constructed `targetExpressionsBySemanticHash` to check if the given
+ * `expression` is semantically equal to any of the target expressions.
+ */
+ def exists(expression: Expression): Boolean = {
+ collectFirst(expression).nonEmpty
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SetOperationLikeResolver.scala
similarity index 51%
rename from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnionResolver.scala
rename to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SetOperationLikeResolver.scala
index cfd81114b030..216cdc679fa9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SetOperationLikeResolver.scala
@@ -19,26 +19,32 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import java.util.HashSet
-import org.apache.spark.sql.catalyst.analysis.{
- withPosition,
- AnsiTypeCoercion,
- TypeCoercion,
- TypeCoercionBase
+import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.analysis.{AnsiTypeCoercion, TypeCoercion, TypeCoercionBase}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, ExprId}
+import org.apache.spark.sql.catalyst.plans.logical.{
+ Except,
+ Intersect,
+ LogicalPlan,
+ Project,
+ SetOperation,
+ Union
}
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, ExprId}
-import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project, Union}
+import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.{DataType, MetadataBuilder}
+import org.apache.spark.sql.types.{DataType, MapType, MetadataBuilder, VariantType}
/**
- * The [[UnionResolver]] performs [[Union]] operator resolution. This operator has 2+
- * children. Resolution involves checking and normalizing child output attributes
- * (data types and nullability).
+ * The [[SetOperationLikeResolver]] performs [[Union]], [[Intersect]] or [[Except]] operator
+ * resolution. These operators have 2+ children. Resolution involves checking and normalizing child
+ * output attributes (data types and nullability).
*/
-class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
- extends TreeNodeResolver[Union, Union] {
+class SetOperationLikeResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
+ extends TreeNodeResolver[LogicalPlan, LogicalPlan] {
private val expressionIdAssigner = expressionResolver.getExpressionIdAssigner
+ private val autoGeneratedAliasProvider = new AutoGeneratedAliasProvider(expressionIdAssigner)
private val scopes = resolver.getNameScopes
+ private val cteRegistry = resolver.getCteRegistry
private val typeCoercion: TypeCoercionBase =
if (conf.ansiEnabled) {
AnsiTypeCoercion
@@ -47,11 +53,9 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
}
/**
- * Resolve the [[Union]] operator:
- * - Retrieve old output and child outputs if the operator is already resolved. This is relevant
- * for partially resolved subtrees from DataFrame programs.
+ * Resolve the [[Union]], [[Intersect]] or [[Except]] operators:
* - Resolve each child in the context of a) New [[NameScope]] b) New [[ExpressionIdAssigner]]
- * mapping. Collect child outputs to coerce them later.
+ * mapping c) CTE scope. Collect child outputs to coerce them later.
* - Perform individual output deduplication to handle the distinct union case described in
* [[performIndividualOutputExpressionIdDeduplication]] scaladoc.
* - Validate that child outputs have same length or throw "NUM_COLUMNS_MISMATCH" otherwise.
@@ -59,54 +63,74 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
* [[typeCoercion.findWiderTypeForTwo]] or throw "INCOMPATIBLE_COLUMN_TYPE" if coercion fails.
* - Add [[Project]] with [[Cast]] on children needing attribute data type widening.
* - Assert that coerced outputs don't have conflicting expression IDs.
- * - Merge transformed outputs: For each column, merge child attributes' types using
- * [[StructType.unionLikeMerge]]. Mark column as nullable if any child attribute is.
+ * - Merge transformed outputs using a separate logic for each operator type.
* - Store merged output in current [[NameScope]].
* - Create a new mapping in [[ExpressionIdAssigner]] using the coerced and validated outputs.
- * - Return the resolved [[Union]] with new children.
+ * - Return the resolved operator with new children optionally wrapped in [[WithCTE]]. See
+ * [[CteScope]] scaladoc for more info.
*/
- override def resolve(unresolvedUnion: Union): Union = {
- val oldOutput = if (unresolvedUnion.resolved) {
- Some(unresolvedUnion.output)
- } else {
- None
- }
+ override def resolve(unresolvedOperator: LogicalPlan): LogicalPlan = {
+ val (resolvedChildren, childOutputs) = resolveChildren(unresolvedOperator)
- val (resolvedChildren, childOutputs) = unresolvedUnion.children.zipWithIndex.map {
- case (unresolvedChild, childIndex) =>
- scopes.withNewScope {
- expressionIdAssigner.withNewMapping(isLeftmostChild = (childIndex == 0)) {
- val resolvedChild = resolver.resolve(unresolvedChild)
- (resolvedChild, scopes.top.output)
- }
- }
- }.unzip
+ expressionIdAssigner.createMappingFromChildMappings()
val (deduplicatedChildren, deduplicatedChildOutputs) =
performIndividualOutputExpressionIdDeduplication(
resolvedChildren,
- childOutputs
+ childOutputs,
+ unresolvedOperator
)
- val (newChildren, newChildOutputs) = if (needToCoerceChildOutputs(deduplicatedChildOutputs)) {
- coerceChildOutputs(
- deduplicatedChildren,
- deduplicatedChildOutputs,
- validateAndDeduceTypes(unresolvedUnion, deduplicatedChildOutputs)
- )
- } else {
- (deduplicatedChildren, deduplicatedChildOutputs)
- }
+ val (newChildren, newChildOutputs) =
+ if (needToCoerceChildOutputs(deduplicatedChildOutputs, unresolvedOperator)) {
+ coerceChildOutputs(
+ deduplicatedChildren,
+ deduplicatedChildOutputs,
+ validateAndDeduceTypes(unresolvedOperator, deduplicatedChildOutputs)
+ )
+ } else {
+ (deduplicatedChildren, deduplicatedChildOutputs)
+ }
ExpressionIdAssigner.assertOutputsHaveNoConflictingExpressionIds(newChildOutputs)
- withPosition(unresolvedUnion) {
- scopes.overwriteTop(Union.mergeChildOutputs(newChildOutputs))
- }
+ val output = mergeChildOutputs(unresolvedOperator, newChildOutputs)
+ scopes.overwriteCurrent(output = Some(output), hiddenOutput = Some(output))
+
+ validateOutputs(unresolvedOperator, output)
- expressionIdAssigner.createMapping(scopes.top.output, oldOutput)
+ val resolvedOperator = unresolvedOperator.withNewChildren(newChildren)
+
+ cteRegistry.currentScope.tryPutWithCTE(
+ unresolvedOperator = unresolvedOperator,
+ resolvedOperator = resolvedOperator
+ )
+ }
- unresolvedUnion.copy(children = newChildren)
+ /**
+ * Resolve `unresolvedSetOperationLike`'s children in the context of new [[NameScope]],
+ * [[ExpressionIdAssigner]] mapping and [[CteScope]].
+ *
+ * [[ExpressionIdAssigner]] child mapping is collected just or the left child, because that's
+ * the only child whose expression IDs get propagated upwards through [[Union]], [[Intersect]] or
+ * [[Except]]. This is an optimization to avoid fast-growing expression ID mappings.
+ */
+ private def resolveChildren(
+ unresolvedOperator: LogicalPlan): (Seq[LogicalPlan], Seq[Seq[Attribute]]) = {
+ unresolvedOperator.children.zipWithIndex.map {
+ case (unresolvedChild, childIndex) =>
+ scopes.withNewScope() {
+ expressionIdAssigner.withNewMapping(collectChildMapping = childIndex == 0) {
+ cteRegistry.withNewScopeUnderMultiChildOperator(
+ unresolvedOperator = unresolvedOperator,
+ unresolvedChild = unresolvedChild
+ ) {
+ val resolvedChild = resolver.resolve(unresolvedChild)
+ (resolvedChild, scopes.current.output)
+ }
+ }
+ }
+ }.unzip
}
/**
@@ -138,6 +162,17 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
* See SPARK-37865 for more details.
*/
private def performIndividualOutputExpressionIdDeduplication(
+ children: Seq[LogicalPlan],
+ childOutputs: Seq[Seq[Attribute]],
+ unresolvedOperator: LogicalPlan
+ ): (Seq[LogicalPlan], Seq[Seq[Attribute]]) = {
+ unresolvedOperator match {
+ case _: Union => doPerformIndividualOutputExpressionIdDeduplication(children, childOutputs)
+ case _ => (children, childOutputs)
+ }
+ }
+
+ private def doPerformIndividualOutputExpressionIdDeduplication(
children: Seq[LogicalPlan],
childOutputs: Seq[Seq[Attribute]]
): (Seq[LogicalPlan], Seq[Seq[Attribute]]) = {
@@ -156,7 +191,11 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
.withMetadata(attribute.metadata)
.putNull("__is_duplicate")
.build()
- Alias(attribute, attribute.name)(explicitMetadata = Some(newMetadata))
+ autoGeneratedAliasProvider.newAlias(
+ child = attribute,
+ name = Some(attribute.name),
+ explicitMetadata = Some(newMetadata)
+ )
} else {
expressionIds.add(attribute.exprId)
@@ -178,31 +217,54 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
* - Output length differs between children. We will throw an appropriate error later during type
* coercion with more diagnostics.
* - Output data types differ between children. We don't care about nullability for type coercion,
- * it will be correctly assigned later by [[Union.mergeChildOutputs]].
+ * it will be correctly assigned later by [[SetOperationLikeResolver.mergeChildOutputs]].
*/
- private def needToCoerceChildOutputs(childOutputs: Seq[Seq[Attribute]]): Boolean = {
+ private def needToCoerceChildOutputs(
+ childOutputs: Seq[Seq[Attribute]],
+ unresolvedOperator: LogicalPlan): Boolean = {
val firstChildOutput = childOutputs.head
childOutputs.tail.exists { childOutput =>
childOutput.length != firstChildOutput.length ||
childOutput.zip(firstChildOutput).exists {
case (lhsAttribute, rhsAttribute) =>
- !DataType.equalsStructurally(
+ !areDataTypesCompatibleInTheContextOfOperator(
+ unresolvedOperator,
lhsAttribute.dataType,
- rhsAttribute.dataType,
- ignoreNullability = true
+ rhsAttribute.dataType
)
}
}
}
+ /**
+ * This method returns whether types are compatible in the context of the specified operator.
+ *
+ * In fixed-point we only use [[DataType.equalsStructurally]] for [[Union]] type coercion. For
+ * [[Except]] and [[Intersect]] we use [[DataTypeUtils.sameType]]. This method ensures we perform
+ * the check for whether coercion is needed in the compatible way to the fixed-point.
+ */
+ private def areDataTypesCompatibleInTheContextOfOperator(
+ unresolvedPlan: LogicalPlan,
+ lhs: DataType,
+ rhs: DataType): Boolean = {
+ unresolvedPlan match {
+ case _: Union => DataType.equalsStructurally(lhs, rhs, ignoreNullability = true)
+ case _: Except | _: Intersect => DataTypeUtils.sameType(lhs, rhs)
+ case other =>
+ throw SparkException.internalError(
+ s"Set operation resolver should not be used for ${other.nodeName}"
+ )
+ }
+ }
+
/**
* Returns a sequence of data types representing the widened data types for each column:
- * - Validates that the number of columns in each child of the `Union` operator are equal.
+ * - Validates that the number of columns in each child of the set operator is equal.
* - Validates that the data types of columns can be widened to a common type.
* - Deduces the widened data types for each column.
*/
private def validateAndDeduceTypes(
- unresolvedUnion: Union,
+ unresolvedOperator: LogicalPlan,
childOutputs: Seq[Seq[Attribute]]): Seq[DataType] = {
val childDataTypes = childOutputs.map(attributes => attributes.map(attr => attr.dataType))
@@ -215,7 +277,7 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
expectedNumColumns,
childColumnTypes,
childIndex,
- unresolvedUnion
+ unresolvedOperator
)
}
@@ -223,7 +285,7 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
case ((widenedColumnType, columnTypeForCurrentRow), columnIndex) =>
typeCoercion.findWiderTypeForTwo(widenedColumnType, columnTypeForCurrentRow).getOrElse {
throwIncompatibleColumnTypeError(
- unresolvedUnion,
+ unresolvedOperator,
columnIndex,
childIndex + 1,
widenedColumnType,
@@ -259,10 +321,10 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
attribute
} else {
outputChanged = true
- Alias(
- Cast(attribute, widenedType, Some(conf.sessionLocalTimeZone)),
- attribute.name
- )()
+ autoGeneratedAliasProvider.newAlias(
+ child = Cast(attribute, widenedType, Some(conf.sessionLocalTimeZone)),
+ name = Some(attribute.name)
+ )
}
}
@@ -275,34 +337,95 @@ class UnionResolver(resolver: Resolver, expressionResolver: ExpressionResolver)
.unzip
}
+ /**
+ * Helper method to call appropriate object method [[mergeChildOutputs]] for each operator.
+ */
+ private def mergeChildOutputs(
+ unresolvedPlan: LogicalPlan,
+ childOutputs: Seq[Seq[Attribute]]): Seq[Attribute] = {
+ unresolvedPlan match {
+ case _: Union => Union.mergeChildOutputs(childOutputs)
+ case _: Except => Except.mergeChildOutputs(childOutputs)
+ case _: Intersect => Intersect.mergeChildOutputs(childOutputs)
+ case other =>
+ throw SparkException.internalError(
+ s"Set operation resolver should not be used for ${other.nodeName}"
+ )
+ }
+ }
+
+ /**
+ * Validate outputs of [[SetOperation]].
+ * - [[MapType]] and [[VariantType]] are currently not supported for [[SetOperations]] and we need
+ * to throw a relevant user-facing error.
+ */
+ private def validateOutputs(unresolvedPlan: LogicalPlan, output: Seq[Attribute]): Unit = {
+ unresolvedPlan match {
+ case _: SetOperation =>
+ output.find(a => hasMapType(a.dataType)).foreach { mapCol =>
+ throwUnsupportedSetOperationOnMapType(mapCol, unresolvedPlan)
+ }
+ output.find(a => hasVariantType(a.dataType)).foreach { variantCol =>
+ throwUnsupportedSetOperationOnVariantType(variantCol, unresolvedPlan)
+ }
+ case _ =>
+ }
+ }
+
+ private def throwUnsupportedSetOperationOnMapType(
+ mapCol: Attribute,
+ unresolvedPlan: LogicalPlan): Unit = {
+ throw QueryCompilationErrors.unsupportedSetOperationOnMapType(
+ mapCol = mapCol,
+ origin = unresolvedPlan.origin
+ )
+ }
+
+ private def throwUnsupportedSetOperationOnVariantType(
+ variantCol: Attribute,
+ unresolvedPlan: LogicalPlan): Unit = {
+ throw QueryCompilationErrors.unsupportedSetOperationOnVariantType(
+ variantCol = variantCol,
+ origin = unresolvedPlan.origin
+ )
+ }
+
private def throwNumColumnsMismatch(
expectedNumColumns: Int,
childColumnTypes: Seq[DataType],
columnIndex: Int,
- unresolvedUnion: Union): Unit = {
+ unresolvedOperator: LogicalPlan): Unit = {
throw QueryCompilationErrors.numColumnsMismatch(
- "UNION",
+ unresolvedOperator.nodeName.toUpperCase(),
expectedNumColumns,
columnIndex + 1,
childColumnTypes.length,
- unresolvedUnion.origin
+ unresolvedOperator.origin
)
}
private def throwIncompatibleColumnTypeError(
- unresolvedUnion: Union,
+ unresolvedOperator: LogicalPlan,
columnIndex: Int,
childIndex: Int,
widenedColumnType: DataType,
columnTypeForCurrentRow: DataType): Nothing = {
throw QueryCompilationErrors.incompatibleColumnTypeError(
- "UNION",
+ unresolvedOperator.nodeName.toUpperCase(),
columnIndex,
childIndex + 1,
widenedColumnType,
columnTypeForCurrentRow,
hint = "",
- origin = unresolvedUnion.origin
+ origin = unresolvedOperator.origin
)
}
+
+ private def hasMapType(dt: DataType): Boolean = {
+ dt.existsRecursively(_.isInstanceOf[MapType])
+ }
+
+ private def hasVariantType(dt: DataType): Boolean = {
+ dt.existsRecursively(_.isInstanceOf[VariantType])
+ }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala
new file mode 100644
index 000000000000..53c7b3f2366c
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SortResolver.scala
@@ -0,0 +1,397 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.{HashMap, IdentityHashMap, LinkedHashMap}
+
+import scala.collection.mutable
+import scala.jdk.CollectionConverters._
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.analysis.{
+ NondeterministicExpressionCollection,
+ UnresolvedAttribute
+}
+import org.apache.spark.sql.catalyst.expressions.{
+ Alias,
+ Attribute,
+ Expression,
+ ExprId,
+ NamedExpression,
+ SortOrder
+}
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan, Project, Sort}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+/**
+ * Resolves a [[Sort]] by resolving its child and order expressions.
+ */
+class SortResolver(operatorResolver: Resolver, expressionResolver: ExpressionResolver)
+ extends TreeNodeResolver[Sort, LogicalPlan]
+ with ResolvesNameByHiddenOutput {
+ override protected val scopes: NameScopeStack = operatorResolver.getNameScopes
+ private val autoGeneratedAliasProvider = new AutoGeneratedAliasProvider(
+ expressionResolver.getExpressionIdAssigner
+ )
+
+ /**
+ * Resolve [[Sort]] operator.
+ *
+ * 1. Resolve [[Sort.child]].
+ * 2. Resolve order expressions using [[ExpressionResolver.resolveExpressionTreeInOperator]].
+ * 3. In case order expressions contain only one element, `UnresolvedAttribute(ALL)`, which
+ * can't be resolved from current (nor from hidden output) - this is done using
+ * [[ResolveAsAllKeyword]], skip previous step and resolve it as an `ALL` keyword (by copying
+ * child's output and transforming it into attributes).
+ * 4. In case there are attributes referenced in the order expressions are resolved using
+ * the hidden output (or in case we order by [[AggregateExpression]]s which are not present
+ * in [[Aggregate.aggregateExpressions]]) update the output of child operator and add a
+ * [[Project]] as a parent of [[Sort]] with original [[Project]]'s output (this is done by
+ * [[ResolvesNameByHiddenOutput]]). Query:
+ * {{{
+ * SELECT col1 FROM VALUES(1, 2) WHERE col2 > 2 ORDER BY col2;
+ * }}}
+ * Has the following unresolved plan:
+ *
+ * 'Sort ['col2 ASC NULLS FIRST], true
+ * +- 'Project ['col1]
+ * +- 'Filter ('col2 > 2)
+ * +- LocalRelation [col1#92225, col2#92226]
+ *
+ * Because `col2` from the [[Sort]] node is resolved using the hidden output, add it to the
+ * [[Project.projectList]] and add a [[Project]] with original project list as a top node:
+ *
+ * Project [col1]
+ * +- Sort [col2 ASC NULLS FIRST], true
+ * +- Project [col1, col2]
+ * +- Filter (col2 > 2)
+ * +- LocalRelation [col1, col2]
+ *
+ * Another example with ordering by [[AggregateExpression]]:
+ * {{{
+ * SELECT col1 FROM VALUES (1, 2) GROUP BY col1, col2 + 1 ORDER BY SUM(col1), col2 + 1;
+ * }}}
+ * Has the following unresolved plan:
+ *
+ * 'Sort ['SUM('col1) ASC NULLS FIRST, ('col2 + 1) ASC NULLS FIRST], true
+ * +- 'Aggregate ['col1, ('col2 + 1)], ['col1]
+ * +- LocalRelation [col1, col2]
+ *
+ * Because neither `SUM(col1)` nor `col2 + 1` from the [[Sort]] node are present in the
+ * [[Aggregate.aggregateExpressions]], add them to it and add a [[Project]] with original
+ * project list as a top node (`SUM(col2)` is extracted in the
+ * [[AggregateExpressionResolver]] whereas `col2 + 1` is extracted using
+ * `extractReferencedGroupingAndAggregateExpressions` helper method):
+ *
+ * Project [col1]
+ * +- Sort [sum(col1)#... ASC NULLS FIRST, (col2 + 1)#... ASC NULLS FIRST], true
+ * +- Aggregate [col1, (col2 + 1)],
+ * [col1, sum(col1) AS sum(col1)#..., (col2 + 1) AS (col2 + 1)#...]
+ * +- LocalRelation [col1, col2]
+ * 5. In case there are non-deterministic expressions in the order expressions, substitute them
+ * with derived attribute references to an artificial [[Project]] list.
+ */
+ override def resolve(unresolvedSort: Sort): LogicalPlan = {
+ val resolvedChild = operatorResolver.resolve(unresolvedSort.child)
+ if (canOrderByAll(unresolvedSort.order)) {
+ val sortOrder = unresolvedSort.order.head
+ val resolvedOrder =
+ scopes.current.output.map(a => sortOrder.copy(child = a.toAttribute))
+ unresolvedSort.copy(child = resolvedChild, order = resolvedOrder)
+ } else {
+ val unresolvedSortWithResolvedChild = unresolvedSort.copy(child = resolvedChild)
+
+ val (resolvedOrderExpressions, missingAttributes, aggregateExpressionsAliased) =
+ resolveOrderExpressions(
+ unresolvedSortWithResolvedChild,
+ scopes.current.output.toArray
+ )
+
+ val (finalOrderExpressions, missingExpressions) = resolvedChild match {
+ case aggregate: Aggregate =>
+ val (cleanedOrderExpressions, referencedGroupingExpressions) =
+ extractReferencedGroupingAndAggregateExpressions(aggregate, resolvedOrderExpressions)
+ (cleanedOrderExpressions, aggregateExpressionsAliased ++ referencedGroupingExpressions)
+ case other =>
+ (resolvedOrderExpressions, missingAttributes)
+ }
+
+ val resolvedChildWithMissingAttributes =
+ insertMissingExpressions(resolvedChild, missingExpressions)
+
+ val resolvedSort = unresolvedSort.copy(
+ child = resolvedChildWithMissingAttributes,
+ order = finalOrderExpressions
+ )
+
+ val sortWithOriginalOutput = retainOriginalOutput(resolvedSort, missingExpressions)
+
+ sortWithOriginalOutput match {
+ case project: Project =>
+ sortWithOriginalOutput
+ case sort: Sort =>
+ tryPullOutNondeterministic(sort)
+ }
+ }
+ }
+
+ /**
+ * Resolve order expressions of an unresolved [[Sort]], returns attributes resolved using hidden
+ * output and extracted [[AggregateExpression]]s. In case of [[UnresolvedAttribute]] resolution,
+ * respect the following order.
+ *
+ * 1. Attribute can be resolved using the current scope:
+ * {{{
+ * -- This one will resolve `col1` from the current scope (its value will be 1)
+ * SELECT col1 FROM VALUES(1, 2) WHERE (SELECT col1 FROM VALUES(3)) ORDER BY col1;
+ * }}}
+ *
+ * 2. Attribute can be resolved using the hidden output. Attribute is added to
+ * `missingAttributes` which is used to update the plan using the
+ * [[ResolvesNameByHiddenOutput]].
+ * {{{
+ * -- This one will resolve `col2` from hidden output
+ * SELECT col1 FROM VALUES(1, 2) WHERE col2 > 2 ORDER BY col2;
+ * }}}
+ *
+ * 3. In case attribute can't be resolved from output nor from hidden output, throw
+ * `UNRESOLVED_COLUMN` exception:
+ * {{{
+ * -- Following queries throw `UNRESOLVED_COLUMN` exception:
+ * SELECT col1 FROM VALUES(1,2) GROUP BY col1 HAVING col1 > 1 ORDER BY col2;
+ * SELECT col1 FROM VALUES(1) ORDER BY col2;
+ * }}}
+ */
+ private def resolveOrderExpressions(
+ unresolvedSort: Sort,
+ projectListArray: Array[NamedExpression]): (Seq[SortOrder], Seq[Attribute], Seq[Alias]) = {
+ val orderByOrdinal = conf.orderByOrdinal
+ val referencedAttributes = new HashMap[ExprId, Attribute]
+ val aggregateExpressionsAliased = new mutable.ArrayBuffer[Alias]
+
+ val resolvedSortOrder = unresolvedSort.order.map { sortOrder =>
+ val partiallyResolvedSortOrderChild =
+ expressionResolver.resolveExpressionTreeInOperator(sortOrder.child, unresolvedSort)
+
+ referencedAttributes.putAll(expressionResolver.getLastReferencedAttributes)
+ aggregateExpressionsAliased ++= expressionResolver.getLastExtractedAggregateExpressionAliases
+
+ val resolvedSortOrderChild = tryReplaceOrdinalsInSortOrderChild(
+ partiallyResolvedSortOrderChild,
+ orderByOrdinal,
+ projectListArray,
+ unresolvedSort
+ )
+
+ sortOrder.copy(child = resolvedSortOrderChild)
+ }
+
+ val missingAttributes = scopes.current.resolveMissingAttributesByHiddenOutput(
+ referencedAttributes
+ )
+
+ (resolvedSortOrder, missingAttributes, aggregateExpressionsAliased.toSeq)
+ }
+
+ /**
+ * Replaces the ordinals with the actual expressions from the resolved project list or throws if
+ * ordinal value is greater than project list size. Do this only in case `conf.orderByOrdinal`
+ * value is `true`.
+ *
+ * Example 1:
+ * {{{
+ * -- This one would order by `col1` and `col2 + 1`
+ * SELECT col1, col2 + 1 FROM VALUES(1, 2) ORDER BY col1, 2;
+ * }}}
+ *
+ * Example 2:
+ * {{{
+ * -- This one would throw `ORDER_BY_POS_OUT_OF_RANGE` error
+ * SELECT col1 FROM VALUES(1, 2) ORDER BY 2;
+ * }}}
+ */
+ private def tryReplaceOrdinalsInSortOrderChild(
+ partiallyResolvedSortOrderChild: Expression,
+ orderByOrdinal: Boolean,
+ projectListArray: Array[NamedExpression],
+ unresolvedSort: Sort): Expression = {
+ if (orderByOrdinal) {
+ TryExtractOrdinal(partiallyResolvedSortOrderChild) match {
+ case Some(ordinal) =>
+ if (ordinal > projectListArray.length) {
+ throw QueryCompilationErrors.orderByPositionRangeError(
+ ordinal,
+ projectListArray.length,
+ unresolvedSort
+ )
+ }
+ projectListArray(ordinal - 1).toAttribute
+ case None => partiallyResolvedSortOrderChild
+ }
+ } else {
+ partiallyResolvedSortOrderChild
+ }
+ }
+
+ /**
+ * Extracts the referenced grouping and aggregate expressions from the order expressions. This is
+ * used to update the output of the child operator and add a [[Project]] as a parent of [[Sort]]
+ * later during the resolution (if needed). Consider the following example:
+ * {{{
+ * SELECT col1 FROM VALUES (1, 2) GROUP BY col1, col2 ORDER BY col2;
+ * }}}
+ *
+ * The unresolved plan would look like this:
+ *
+ * 'Sort ['col2 ASC NULLS FIRST], true
+ * +- 'Aggregate ['col1, 'col2], ['col1]
+ * +- LocalRelation [col1, col2]
+ *
+ * As it can be seen, `col2` (ordering expression) is not present in the [[Aggregate]] operator
+ * , and thus we return it from this method. The plan will be altered later during the resolution
+ * using the [[ResolvesNameByHiddenOutput]] (`col2` will be added to
+ * [[Aggregate.aggregateExpressions]], [[Project]] will be added as a top node with original
+ * [[Aggregate]] output) and it will look like:
+ *
+ * Project [col1]
+ * +- Sort [col2 ASC NULLS FIRST], true
+ * +- Aggregate [col1, col2], [col1, col2]
+ * +- LocalRelation [col1, col2]
+ */
+ private def extractReferencedGroupingAndAggregateExpressions(
+ aggregate: Aggregate,
+ sortOrderEntries: Seq[SortOrder]): (Seq[SortOrder], Seq[NamedExpression]) = {
+ val aliasChildToAliasInAggregateExpressions = new IdentityHashMap[Expression, Alias]
+ val aggregateExpressionsSemanticComparator = new SemanticComparator(
+ aggregate.aggregateExpressions.map {
+ case alias: Alias =>
+ aliasChildToAliasInAggregateExpressions.put(alias.child, alias)
+ alias.child
+ case other => other
+ }
+ )
+
+ val groupingExpressionsSemanticComparator = new SemanticComparator(
+ aggregate.groupingExpressions
+ )
+
+ val referencedGroupingExpressions = new mutable.ArrayBuffer[NamedExpression]
+ val transformedSortOrderEntries = sortOrderEntries.map { sortOrder =>
+ sortOrder.copy(child = sortOrder.child.transform {
+ case expression: Expression =>
+ extractReferencedGroupingAndAggregateExpressionsFromOrderExpression(
+ expression = expression,
+ aggregateExpressionsSemanticComparator = aggregateExpressionsSemanticComparator,
+ groupingExpressionsSemanticComparator = groupingExpressionsSemanticComparator,
+ aliasChildToAliasInAggregateExpressions = aliasChildToAliasInAggregateExpressions,
+ referencedGroupingExpressions = referencedGroupingExpressions
+ )
+ })
+ }
+ (transformedSortOrderEntries, referencedGroupingExpressions.toSeq)
+ }
+
+ private def extractReferencedGroupingAndAggregateExpressionsFromOrderExpression(
+ expression: Expression,
+ aggregateExpressionsSemanticComparator: SemanticComparator,
+ groupingExpressionsSemanticComparator: SemanticComparator,
+ aliasChildToAliasInAggregateExpressions: IdentityHashMap[Expression, Alias],
+ referencedGroupingExpressions: mutable.ArrayBuffer[NamedExpression]): Expression = {
+ aggregateExpressionsSemanticComparator.collectFirst(expression) match {
+ case Some(attribute: Attribute)
+ if !aliasChildToAliasInAggregateExpressions.containsKey(attribute) =>
+ attribute
+ case Some(expression) =>
+ aliasChildToAliasInAggregateExpressions.get(expression) match {
+ case null =>
+ throw SparkException.internalError(
+ s"No parent alias for expression $expression while extracting aggregate" +
+ s"expressions in Sort operator."
+ )
+ case alias: Alias =>
+ alias.toAttribute
+ }
+ case None if groupingExpressionsSemanticComparator.exists(expression) =>
+ expression match {
+ case attribute: Attribute =>
+ referencedGroupingExpressions += attribute
+ attribute
+ case other =>
+ val alias = autoGeneratedAliasProvider.newAlias(child = other)
+ referencedGroupingExpressions += alias
+ alias.toAttribute
+ }
+ case None => expression
+ }
+ }
+
+ /**
+ * In case there are non-deterministic expressions in `order` expressions replace them with
+ * attributes created out of corresponding non-deterministic expression. Example:
+ *
+ * {{{ SELECT 1 ORDER BY RAND(); }}}
+ *
+ * This query would have the following analyzed plan:
+ *
+ * Project [1]
+ * +- Sort [_nondeterministic ASC NULLS FIRST], true
+ * +- Project [1, rand(...) AS _nondeterministic#...]
+ * +- Project [1 AS 1#...]
+ * +- OneRowRelation
+ */
+ private def tryPullOutNondeterministic(sort: Sort): LogicalPlan = {
+ val nondeterministicToAttributes: LinkedHashMap[Expression, NamedExpression] =
+ NondeterministicExpressionCollection.getNondeterministicToAttributes(
+ sort.order.map(_.child)
+ )
+
+ if (!nondeterministicToAttributes.isEmpty) {
+ val newChild = Project(
+ scopes.current.output ++ nondeterministicToAttributes.values.asScala.toSeq,
+ sort.child
+ )
+ val resolvedOrder = sort.order.map { sortOrder =>
+ sortOrder.copy(
+ child = PullOutNondeterministicExpressionInExpressionTree(
+ sortOrder.child,
+ nondeterministicToAttributes
+ )
+ )
+ }
+ val resolvedSort = sort.copy(
+ order = resolvedOrder,
+ child = newChild
+ )
+ Project(projectList = scopes.current.output, child = resolvedSort)
+ } else {
+ sort
+ }
+ }
+
+ private def canOrderByAll(expressions: Seq[SortOrder]): Boolean = {
+ val isOrderByAll = expressions match {
+ case Seq(SortOrder(unresolvedAttribute: UnresolvedAttribute, _, _, _)) =>
+ unresolvedAttribute.equalsIgnoreCase("ALL")
+ case _ => false
+ }
+ isOrderByAll && scopes.current
+ .resolveMultipartName(Seq("ALL"), canResolveNameByHiddenOutput = true)
+ .candidates
+ .isEmpty
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala
new file mode 100644
index 000000000000..c36024e7269e
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryExpressionResolver.scala
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.analysis.{UnresolvedInlineTable, ValidateSubqueryExpression}
+import org.apache.spark.sql.catalyst.expressions.{
+ AttributeReference,
+ Exists,
+ Expression,
+ InSubquery,
+ ListQuery,
+ ScalarSubquery,
+ SubExprUtils,
+ SubqueryExpression
+}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+/**
+ * [[SubqueryExpressionResolver]] resolves specific [[SubqueryExpression]]s, such as
+ * [[ScalarSubquery]], [[ListQuery]], and [[Exists]].
+ */
+class SubqueryExpressionResolver(expressionResolver: ExpressionResolver, resolver: Resolver) {
+ private val scopes = resolver.getNameScopes
+ private val traversals = expressionResolver.getExpressionTreeTraversals
+ private val cteRegistry = resolver.getCteRegistry
+ private val subqueryRegistry = resolver.getSubqueryRegistry
+ private val expressionIdAssigner = expressionResolver.getExpressionIdAssigner
+ private val typeCoercionResolver = expressionResolver.getGenericTypeCoercionResolver
+
+ /**
+ * Resolve [[ScalarSubquery]]:
+ * - Resolve the subquery plan;
+ * - Get outer references;
+ * - Type coerce it;
+ * - Validate it.
+ */
+ def resolveScalarSubquery(unresolvedScalarSubquery: ScalarSubquery): Expression = {
+ traversals.current.parentOperator match {
+ case unresolvedInlineTable: UnresolvedInlineTable =>
+ throw QueryCompilationErrors.inlineTableContainsScalarSubquery(unresolvedInlineTable)
+ case _ =>
+ }
+
+ val resolvedSubqueryExpressionPlan = resolveSubqueryExpressionPlan(
+ unresolvedScalarSubquery.plan
+ )
+
+ val resolvedScalarSubquery = unresolvedScalarSubquery.copy(
+ plan = resolvedSubqueryExpressionPlan.plan,
+ outerAttrs = resolvedSubqueryExpressionPlan.outerExpressions
+ )
+
+ val coercedScalarSubquery =
+ typeCoercionResolver.resolve(resolvedScalarSubquery).asInstanceOf[ScalarSubquery]
+
+ validateSubqueryExpression(coercedScalarSubquery)
+
+ coercedScalarSubquery
+ }
+
+ /**
+ * Resolve [[InSubquery]]:
+ * - Resolve the underlying [[ListQuery]];
+ * - Resolve the values;
+ * - Type coerce it;
+ * - Validate it.
+ */
+ def resolveInSubquery(unresolvedInSubquery: InSubquery): Expression = {
+ val resolvedQuery =
+ expressionResolver.resolve(unresolvedInSubquery.query).asInstanceOf[ListQuery]
+
+ val resolvedValues = unresolvedInSubquery.values.map { value =>
+ expressionResolver.resolve(value)
+ }
+
+ val resolvedInSubquery =
+ unresolvedInSubquery.copy(values = resolvedValues, query = resolvedQuery)
+
+ val coercedInSubquery =
+ typeCoercionResolver.resolve(resolvedInSubquery).asInstanceOf[InSubquery]
+
+ validateSubqueryExpression(coercedInSubquery.query)
+
+ coercedInSubquery
+ }
+
+ /**
+ * Resolve [[ListSubquery]], which is always a child of the [[InSubquery]].
+ */
+ def resolveListQuery(unresolvedListQuery: ListQuery): Expression = {
+ val resolvedSubqueryExpressionPlan = resolveSubqueryExpressionPlan(unresolvedListQuery.plan)
+
+ unresolvedListQuery.copy(
+ plan = resolvedSubqueryExpressionPlan.plan,
+ outerAttrs = resolvedSubqueryExpressionPlan.outerExpressions,
+ numCols = resolvedSubqueryExpressionPlan.output.size
+ )
+ }
+
+ /**
+ * Resolve [[Exists]] subquery:
+ * - Resolve the subquery plan;
+ * - Get outer references;
+ * - Type coerce it;
+ * - Validate it.
+ */
+ def resolveExists(unresolvedExists: Exists): Expression = {
+ val resolvedSubqueryExpressionPlan = resolveSubqueryExpressionPlan(unresolvedExists.plan)
+
+ val resolvedExists = unresolvedExists.copy(
+ plan = resolvedSubqueryExpressionPlan.plan,
+ outerAttrs = resolvedSubqueryExpressionPlan.outerExpressions
+ )
+
+ val coercedExists = typeCoercionResolver.resolve(resolvedExists).asInstanceOf[Exists]
+
+ validateSubqueryExpression(coercedExists)
+
+ coercedExists
+ }
+
+ /**
+ * Resolve [[SubqueryExpression]] plan. Subquery expressions require:
+ * - Fresh [[NameScope]] to isolate the name resolution;
+ * - Fresh [[ExpressionIdAssigner]] mapping, because it's a separate plan branch;
+ * - Fresh [[CteScope]] with the root flag set so that [[CTERelationDefs]] are merged
+ * under the subquery root.
+ * - Fresh [[SubqueryRegistry]] scope to isolate the subquery expression plan resolution.
+ */
+ private def resolveSubqueryExpressionPlan(
+ unresolvedSubqueryPlan: LogicalPlan): ResolvedSubqueryExpressionPlan = {
+ val resolvedSubqueryExpressionPlan = scopes.withNewScope(isSubqueryRoot = true) {
+ expressionIdAssigner.withNewMapping(isSubqueryRoot = true) {
+ cteRegistry.withNewScope(isRoot = true) {
+ subqueryRegistry.withNewScope() {
+ val resolvedPlan = resolver.resolve(unresolvedSubqueryPlan)
+
+ ResolvedSubqueryExpressionPlan(
+ plan = resolvedPlan,
+ output = scopes.current.output,
+ outerExpressions = getOuterExpressions(resolvedPlan)
+ )
+ }
+ }
+ }
+ }
+
+ for (expression <- resolvedSubqueryExpressionPlan.outerExpressions) {
+ expressionResolver.validateExpressionUnderSupportedOperator(expression)
+ }
+
+ resolvedSubqueryExpressionPlan
+ }
+
+ /**
+ * Get outer expressions from the subquery plan. These are the expressions that are actual
+ * [[AttributeReference]]s or [[AggregateExpression]]s with outer references in the subtree.
+ * [[AggregateExpressionResolver]] strips out the outer aggregate expression subtrees, but
+ * for [[SubqueryExpression.outerAttrs]] to be well-formed, we need to put those back. After
+ * that we validate the outer expressions.
+ *
+ * We reuse [[SubExprUtils.getOuterReferences]] for the top-down traversal to stay compatible
+ * with the fixed-point Analyzer, because the order of [[SubqueryExpression.outerAttrs]] is a
+ * part of an implicit alias name for that expression.
+ */
+ private def getOuterExpressions(subQuery: LogicalPlan): Seq[Expression] = {
+ val subqueryScope = subqueryRegistry.currentScope
+
+ SubExprUtils.getOuterReferences(subQuery).map {
+ case attribute: AttributeReference =>
+ subqueryScope.getOuterAggregateExpression(attribute.exprId).getOrElse(attribute)
+ case other =>
+ other
+ }
+ }
+
+ /**
+ * Generically validates [[SubqueryExpression]]. Should not be done in the main pass, because the
+ * logic is too sophisticated.
+ */
+ private def validateSubqueryExpression(subqueryExpression: SubqueryExpression): Unit = {
+ ValidateSubqueryExpression(traversals.current.parentOperator, subqueryExpression)
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala
new file mode 100644
index 000000000000..9b7517781843
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/SubqueryScope.scala
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import java.util.{ArrayDeque, ArrayList, HashMap}
+
+import scala.jdk.CollectionConverters._
+
+import org.apache.spark.sql.catalyst.expressions.{Alias, ExprId}
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+
+/**
+ * The [[SubqueryScope]] is managed through the whole resolution process of a given
+ * [[SubqueryExpression]] plan.
+ *
+ * The reason why we need this scope is that [[AggregateExpression]]s with [[OuterReference]]s are
+ * handled in a special way. Consider this query:
+ *
+ * {{{
+ * -- t1.col2 is an outer reference
+ * SELECT col1 FROM VALUES (1, 2) t1 GROUP BY col1 HAVING (
+ * SELECT * FROM VALUES (1, 2) t2 WHERE t2.col2 == MAX(t1.col2)
+ * )
+ * }}}
+ *
+ * During the [[Exists]] resolution inside the HAVING clause we encounter "t1.col2" name, which is
+ * resolved to an [[OuterReference]]. There's an [[AggregateExpression]] on top of it. This whole
+ * expression is not local to the subquery, and thus it belongs to an outer [[Aggregate]]
+ * operator below the `HAVING` clause. We need top pull it up outside of the subquery, and insert
+ * it in the [[Aggregate]] operator. So the resolution order is as follows:
+ * - Resolve "t1.col2" to an [[OuterReference]] in [[ExpressionResolver.resolveAttribute]];
+ * - Resolve the [[AggregateExpression]] in [[AggregateExpressionResolver.resolve]];
+ * - Detect an outer reference below the aggregate expression, cut the whole subtree with outer
+ * references stripped away, alias it and insert it in this [[SubqueryScope]];
+ * - Replace the aggregate expression with an [[OuterReference]] to the [[AttributeReference]] from
+ * that artificial [[Alias]];
+ * - When the resolution of the [[SubqueryExpression]] is finished,
+ * [[SubqueryRegistry.withNewScope]] merges the lower scope to the upper one, and all the
+ * outer aggregate expression references are appended to the common
+ * [[lowerAliasedOuterAggregateExpressions]] list.
+ * - Finally, the resolution of the `HAVING` clause can insert the missing aggregate expression
+ * into the lower [[Aggregate]] operator. During this process we must call
+ * [[ExpressionIdAssigner.mapExpression]] on the new alias, because this auto-generated alias
+ * is new to the query plan, so that [[ExpressionIdAssigner]] remembers it.
+ *
+ * Notes:
+ * - Spark only supports outer aggregates in the subqueries inside `HAVING`;
+ * - The subtree under a given [[AggregateExpression]] can be arbitrary, but must contain either
+ * local or outer references, the mixed set is disallowed.
+ * - We can have several subquery expressions in HAVING clause, that's why we append outer
+ * aggregate expressions from lower scopes in [[mergeChildScope]].
+ *
+ * @param isOuterAggregateAllowed Whether outer aggregate expressions are allowed in this scope.
+ * Currenly Spark only supports those in HAVING.
+ */
+class SubqueryScope(val isOuterAggregateAllowed: Boolean = false) {
+ private val outerAliasedAggregateExpressions = new ArrayList[Alias]
+ private val outerAliasedAggregateExpressionById = new HashMap[ExprId, AggregateExpression]
+ private val lowerAliasedOuterAggregateExpressions = new ArrayList[Alias]
+
+ /**
+ * Add an outer `aggregateExpression` to this scope. The `alias` is auto-generated and will be
+ * later inserted into the outer [[Aggregate]] operator.
+ */
+ def addOuterAggregateExpression(alias: Alias, aggregateExpression: AggregateExpression): Unit = {
+ outerAliasedAggregateExpressions.add(alias)
+ outerAliasedAggregateExpressionById.put(alias.exprId, aggregateExpression)
+ }
+
+ /**
+ * Get the outer `aggregateExpression` by its `aliasId`. This is used in the
+ * [[SubqueryExpressionResolver]] to replace the collected outer references with the original
+ * [[AggregateExpression]] subtrees, which is required for a well-formed
+ * [[SubqueryExpression.outerAttrs]].
+ */
+ def getOuterAggregateExpression(aliasId: ExprId): Option[AggregateExpression] = {
+ Option(outerAliasedAggregateExpressionById.get(aliasId))
+ }
+
+ /**
+ * Get the outer aggregate expression aliased from the lower subquery scope.
+ */
+ def getLowerOuterAggregateExpressionAliases: Seq[Alias] = {
+ lowerAliasedOuterAggregateExpressions.asScala.toSeq
+ }
+
+ /**
+ * Merge `childScope` by extending our `lowerAliasedOuterAggregateExpressions` with
+ * `childScope.outerAliasedAggregateExpressions`.
+ */
+ def mergeChildScope(childScope: SubqueryScope): Unit = {
+ lowerAliasedOuterAggregateExpressions.addAll(childScope.outerAliasedAggregateExpressions)
+ }
+}
+
+/**
+ * The [[SubqueryRegistry]] manages the stack of [[SubqueryScope]]s during the resolution of
+ * the whole SQL query. Every new [[SubqueryScope]] has its own isolated scope.
+ */
+class SubqueryRegistry {
+ private val stack = new ArrayDeque[SubqueryScope]
+ stack.push(new SubqueryScope)
+
+ /**
+ * Get the current [[SubqueryScope]].
+ */
+ def currentScope: SubqueryScope = stack.peek()
+
+ /**
+ * A RAII-wrapper for pushing/popping scopes. This is used by the [[SubqueryExpressionResolver]]
+ * to create a new scope for each [[SubqueryExpression]].
+ */
+ def withNewScope[R](isOuterAggregateAllowed: Boolean = false)(body: => R): R = {
+ stack.push(new SubqueryScope(isOuterAggregateAllowed = isOuterAggregateAllowed))
+ try {
+ body
+ } finally {
+ val childScope = stack.pop()
+ currentScope.mergeChildScope(childScope)
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimeAddResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimeAddResolver.scala
index 74b101273aaa..70d69daee5cb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimeAddResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimeAddResolver.scala
@@ -31,8 +31,8 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, TimeAdd}
class TimeAddResolver(
expressionResolver: ExpressionResolver,
timezoneAwareExpressionResolver: TimezoneAwareExpressionResolver)
- extends TreeNodeResolver[TimeAdd, Expression]
- with ResolvesExpressionChildren {
+ extends TreeNodeResolver[TimeAdd, Expression]
+ with ResolvesExpressionChildren {
private val typeCoercionTransformations: Seq[Expression => Expression] =
if (conf.ansiEnabled) {
@@ -44,8 +44,8 @@ class TimeAddResolver(
new TypeCoercionResolver(timezoneAwareExpressionResolver, typeCoercionTransformations)
override def resolve(unresolvedTimeAdd: TimeAdd): Expression = {
- val timeAddWithResolvedChildren: TimeAdd =
- withResolvedChildren(unresolvedTimeAdd, expressionResolver.resolve)
+ val timeAddWithResolvedChildren =
+ withResolvedChildren(unresolvedTimeAdd, expressionResolver.resolve _)
val timeAddWithTypeCoercion: Expression = typeCoercionResolver
.resolve(timeAddWithResolvedChildren)
timezoneAwareExpressionResolver.withResolvedTimezone(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolver.scala
index 5ba08c0c3edb..539202c06a8a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolver.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import org.apache.spark.sql.catalyst.expressions.{Expression, TimeZoneAwareExpression}
+import org.apache.spark.sql.catalyst.expressions.{Cast, Expression, TimeZoneAwareExpression}
/**
* Resolves [[TimeZoneAwareExpressions]] by applying the session's local timezone.
@@ -32,17 +32,24 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, TimeZoneAwareExpre
class TimezoneAwareExpressionResolver(expressionResolver: TreeNodeResolver[Expression, Expression])
extends TreeNodeResolver[TimeZoneAwareExpression, Expression]
with ResolvesExpressionChildren {
+ val typeCoercionResolver = new TypeCoercionResolver(this)
/**
* Resolves a [[TimeZoneAwareExpression]] by resolving its children and applying a timezone.
+ * If the resolved expression is a [[Cast]], type coercion is not needed. Otherwise we apply
+ * [[TypeCoercionResolver]] to the resolved expression.
*
* @param unresolvedTimezoneExpression The [[TimeZoneAwareExpression]] to resolve.
- * @return A resolved [[Expression]] with the session's local timezone applied.
+ * @return A resolved [[Expression]] with the session's local timezone applied, and optionally
+ * type coerced.
*/
override def resolve(unresolvedTimezoneExpression: TimeZoneAwareExpression): Expression = {
val expressionWithResolvedChildren =
- withResolvedChildren(unresolvedTimezoneExpression, expressionResolver.resolve)
- withResolvedTimezoneCopyTags(expressionWithResolvedChildren, conf.sessionLocalTimeZone)
+ withResolvedChildren(unresolvedTimezoneExpression, expressionResolver.resolve _)
+ withResolvedTimezone(expressionWithResolvedChildren, conf.sessionLocalTimeZone) match {
+ case cast: Cast => cast
+ case other => typeCoercionResolver.resolve(other)
+ }
}
/**
@@ -55,7 +62,7 @@ class TimezoneAwareExpressionResolver(expressionResolver: TreeNodeResolver[Expre
* @param timeZoneId The timezone ID to apply.
* @return A new [[TimeZoneAwareExpression]] with the specified timezone and original tags.
*/
- def withResolvedTimezoneCopyTags(expression: Expression, timeZoneId: String): Expression =
+ def withResolvedTimezone(expression: Expression, timeZoneId: String): Expression =
expression match {
case timezoneExpression: TimeZoneAwareExpression if timezoneExpression.timeZoneId.isEmpty =>
val withTimezone = timezoneExpression.withTimeZone(timeZoneId)
@@ -63,14 +70,4 @@ class TimezoneAwareExpressionResolver(expressionResolver: TreeNodeResolver[Expre
withTimezone
case other => other
}
-
- /**
- * Apply timezone to [[TimeZoneAwareExpression]] expressions.
- */
- def withResolvedTimezone(expression: Expression, timeZoneId: String): Expression =
- expression match {
- case timezoneExpression: TimeZoneAwareExpression if timezoneExpression.timeZoneId.isEmpty =>
- timezoneExpression.withTimeZone(timeZoneId)
- case other => other
- }
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TryExtractOrdinal.scala
similarity index 61%
copy from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
copy to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TryExtractOrdinal.scala
index c170941ce534..42766a78e248 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/TryExtractOrdinal.scala
@@ -17,17 +17,18 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.expressions.{Expression, IntegerLiteral}
-trait ResolvesExpressionChildren {
-
- /**
- * Resolves generic [[Expression]] children and returns its copy with children resolved.
- */
- protected def withResolvedChildren[ExpressionType <: Expression](
- unresolvedExpression: ExpressionType,
- resolveChild: Expression => Expression): ExpressionType = {
- val newChildren = unresolvedExpression.children.map(resolveChild(_))
- unresolvedExpression.withNewChildren(newChildren).asInstanceOf[ExpressionType]
+/**
+ * Try to extract ordinal from an expression. Return `Some(ordinal)` if the type of the expression
+ * is [[IntegerLitera]], `None` otherwise.
+ */
+object TryExtractOrdinal {
+ def apply(expression: Expression): Option[Int] = {
+ expression match {
+ case IntegerLiteral(literal) =>
+ Some(literal)
+ case other => None
+ }
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnaryMinusResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnaryMinusResolver.scala
index 04089512b31b..7f425c570859 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnaryMinusResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnaryMinusResolver.scala
@@ -39,8 +39,8 @@ class UnaryMinusResolver(
new TypeCoercionResolver(timezoneAwareExpressionResolver, typeCoercionTransformations)
override def resolve(unresolvedUnaryMinus: UnaryMinus): Expression = {
- val unaryMinusWithResolvedChildren: UnaryMinus =
- withResolvedChildren(unresolvedUnaryMinus, expressionResolver.resolve)
+ val unaryMinusWithResolvedChildren =
+ withResolvedChildren(unresolvedUnaryMinus, expressionResolver.resolve _)
typeCoercionResolver.resolve(unaryMinusWithResolvedChildren)
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnresolvedCteRelationRef.scala
similarity index 61%
copy from sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
copy to sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnresolvedCteRelationRef.scala
index c170941ce534..d0011557514d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolvesExpressionChildren.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnresolvedCteRelationRef.scala
@@ -17,17 +17,13 @@
package org.apache.spark.sql.catalyst.analysis.resolver
-import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.catalyst.analysis.{NamedRelation, UnresolvedLeafNode}
-trait ResolvesExpressionChildren {
-
- /**
- * Resolves generic [[Expression]] children and returns its copy with children resolved.
- */
- protected def withResolvedChildren[ExpressionType <: Expression](
- unresolvedExpression: ExpressionType,
- resolveChild: Expression => Expression): ExpressionType = {
- val newChildren = unresolvedExpression.children.map(resolveChild(_))
- unresolvedExpression.withNewChildren(newChildren).asInstanceOf[ExpressionType]
- }
-}
+/**
+ * A reference to a CTE definition in the form of an unresolved relation. This node is introduced by
+ * [[IdentifierAndCteSubstitutor]] to replace the CTE reference to avoid ineffective catalog RPC
+ * lookups in [[MetadataResolver]].
+ */
+case class UnresolvedCteRelationRef(override val name: String)
+ extends UnresolvedLeafNode
+ with NamedRelation
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnsupportedExpressionInOperatorValidation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnsupportedExpressionInOperatorValidation.scala
new file mode 100644
index 000000000000..af093315e586
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/UnsupportedExpressionInOperatorValidation.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.sql.catalyst.expressions.{
+ Expression,
+ Generator,
+ WindowExpression
+}
+import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
+import org.apache.spark.sql.catalyst.plans.logical.{
+ Aggregate,
+ BaseEvalPythonUDTF,
+ CollectMetrics,
+ Generate,
+ LateralJoin,
+ LogicalPlan,
+ Project,
+ Window
+}
+import org.apache.spark.sql.errors.QueryCompilationErrors
+
+object UnsupportedExpressionInOperatorValidation {
+
+ /**
+ * Check that `expression` is allowed to exist in `operator`'s expression tree.
+ */
+ def isExpressionInUnsupportedOperator(expression: Expression, operator: LogicalPlan): Boolean = {
+ expression match {
+ case _: WindowExpression => operator.isInstanceOf[Window]
+ case _: AggregateExpression =>
+ !(operator.isInstanceOf[Project] ||
+ operator.isInstanceOf[Aggregate] ||
+ operator.isInstanceOf[Window] ||
+ operator.isInstanceOf[CollectMetrics] ||
+ onlyInLateralSubquery(operator))
+ case _: Generator =>
+ !(operator.isInstanceOf[Generate] ||
+ operator.isInstanceOf[BaseEvalPythonUDTF])
+ case _ =>
+ false
+ }
+ }
+
+ private def onlyInLateralSubquery(operator: LogicalPlan): Boolean = {
+ operator.isInstanceOf[LateralJoin] && {
+ // TODO: check if we are resolving a lateral join condition once lateral join is supported.
+ throw QueryCompilationErrors.unsupportedSinglePassAnalyzerFeature(
+ s"${operator.getClass} operator resolution"
+ )
+ }
+ }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
index e7e9c5ec822a..ea89db614ad4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ViewResolver.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.analysis.resolver
import java.util.ArrayDeque
-import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
+import org.apache.spark.sql.catalyst.analysis.{AnalysisContext, UnresolvedRelation}
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View}
import org.apache.spark.sql.connector.catalog.CatalogManager
import org.apache.spark.sql.errors.QueryCompilationErrors
@@ -97,22 +97,27 @@ class ViewResolver(resolver: Resolver, catalogManager: CatalogManager)
*/
private def withViewResolutionContext(unresolvedView: View)(
body: => LogicalPlan): (LogicalPlan, ViewResolutionContext) = {
- val viewResolutionContext = if (viewResolutionContextStack.isEmpty()) {
- ViewResolutionContext(
- nestedViewDepth = 1,
- maxNestedViewDepth = conf.maxNestedViewDepth
+ AnalysisContext.withAnalysisContext(unresolvedView.desc) {
+ val prevContext = if (viewResolutionContextStack.isEmpty()) {
+ ViewResolutionContext(
+ nestedViewDepth = 0,
+ maxNestedViewDepth = conf.maxNestedViewDepth
+ )
+ } else {
+ viewResolutionContextStack.peek()
+ }
+
+ val viewResolutionContext = prevContext.copy(
+ nestedViewDepth = prevContext.nestedViewDepth + 1
)
- } else {
- val prevContext = viewResolutionContextStack.peek()
- prevContext.copy(nestedViewDepth = prevContext.nestedViewDepth + 1)
- }
- viewResolutionContext.validate(unresolvedView)
+ viewResolutionContext.validate(unresolvedView)
- viewResolutionContextStack.push(viewResolutionContext)
- try {
- (body, viewResolutionContext)
- } finally {
- viewResolutionContextStack.pop()
+ viewResolutionContextStack.push(viewResolutionContext)
+ try {
+ (body, viewResolutionContext)
+ } finally {
+ viewResolutionContextStack.pop()
+ }
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 5431f01c3c18..93acb39944fa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -274,6 +274,24 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val ANALYZER_SINGLE_PASS_RESOLVER_ENABLED_TENTATIVELY =
+ buildConf("spark.sql.analyzer.singlePassResolver.enabledTentatively")
+ .internal()
+ .doc(
+ "When true, use the single-pass Resolver instead of the fixed-point Analyzer only if " +
+ "a SQL query or a DataFrame program is fully supported by the single-pass Analyzer. " +
+ "This is an alternative Analyzer framework, which resolves the parsed logical plan in a " +
+ "single post-order traversal. It uses ExpressionResolver to resolve expressions and " +
+ "NameScope to control the visibility of names. In contrast to the current fixed-point " +
+ "framework, subsequent in-tree traversals are disallowed. Most of the fixed-point " +
+ "Analyzer code is reused in the form of specific node transformation functions " +
+ "(AliasResolution.resolve, FunctionResolution.resolveFunction, etc)." +
+ "This feature is currently under development."
+ )
+ .version("4.0.0")
+ .booleanConf
+ .createWithDefault(false)
+
val ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER =
buildConf("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy")
.internal()
@@ -298,6 +316,19 @@ object SQLConf {
.booleanConf
.createWithDefault(Utils.isTesting)
+ val ANALYZER_DUAL_RUN_SAMPLE_RATE =
+ buildConf("spark.sql.analyzer.singlePassResolver.dualRunSampleRate")
+ .internal()
+ .doc(
+ "Represents the rate of queries that will be run in both fixed-point and single-pass " +
+ "mode (dual run). It should be taken into account that the sample rate is not a strict " +
+ "percentage (in tests we don't sample). It is determined whether query should be run in " +
+ "dual run mode by comparing a random value with the value of this flag."
+ )
+ .version("4.1.0")
+ .doubleConf
+ .createWithDefault(if (Utils.isTesting) 1.0 else 0.001)
+
val ANALYZER_SINGLE_PASS_RESOLVER_VALIDATION_ENABLED =
buildConf("spark.sql.analyzer.singlePassResolver.validationEnabled")
.internal()
@@ -328,6 +359,16 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
+ val ANALYZER_SINGLE_PASS_RESOLVER_THROW_FROM_RESOLVER_GUARD =
+ buildConf("spark.sql.analyzer.singlePassResolver.throwFromResolverGuard")
+ .internal()
+ .doc(
+ "When set to true, ResolverGuard will throw a descriptive error on unsupported features."
+ )
+ .version("4.1.0")
+ .booleanConf
+ .createWithDefault(false)
+
val MULTI_COMMUTATIVE_OP_OPT_THRESHOLD =
buildConf("spark.sql.analyzer.canonicalization.multiCommutativeOpMemoryOptThreshold")
.internal()
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/LimitLikeExpressionValidatorSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/LimitLikeExpressionValidatorSuite.scala
new file mode 100644
index 000000000000..de4ba6d66ada
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/LimitLikeExpressionValidatorSuite.scala
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis.resolver
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast, Literal}
+import org.apache.spark.sql.catalyst.plans.logical.{GlobalLimit, LocalLimit, Offset, Tail}
+import org.apache.spark.sql.errors.QueryErrorsBase
+import org.apache.spark.sql.types.IntegerType
+
+class LimitLikeExpressionValidatorSuite extends SparkFunSuite with QueryErrorsBase {
+
+ private val limitLikeExpressionValidator = new LimitLikeExpressionValidator
+ private val testCases = Seq(
+ (LocalLimit(_, null), "localLimit", "limit"),
+ (GlobalLimit(_, null), "globalLimit", "limit"),
+ (Offset(_, null), "offset", "offset"),
+ (Tail(_, null), "tail", "tail")
+ )
+
+ for ((planBuilder, name, simpleName) <- testCases) {
+ test(s"Basic $name without errors") {
+ val expr = Literal(42, IntegerType)
+ val plan = planBuilder(expr)
+ assert(limitLikeExpressionValidator.validateLimitLikeExpr(expr, plan) == expr)
+ }
+
+ test(s"Unfoldable $name") {
+ val col = AttributeReference(name = "foo", dataType = IntegerType)()
+ val plan = planBuilder(col)
+ checkError(
+ exception = intercept[AnalysisException] {
+ limitLikeExpressionValidator.validateLimitLikeExpr(col, plan)
+ },
+ condition = "INVALID_LIMIT_LIKE_EXPRESSION.IS_UNFOLDABLE",
+ parameters = Map("name" -> simpleName, "expr" -> toSQLExpr(col))
+ )
+ }
+
+ test(s"$name with non-integer") {
+ val anyNonInteger = Literal("42")
+ val plan = planBuilder(anyNonInteger)
+ checkError(
+ exception = intercept[AnalysisException] {
+ limitLikeExpressionValidator.validateLimitLikeExpr(anyNonInteger, plan)
+ },
+ condition = "INVALID_LIMIT_LIKE_EXPRESSION.DATA_TYPE",
+ parameters = Map(
+ "name" -> simpleName,
+ "expr" -> toSQLExpr(anyNonInteger),
+ "dataType" -> toSQLType(anyNonInteger.dataType)
+ )
+ )
+ }
+
+ test(s"$name with null") {
+ val expr = Cast(Literal(null), IntegerType)
+ val plan = planBuilder(expr)
+ checkError(
+ exception = intercept[AnalysisException] {
+ limitLikeExpressionValidator.validateLimitLikeExpr(expr, plan)
+ },
+ condition = "INVALID_LIMIT_LIKE_EXPRESSION.IS_NULL",
+ parameters = Map("name" -> simpleName, "expr" -> toSQLExpr(expr))
+ )
+ }
+
+ test(s"$name with negative integer") {
+ val expr = Literal(-1, IntegerType)
+ val plan = planBuilder(expr)
+ checkError(
+ exception = intercept[AnalysisException] {
+ limitLikeExpressionValidator.validateLimitLikeExpr(expr, plan)
+ },
+ condition = "INVALID_LIMIT_LIKE_EXPRESSION.IS_NEGATIVE",
+ parameters =
+ Map("name" -> simpleName, "expr" -> toSQLExpr(expr), "v" -> toSQLValue(-1, IntegerType))
+ )
+ }
+ }
+
+ test("LIMIT with OFFSET sum exceeds max int") {
+ val expr = Literal(Int.MaxValue, IntegerType)
+ val plan = LocalLimit(expr, Offset(expr, null))
+ checkError(
+ exception = intercept[AnalysisException] {
+ limitLikeExpressionValidator.validateLimitLikeExpr(expr, plan)
+ },
+ condition = "SUM_OF_LIMIT_AND_OFFSET_EXCEEDS_MAX_INT",
+ parameters = Map("limit" -> Int.MaxValue.toString, "offset" -> Int.MaxValue.toString)
+ )
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateExpressionResolverSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateExpressionResolverSuite.scala
index eb2a00dfa6c1..a4f663b473b7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateExpressionResolverSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateExpressionResolverSuite.scala
@@ -36,16 +36,17 @@ class AggregateExpressionResolverSuite extends QueryTest with SharedSparkSession
resolver.resolve(query)
}
- test("Unsupported parent operator") {
+ test("Invalid WHERE condition") {
val resolver = createResolver()
- val query = table.limit($"count".function("a".attr))
- checkErrorMatchPVals(
+ val query = table.where($"count".function("a".attr) > 0)
+ checkError(
exception = intercept[AnalysisException] {
resolver.resolve(query)
},
- condition = "UNSUPPORTED_EXPR_FOR_OPERATOR",
+ condition = "INVALID_WHERE_CONDITION",
parameters = Map(
- "invalidExprSqls" -> """count\(a#\d+\)"""
+ "condition" -> "\"(count(a) > 0)\"",
+ "expressionList" -> "count(a)"
)
)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateResolverSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateResolverSuite.scala
new file mode 100644
index 000000000000..bcd474f330de
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AggregateResolverSuite.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.analysis.resolver
+
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.resolver.{Resolver, ResolverRunner}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.ExprUtils.toSQLExpr
+import org.apache.spark.sql.catalyst.expressions.aggregate.AnyValue
+import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AggregateResolverSuite extends QueryTest with SharedSparkSession {
+ private val table = LocalRelation.fromExternalRows(
+ Seq("a".attr.int, "b".attr.int),
+ Seq(Row(1), Row(1))
+ )
+
+ test("Valid group by") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy("a".attr)("a".attr)
+ resolverRunner.resolve(query)
+ }
+
+ test("Valid group by all") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy("all".attr)("a".attr)
+ resolverRunner.resolve(query)
+ }
+
+ test("Valid group by ordinal") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy(intToLiteral(1))(intToLiteral(1))
+ resolverRunner.resolve(query)
+ }
+
+ test("Group by aggregate function") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy($"count".function(intToLiteral(1)))(intToLiteral(1))
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "GROUP_BY_AGGREGATE",
+ parameters = Map("sqlExpr" -> $"count".function(intToLiteral(1)).sql)
+ )
+ }
+
+ test("Group by ordinal which refers to aggregate function") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy(intToLiteral(1))($"count".function(intToLiteral(1)))
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "GROUP_BY_POS_AGGREGATE",
+ parameters =
+ Map("index" -> "1", "aggExpr" -> $"count".function(intToLiteral(1)).as("count(1)").sql)
+ )
+ }
+
+ test("Group by ordinal out of range") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy(intToLiteral(100))(intToLiteral(1))
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "GROUP_BY_POS_OUT_OF_RANGE",
+ parameters = Map("index" -> "100", "size" -> "1")
+ )
+ }
+
+ test("Select a column which is not in the group by clause") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy("b".attr)("a".attr)
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "MISSING_AGGREGATION",
+ parameters = Map(
+ "expression" -> toSQLExpr("a".attr),
+ "expressionAnyValue" -> toSQLExpr(new AnyValue("a".attr))
+ )
+ )
+ }
+
+ test("Nested aggregate function") {
+ val resolverRunner = createResolverRunner()
+ val query =
+ table.groupBy("a".attr)($"count".function($"count".function(intToLiteral(1))))
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "NESTED_AGGREGATE_FUNCTION",
+ parameters = Map.empty
+ )
+ }
+
+ test("Aggregate function with nondeterministic expression") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy("a".attr)($"count".function(rand(1)))
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "AGGREGATE_FUNCTION_WITH_NONDETERMINISTIC_EXPRESSION",
+ parameters = Map("sqlExpr" -> toSQLExpr($"count".function(rand(1))))
+ )
+ }
+
+ test("Group by ordinal with a star in the aggregate expression list") {
+ val resolverRunner = createResolverRunner()
+ val query = table.groupBy(intToLiteral(1))(star())
+ checkError(
+ exception = intercept[AnalysisException] {
+ resolverRunner.resolve(query)
+ },
+ condition = "STAR_GROUP_BY_POS",
+ parameters = Map.empty
+ )
+ }
+
+ private def createResolverRunner(): ResolverRunner = {
+ val resolver = new Resolver(
+ catalogManager = spark.sessionState.catalogManager,
+ extensions = spark.sessionState.analyzer.singlePassResolverExtensions
+ )
+ new ResolverRunner(resolver)
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AliasResolverSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AliasResolverSuite.scala
new file mode 100644
index 000000000000..227d5bd8b95e
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/AliasResolverSuite.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.analysis.resolver
+
+import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.resolver.{Resolver, ResolverRunner}
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.dsl.plans._
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+import org.apache.spark.sql.catalyst.plans.NormalizePlan
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan}
+import org.apache.spark.sql.test.SharedSparkSession
+
+class AliasResolverSuite extends QueryTest with SharedSparkSession {
+ private val table = LocalRelation.fromExternalRows(
+ Seq("a".attr.int),
+ Seq(Row(1))
+ )
+
+ test("Implicit alias resolution") {
+ val query = table.select("a".attr + 1)
+ val answer = table.select(($"a".int + 1).as("(a + 1)"))
+ checkAnswer(query, answer, answer)
+ }
+
+ test("Nested aliases collapsed") {
+ val query = table.select("a".attr.as("innerAlias").as("outerAlias"))
+ val answer = table.select($"a".int.as("outerAlias"))
+ checkAnswer(query, answer, answer)
+ }
+
+ test("Arbitrary number of nested aliases") {
+ var projectList: NamedExpression = "a".attr
+ for (index <- 0 until 100) {
+ projectList = projectList.as("alias_" + index)
+ }
+ val query = table.select(projectList)
+ val answer = table.select($"a".int.as("alias_99"))
+ checkAnswer(query, answer, answer)
+ }
+
+ test("Nested aliases in the middle of expression tree") {
+ val query = table.select("a".attr.as("innerAlias").as("outerAlias") + 1)
+ val resolverAnswer = table.select(($"a".int.as("outerAlias") + 1).as("(a AS outerAlias + 1)"))
+ val resolverRunnerAnswer = table.select(($"a".int + 1).as("(a AS outerAlias + 1)"))
+ checkAnswer(query, resolverAnswer, resolverRunnerAnswer)
+ }
+
+ private def checkAnswer(
+ query: LogicalPlan,
+ expectedResolverResult: LogicalPlan,
+ expectedResolverRunnerResult: LogicalPlan): Unit = {
+ val resolver = new Resolver(
+ catalogManager = spark.sessionState.catalogManager,
+ extensions = spark.sessionState.analyzer.singlePassResolverExtensions
+ )
+ val resolverRunner = new ResolverRunner(new Resolver(
+ catalogManager = spark.sessionState.catalogManager,
+ extensions = spark.sessionState.analyzer.singlePassResolverExtensions
+ ))
+
+ val resolverResult = resolver.resolve(query)
+ val resolverRunnerResult = resolverRunner.resolve(query)
+
+ assert(NormalizePlan(resolverResult) == NormalizePlan(expectedResolverResult))
+ assert(NormalizePlan(resolverRunnerResult) == NormalizePlan(expectedResolverRunnerResult))
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExplicitlyUnsupportedResolverFeatureSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExplicitlyUnsupportedResolverFeatureSuite.scala
index f5dbe600f046..2b5bcc64d0d8 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExplicitlyUnsupportedResolverFeatureSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExplicitlyUnsupportedResolverFeatureSuite.scala
@@ -26,8 +26,6 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.test.SharedSparkSession
class ExplicitlyUnsupportedResolverFeatureSuite extends QueryTest with SharedSparkSession {
- import testImplicits._
-
test("Unsupported table types") {
withTable("csv_table") {
spark.sql("CREATE TABLE csv_table (col1 INT) USING CSV;").collect()
@@ -68,14 +66,6 @@ class ExplicitlyUnsupportedResolverFeatureSuite extends QueryTest with SharedSpa
)
}
- test("Missing attribute propagation") {
- val df = Seq(("test1", 0), ("test2", 1)).toDF("name", "id")
- checkResolution(
- df.select(df("name")).filter(df("id") === 0).queryExecution.logical,
- shouldPass = false
- )
- }
-
private def checkResolution(sqlText: String, shouldPass: Boolean = false): Unit = {
val unresolvedPlan = spark.sessionState.sqlParser.parsePlan(sqlText)
checkResolution(unresolvedPlan, shouldPass)
@@ -92,7 +82,8 @@ class ExplicitlyUnsupportedResolverFeatureSuite extends QueryTest with SharedSpa
val resolver = new Resolver(
spark.sessionState.catalogManager,
- extensions = spark.sessionState.analyzer.singlePassResolverExtensions
+ extensions = spark.sessionState.analyzer.singlePassResolverExtensions,
+ metadataResolverExtensions = spark.sessionState.analyzer.singlePassMetadataResolverExtensions
)
wrapper {
resolver.lookupMetadataAndResolve(plan)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExpressionIdAssignerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExpressionIdAssignerSuite.scala
index f12cd70fbe64..ae290caca0e0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExpressionIdAssignerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/analysis/resolver/ExpressionIdAssignerSuite.scala
@@ -22,7 +22,7 @@ import java.util.IdentityHashMap
import scala.collection.mutable.{ArrayBuffer, HashMap}
import org.apache.spark.SparkException
-import org.apache.spark.sql.{QueryTest, Row}
+import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.catalyst.analysis.resolver.{ExpressionIdAssigner, Resolver}
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
@@ -40,6 +40,8 @@ class ExpressionIdAssignerSuite extends QueryTest with SharedSparkSession {
private val col2Integer = AttributeReference(name = "col2", dataType = IntegerType)()
private val col2IntegerAlias = Alias(col2Integer, "b")()
private val col3Integer = AttributeReference(name = "col3", dataType = IntegerType)()
+ private val col4Integer = AttributeReference(name = "col4", dataType = IntegerType)()
+ private val col5Integer = AttributeReference(name = "col5", dataType = IntegerType)()
private val CONSTRAINTS_VALIDATED = TreeNodeTag[Boolean]("constraints_validated")
@@ -63,23 +65,31 @@ class ExpressionIdAssignerSuite extends QueryTest with SharedSparkSession {
val assigner = new ExpressionIdAssigner
intercept[SparkException] {
- assigner.createMapping()
- assigner.createMapping()
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
}
assigner.withNewMapping() {
- assigner.createMapping()
-
assigner.withNewMapping() {
- assigner.createMapping()
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
+
+ intercept[SparkException] {
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
+ }
intercept[SparkException] {
- assigner.createMapping()
+ assigner.createMappingFromChildMappings()
}
}
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
+
+ intercept[SparkException] {
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
+ }
+
intercept[SparkException] {
- assigner.createMapping()
+ assigner.createMappingFromChildMappings()
}
}
}
@@ -88,249 +98,350 @@ class ExpressionIdAssignerSuite extends QueryTest with SharedSparkSession {
val assigner = new ExpressionIdAssigner
intercept[SparkException] {
- assigner.createMapping(
- newOutput = Seq(col1Integer.newInstance()),
- oldOutput = Some(Seq(col1Integer, col2Integer))
+ val oldOperator = LocalRelation(output = Seq(col1Integer, col2Integer))
+
+ assigner.createMappingForLeafOperator(
+ newOperator = LocalRelation(output = Seq(col1Integer.newInstance())),
+ oldOperator = Some(oldOperator)
)
}
}
- test("Left branch: Single AttributeReference") {
+ test("Collect child mappings without creating a lower mapping first") {
val assigner = new ExpressionIdAssigner
- assigner.createMapping()
+ assigner.withNewMapping() {}
- val col1IntegerMapped = assigner.mapExpression(col1Integer)
- assert(col1IntegerMapped.isInstanceOf[AttributeReference])
- assert(col1IntegerMapped.exprId != col1Integer.exprId)
+ intercept[SparkException] {
+ assigner.withNewMapping(collectChildMapping = true) {}
+ }
+ }
+
+ test("Create mapping from absent chid mappings") {
+ val assigner = new ExpressionIdAssigner
+
+ assigner.withNewMapping() {
+ val oldOperator = LocalRelation(output = Seq(col1Integer))
+
+ assigner.createMappingForLeafOperator(
+ newOperator = LocalRelation(output = Seq(col1Integer.newInstance())),
+ oldOperator = Some(oldOperator)
+ )
+ }
+
+ intercept[SparkException] {
+ assigner.createMappingFromChildMappings()
+ }
+ }
+
+ test("Dangling references") {
+ val assigner = new ExpressionIdAssigner
+
+ assigner.withNewMapping() {
+ assigner.createMappingForLeafOperator(newOperator = LocalRelation())
- val col1IntegerReferenced = assigner.mapExpression(col1Integer)
- assert(col1IntegerReferenced.isInstanceOf[AttributeReference])
- assert(col1IntegerReferenced.exprId == col1IntegerMapped.exprId)
+ intercept[SparkException] {
+ assigner.mapExpression(col1Integer)
+ }
+ }
+
+ assigner.withNewMapping() {
+ val oldOperator = LocalRelation(output = Seq(col1Integer))
+
+ assigner.createMappingForLeafOperator(
+ newOperator = LocalRelation(output = Seq(col1Integer.newInstance())),
+ oldOperator = Some(oldOperator)
+ )
- val col1IntegerMappedReferenced = assigner.mapExpression(col1IntegerMapped)
- assert(col1IntegerMappedReferenced.isInstanceOf[AttributeReference])
- assert(col1IntegerMappedReferenced.exprId == col1IntegerMapped.exprId)
+ intercept[SparkException] {
+ assigner.mapExpression(col2Integer)
+ }
+ }
}
- test("Right branch: Single AttributeReference") {
+ test("Single AttributeReference") {
val assigner = new ExpressionIdAssigner
+
assigner.withNewMapping() {
- assigner.createMapping()
+ val oldOperator = LocalRelation(output = Seq(col1Integer))
+
+ assert(assigner.shouldPreserveLeafOperatorIds(oldOperator))
+ assigner.createMappingForLeafOperator(
+ newOperator = oldOperator
+ )
val col1IntegerMapped = assigner.mapExpression(col1Integer)
- assert(col1IntegerMapped.isInstanceOf[AttributeReference])
- assert(col1IntegerMapped.exprId != col1Integer.exprId)
+ assert(col1IntegerMapped.exprId == col1Integer.exprId)
val col1IntegerReferenced = assigner.mapExpression(col1Integer)
- assert(col1IntegerReferenced.isInstanceOf[AttributeReference])
assert(col1IntegerReferenced.exprId == col1IntegerMapped.exprId)
val col1IntegerMappedReferenced = assigner.mapExpression(col1IntegerMapped)
- assert(col1IntegerMappedReferenced.isInstanceOf[AttributeReference])
assert(col1IntegerMappedReferenced.exprId == col1IntegerMapped.exprId)
}
- }
- test("Left branch: Single Alias") {
- val assigner = new ExpressionIdAssigner
+ assigner.withNewMapping() {
+ val oldOperator = LocalRelation(output = Seq(col1Integer))
- assigner.createMapping()
+ val col1IntegerNew = col1Integer.newInstance()
- val col1IntegerAliasMapped = assigner.mapExpression(col1IntegerAlias)
- assert(col1IntegerAliasMapped.isInstanceOf[Alias])
- assert(col1IntegerAliasMapped.exprId == col1IntegerAlias.exprId)
+ assert(!assigner.shouldPreserveLeafOperatorIds(oldOperator))
+ assigner.createMappingForLeafOperator(
+ newOperator = LocalRelation(output = Seq(col1IntegerNew)),
+ oldOperator = Some(oldOperator)
+ )
- val col1IntegerAliasReferenced = assigner.mapExpression(col1IntegerAlias.toAttribute)
- assert(col1IntegerAliasReferenced.isInstanceOf[AttributeReference])
- assert(col1IntegerAliasReferenced.exprId == col1IntegerAliasMapped.exprId)
+ val col1IntegerMapped = assigner.mapExpression(col1Integer)
+ assert(col1IntegerMapped.exprId != col1Integer.exprId)
+ assert(col1IntegerMapped.exprId == col1IntegerNew.exprId)
- val col1IntegerAliasMappedReferenced =
- assigner.mapExpression(col1IntegerAliasMapped.toAttribute)
- assert(col1IntegerAliasMappedReferenced.isInstanceOf[AttributeReference])
- assert(col1IntegerAliasMappedReferenced.exprId == col1IntegerAliasMapped.exprId)
+ val col1IntegerReferenced = assigner.mapExpression(col1Integer)
+ assert(col1IntegerReferenced.exprId == col1IntegerMapped.exprId)
- val col1IntegerAliasMappedAgain = assigner.mapExpression(col1IntegerAlias)
- assert(col1IntegerAliasMappedAgain.isInstanceOf[Alias])
- assert(col1IntegerAliasMappedAgain.exprId != col1IntegerAlias.exprId)
- assert(col1IntegerAliasMappedAgain.exprId != col1IntegerAliasMapped.exprId)
+ val col1IntegerMappedReferenced = assigner.mapExpression(col1IntegerMapped)
+ assert(col1IntegerMappedReferenced.exprId == col1IntegerMapped.exprId)
+ }
}
- test("Right branch: Single Alias") {
+ test("Single Alias") {
val assigner = new ExpressionIdAssigner
+
assigner.withNewMapping() {
- assigner.createMapping()
+ val oldOperator = LocalRelation(output = Seq(col1Integer))
+
+ assert(assigner.shouldPreserveLeafOperatorIds(oldOperator))
+ assigner.createMappingForLeafOperator(
+ newOperator = oldOperator
+ )
val col1IntegerAliasMapped = assigner.mapExpression(col1IntegerAlias)
- assert(col1IntegerAliasMapped.isInstanceOf[Alias])
- assert(col1IntegerAliasMapped.exprId != col1IntegerAlias.exprId)
+ assert(col1IntegerAliasMapped.exprId == col1IntegerAlias.exprId)
val col1IntegerAliasReferenced = assigner.mapExpression(col1IntegerAlias.toAttribute)
- assert(col1IntegerAliasReferenced.isInstanceOf[AttributeReference])
assert(col1IntegerAliasReferenced.exprId == col1IntegerAliasMapped.exprId)
val col1IntegerAliasMappedReferenced =
assigner.mapExpression(col1IntegerAliasMapped.toAttribute)
- assert(col1IntegerAliasMappedReferenced.isInstanceOf[AttributeReference])
assert(col1IntegerAliasMappedReferenced.exprId == col1IntegerAliasMapped.exprId)
val col1IntegerAliasMappedAgain = assigner.mapExpression(col1IntegerAlias)
- assert(col1IntegerAliasMappedAgain.isInstanceOf[Alias])
assert(col1IntegerAliasMappedAgain.exprId != col1IntegerAlias.exprId)
assert(col1IntegerAliasMappedAgain.exprId != col1IntegerAliasMapped.exprId)
}
- }
- test("Left branch: Create mapping with new output") {
- val assigner = new ExpressionIdAssigner
+ assigner.withNewMapping() {
+ val oldOperator = LocalRelation(output = Seq(col1Integer))
- assigner.createMapping(newOutput = Seq(col1Integer, col2Integer))
+ assert(!assigner.shouldPreserveLeafOperatorIds(oldOperator))
+ assigner.createMappingForLeafOperator(
+ newOperator = LocalRelation(output = Seq(col1Integer.newInstance())),
+ oldOperator = Some(oldOperator)
+ )
- val col1IntegerReferenced = assigner.mapExpression(col1Integer)
- assert(col1IntegerReferenced.isInstanceOf[AttributeReference])
- assert(col1IntegerReferenced.exprId == col1Integer.exprId)
+ val col1IntegerAliasMapped = assigner.mapExpression(col1IntegerAlias)
+ assert(col1IntegerAliasMapped.exprId != col1IntegerAlias.exprId)
- val col2IntegerReferenced = assigner.mapExpression(col2Integer)
- assert(col2IntegerReferenced.isInstanceOf[AttributeReference])
- assert(col2IntegerReferenced.exprId == col2Integer.exprId)
+ val col1IntegerAliasReferenced = assigner.mapExpression(col1IntegerAlias.toAttribute)
+ assert(col1IntegerAliasReferenced.exprId == col1IntegerAliasMapped.exprId)
- val col2IntegerAliasMapped = assigner.mapExpression(col2IntegerAlias)
- assert(col2IntegerAliasMapped.isInstanceOf[Alias])
- assert(col2IntegerAliasMapped.exprId == col2IntegerAlias.exprId)
- assert(col2IntegerAliasMapped.exprId != col2Integer.exprId)
+ val col1IntegerAliasMappedReferenced =
+ assigner.mapExpression(col1IntegerAliasMapped.toAttribute)
+ assert(col1IntegerAliasMappedReferenced.exprId == col1IntegerAliasMapped.exprId)
- val col3IntegerMapped = assigner.mapExpression(col3Integer)
- assert(col3IntegerMapped.isInstanceOf[AttributeReference])
- assert(col3IntegerMapped.exprId != col3Integer.exprId)
+ val col1IntegerAliasMappedAgain = assigner.mapExpression(col1IntegerAlias)
+ assert(col1IntegerAliasMappedAgain.exprId != col1IntegerAlias.exprId)
+ assert(col1IntegerAliasMappedAgain.exprId != col1IntegerAliasMapped.exprId)
+ }
}
- test("Right branch: Create mapping with new output") {
+ test("Attributes and aliases") {
val assigner = new ExpressionIdAssigner
+
assigner.withNewMapping() {
- assigner.createMapping(newOutput = Seq(col1Integer, col2Integer))
+ val oldOperator = LocalRelation(output = Seq(col1Integer, col2Integer))
+ assert(assigner.shouldPreserveLeafOperatorIds(oldOperator))
+ assigner.createMappingForLeafOperator(
+ newOperator = oldOperator
+ )
val col1IntegerReferenced = assigner.mapExpression(col1Integer)
- assert(col1IntegerReferenced.isInstanceOf[AttributeReference])
... 2354 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org