You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/03/07 12:58:21 UTC

(spark) branch master updated: [MINOR][SQL] Slightly refactor and optimize illegaility check in Recursive CTE Subqueries

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 93675c6715b3 [MINOR][SQL] Slightly refactor and optimize illegaility check in Recursive CTE Subqueries
93675c6715b3 is described below

commit 93675c6715b315b2c1e05bf1193ab11c918a34f5
Author: pavle-martinovic_data <pa...@databricks.com>
AuthorDate: Fri Mar 7 20:58:07 2025 +0800

    [MINOR][SQL] Slightly refactor and optimize illegaility check in Recursive CTE Subqueries
    
    ### What changes were proposed in this pull request?
    
    Change the place where we check whether there is a recursive CTE within a subquery. Also, change implementation to be instead of collecting all subqueries into one array, we do an in-place traversal of everything to check.
    
    ### Why are the changes needed?
    
    It's more efficient to do in-place traversal instead of collecting subqueries to an array and traverse, so this change is a small optimization.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Will be tested in [49955](https://github.com/apache/spark/pull/49955).
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #50208 from Pajaraja/pavle-martinovic_data/SmallOptimizeToCTERefIllegal.
    
    Authored-by: pavle-martinovic_data <pa...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../spark/sql/catalyst/analysis/CheckAnalysis.scala | 21 +++++++++++++++++----
 .../sql/catalyst/analysis/ResolveWithCTE.scala      | 16 ----------------
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 696c1e10d060..c8678d994ba8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
 import org.apache.spark.{SparkException, SparkThrowable}
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.ExtendedAnalysisException
-import org.apache.spark.sql.catalyst.analysis.ResolveWithCTE.{checkForSelfReferenceInSubquery, checkIfSelfReferenceIsPlacedCorrectly}
+import org.apache.spark.sql.catalyst.analysis.ResolveWithCTE.checkIfSelfReferenceIsPlacedCorrectly
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, ListAgg, Median, PercentileCont, PercentileDisc}
 import org.apache.spark.sql.catalyst.optimizer.InlineCTE
@@ -279,9 +279,6 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString
       case _ =>
     }
 
-    // Check if there is any self-reference within subqueries
-    checkForSelfReferenceInSubquery(plan)
-
     // We transform up and order the rules so as to catch the first possible failure instead
     // of the result of cascading resolution failures.
     plan.foreachUp {
@@ -1037,10 +1034,26 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString
     if (expr.plan.isStreaming) {
       plan.failAnalysis("INVALID_SUBQUERY_EXPRESSION.STREAMING_QUERY", Map.empty)
     }
+    assertNoRecursiveCTE(expr.plan)
     checkAnalysis0(expr.plan)
     ValidateSubqueryExpression(plan, expr)
   }
 
+  private def assertNoRecursiveCTE(plan: LogicalPlan): Unit = {
+    plan.foreach {
+      case r: CTERelationRef if r.recursive =>
+        throw new AnalysisException(
+          errorClass = "INVALID_RECURSIVE_REFERENCE.PLACE",
+          messageParameters = Map.empty)
+      case p => p.expressions.filter(_.containsPattern(PLAN_EXPRESSION)).foreach {
+        expr => expr.foreach {
+          case s: SubqueryExpression => assertNoRecursiveCTE(s.plan)
+          case _ =>
+        }
+      }
+    }
+  }
+
   /**
    * Validate that collected metrics names are unique. The same name cannot be used for metrics
    * with different results. However multiple instances of metrics with with same result and name
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala
index 7d2c5b7d2e2d..2b2317833d29 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveWithCTE.scala
@@ -187,22 +187,6 @@ object ResolveWithCTE extends Rule[LogicalPlan] {
     }
   }
 
-  /**
-   * Checks if there is any self-reference within subqueries and throws an error
-   * if that is the case.
-   */
-  def checkForSelfReferenceInSubquery(plan: LogicalPlan): Unit = {
-    plan.subqueriesAll.foreach { subquery =>
-      subquery.foreach {
-        case r: CTERelationRef if r.recursive =>
-          throw new AnalysisException(
-            errorClass = "INVALID_RECURSIVE_REFERENCE.PLACE",
-            messageParameters = Map.empty)
-        case _ =>
-      }
-    }
-  }
-
   /**
    * Counts number of self-references in a recursive CTE definition and throws an error
    * if that number is bigger than 1.


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org