(spark) branch branch-4.0 updated: [SPARK-59019][4.0][SQL] Make RewriteWithExpression non-excludable

[pt...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

peter-toth pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new 13173ad859b5 [SPARK-59019][4.0][SQL] Make RewriteWithExpression non-excludable
13173ad859b5 is described below

commit 13173ad859b5879b77daaed8b9d84da969cc377f
Author: Shrirang Mhalgi <sh...@gmail.com>
AuthorDate: Tue Sep 1 11:52:14 2026 +0200

    [SPARK-59019][4.0][SQL] Make RewriteWithExpression non-excludable
    
    Backport of #58323 to branch-4.0.
    
    ### What changes were proposed in this pull request?
    
    Same as #58323 - add `RewriteWithExpression` to `nonExcludableRules`.
    
    ### Why are the changes needed?
    
    The bug exists since SPARK-45760 introduced `RewriteWithExpression` in 4.0.0. `Between` uses `With(input)` and `RewriteWithExpression` is not in `nonExcludableRules` on branch-4.0.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Same as #58323.
    
    ### How was this patch tested?
    
    Same regression test as #58323.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #58441 from shrirangmhalgi/SPARK-59019-backport-4.0.
    
    Authored-by: Shrirang Mhalgi <sh...@gmail.com>
    Signed-off-by: Peter Toth <pe...@gmail.com>
---
 .../apache/spark/sql/catalyst/optimizer/Optimizer.scala  |  3 +++
 .../test/scala/org/apache/spark/sql/SQLQuerySuite.scala  | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 864d9b2cf9a8..e18034bed19c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -286,6 +286,9 @@ abstract class Optimizer(catalogManager: CatalogManager)
   def nonExcludableRules: Seq[String] =
     Seq(
       FinishAnalysis.ruleName,
+      // ReplaceExpressions (in FinishAnalysis) turns Between/NullIf into the Unevaluable
+      // With expression; excluding this rule leaks it into codegen and fails with INTERNAL_ERROR.
+      RewriteWithExpression.ruleName,
       RewriteDistinctAggregates.ruleName,
       ReplaceDeduplicateWithAggregate.ruleName,
       ReplaceIntersectWithSemiJoin.ruleName,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index f294ff81021d..3f50f81871ee 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.ExtendedAnalysisException
 import org.apache.spark.sql.catalyst.expressions.{CodegenObjectFactoryMode, GenericRow, Hex}
 import org.apache.spark.sql.catalyst.expressions.Cast._
 import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
-import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
+import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite, RewriteWithExpression}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.catalyst.plans.logical.{LocalLimit, Project, RepartitionByExpression, Sort}
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
@@ -4937,6 +4937,20 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
 
     checkAnswer(df, Seq(Row(null, null, 820), Row(null, "east", 420), Row("a", null, 370)))
   }
+
+  test("SPARK-59019: BETWEEN succeeds when RewriteWithExpression is in excludedRules") {
+    // RewriteWithExpression is non-excludable, so adding it to excludedRules has no effect.
+    // Before the fix, this threw INTERNAL_ERROR because With nodes reached codegen.
+    withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key ->
+      RewriteWithExpression.ruleName) {
+      checkAnswer(
+        sql("SELECT x BETWEEN 1 AND 2 AS in_range FROM (VALUES (1)) AS t(x)"),
+        Row(true))
+      checkAnswer(
+        sql("SELECT x BETWEEN 1 AND 2 AS in_range FROM (VALUES (3)) AS t(x)"),
+        Row(false))
+    }
+  }
 }
 
 case class Foo(bar: Option[String])


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org