You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by "jinchengchenghh (via GitHub)" <gi...@apache.org> on 2025/12/01 15:16:18 UTC

Re: [PR] [GLUTEN-11088][CORE] Fix Spark 4.0 GlutenDynamicPartitionPruningV1SuiteAEOn [incubator-gluten]

jinchengchenghh commented on code in PR #11212:
URL: https://github.com/apache/incubator-gluten/pull/11212#discussion_r2573246548


##########
gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenDynamicPartitionPruningSuite.scala:
##########
@@ -577,64 +572,6 @@ class GlutenDynamicPartitionPruningV1SuiteAEOff
       }
     }
   }
-
-  // TODO: fix in Spark-4.0
-  ignoreGluten(
-    "Subquery reuse across the whole plan",
-    DisableAdaptiveExecution("DPP in AQE must reuse broadcast")) {
-    withSQLConf(
-      SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
-      SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
-      SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false"
-    ) {
-      withTable("df1", "df2") {
-        spark
-          .range(100)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df1")
-
-        spark
-          .range(10)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df2")
-
-        val df = sql("""
-                       |SELECT df1.id, df2.k
-                       |FROM df1 JOIN df2 ON df1.k = df2.k
-                       |WHERE df2.id < (SELECT max(id) FROM df2 WHERE id <= 2)
-                       |""".stripMargin)
-
-        checkPartitionPruningPredicate(df, true, false)
-
-        checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
-
-        val plan = df.queryExecution.executedPlan
-
-        val subqueryIds = plan.collectWithSubqueries { case s: SubqueryExec => s.id }
-        val reusedSubqueryIds = plan.collectWithSubqueries {
-          case rs: ReusedSubqueryExec => rs.child.id
-        }
-
-        // By default Gluten pushes more filters than vanilla Spark.
-        //
-        // See also org.apache.gluten.execution.FilterHandler#applyFilterPushdownToScan
-        // See also DynamicPartitionPruningSuite.scala:1362
-        assert(subqueryIds.size == 3, "Whole plan subquery reusing not working correctly")

Review Comment:
   Maybe the previous version has extra subquery, the filter should not affect ReusedSubquery, Gluten with Spark4.0 result is same with jvm Spark



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org