(gluten) branch main updated: [VL] Fix str_to_map MAP_KEY_DEDUP_POLICY comparison for Spark 4.1 compatibility (#12035)

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

liuneng1994 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new d904cb4f45 [VL] Fix str_to_map MAP_KEY_DEDUP_POLICY comparison for Spark 4.1 compatibility (#12035)
d904cb4f45 is described below

commit d904cb4f45cb070ad879cc3dfbea839532aaed97
Author: Surendra Singh Lilhore <45...@users.noreply.github.com>
AuthorDate: Tue May 5 10:49:14 2026 +0530

    [VL] Fix str_to_map MAP_KEY_DEDUP_POLICY comparison for Spark 4.1 compatibility (#12035)
    
    Starting from Spark 4.1, SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY) returns an enum value instead of a String. The existing comparison against the .toString of the EXCEPTION variant would always evaluate to not-equal (comparing enum != String), incorrectly rejecting str_to_map offloading even when the policy is set to EXCEPTION.
    
    Add .toString on the left side so both operands are compared as Strings, restoring correct behavior across all Spark versions.
    
    What changes are proposed in this pull request?
    Fix the code but to make code compatible with Spark 4.1
    
    How was this patch tested?
    Added new UT test to verify the fix.
    
    Was this patch authored or co-authored using generative AI tooling?
    GitHub Copilot CLI (powered by Claude Opus 4.6)
    
    Related issue: #12034
---
 .../backendsapi/velox/VeloxSparkPlanExecApi.scala  |  5 ++++-
 .../functions/ScalarFunctionsValidateSuite.scala   | 26 ++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index ac3e49399f..b7a1e172b2 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -924,8 +924,11 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with Logging {
       substraitExprName: String,
       children: Seq[ExpressionTransformer],
       expr: Expression): ExpressionTransformer = {
+    // Calling `.toString` on both sides ensures compatibility across all Spark versions.
+    // Starting from Spark 4.1, `SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)` returns
+    // an enum instead of a String.
     if (
-      SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY)
+      SQLConf.get.getConf(SQLConf.MAP_KEY_DEDUP_POLICY).toString
         != SQLConf.MapKeyDedupPolicy.EXCEPTION.toString
     ) {
       GlutenExceptionUtil.throwsNotFullySupported(
diff --git a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
index 4deff767bf..cd49c23029 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/functions/ScalarFunctionsValidateSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.SparkException
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.optimizer.NullPropagation
 import org.apache.spark.sql.execution.ProjectExec
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 
 abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
@@ -1607,4 +1608,29 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
       df.collect()
     }
   }
+
+  test("str_to_map with MAP_KEY_DEDUP_POLICY EXCEPTION") {
+    withTempPath {
+      path =>
+        Seq("a:1,b:2,c:3", "x:10,y:20", "single:99")
+          .toDF("s")
+          .write
+          .parquet(path.getCanonicalPath)
+
+        spark.read.parquet(path.getCanonicalPath).createOrReplaceTempView("str_to_map_tbl")
+        log.info("Testing str_to_map offloading with MAP_KEY_DEDUP_POLICY set to EXCEPTION")
+
+        withSQLConf(
+          SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.EXCEPTION.toString) {
+          // Explicit delimiters
+          runQueryAndCompare("SELECT s, str_to_map(s, ',', ':') FROM str_to_map_tbl") {
+            checkGlutenPlan[ProjectExecTransformer]
+          }
+          // Default delimiters (, and :)
+          runQueryAndCompare("SELECT s, str_to_map(s) FROM str_to_map_tbl") {
+            checkGlutenPlan[ProjectExecTransformer]
+          }
+        }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org