You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/09/21 05:37:43 UTC

(spark) branch master updated: [SPARK-59520][PYTHON] Perform PySpark RDD conversion under tracked execution

This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 9de1c68dfe68 [SPARK-59520][PYTHON] Perform PySpark RDD conversion under tracked execution
9de1c68dfe68 is described below

commit 9de1c68dfe686f15373f03a4290b7cbb8765af93
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon Sep 21 14:37:23 2026 +0900

    [SPARK-59520][PYTHON] Perform PySpark RDD conversion under tracked execution
    
    ### What changes were proposed in this pull request?
    
    `DataFrame.rdd` in PySpark calls `Dataset.javaToPython` on the JVM side to build the Python RDD. Unlike the classic Scala `Dataset.rdd`, which was wrapped in `withNewRDDExecutionId` by SPARK-50994, the no-arg `javaToPython` executed the plan without an execution context. This PR wraps it in `withNewRDDExecutionId` so the RDD conversion runs under a tracked SQL execution, reusing the existing helper (generalized to accept the `QueryExecution` to track).
    
    ### Why are the changes needed?
    
    Without a tracked execution, the thread-local Spark properties carrying the session SQL configs are not propagated to the executors during the RDD conversion. Some operations depend on these being available in the `TaskContext` on the executor side, for example the non-vectorized Parquet reader reads `spark.sql.caseSensitive` from `SQLConf`. When the conf is not propagated, the read resolves columns incorrectly, producing wrong results or failing the read. This is the PySpark counterp [...]
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Session SQL configs are now correctly propagated to the executors when a PySpark `DataFrame` is converted to an RDD, so operations that read configs on the executor side (e.g. the non-vectorized Parquet reader honoring `spark.sql.caseSensitive`) behave correctly.
    
    ### How was this patch tested?
    
    Added a test in `pyspark.sql.tests.test_dataframe` mirroring the SPARK-50994 scenario: with `spark.sql.caseSensitive=true` and the non-vectorized Parquet reader, a `DataFrame` with two same-named but differently-cased columns is written to Parquet, deduplicated, and converted to an RDD. Without the fix the executor-side read does not see the conf and fails; with the fix the RDD yields the correct deduplicated rows.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Isaac
    
    This pull request and its description were written by Isaac.
    
    Closes #58808 from HyukjinKwon/SPARK-59520.
    
    Authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Hyukjin Kwon <hy...@databricks.com>
---
 .../sql/tests/connect/test_parity_dataframe.py     |  4 +++
 python/pyspark/sql/tests/test_dataframe.py         | 31 ++++++++++++++++++++++
 .../org/apache/spark/sql/classic/Dataset.scala     | 13 ++++++---
 3 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/python/pyspark/sql/tests/connect/test_parity_dataframe.py b/python/pyspark/sql/tests/connect/test_parity_dataframe.py
index decb027f8e21..fe38894e0776 100644
--- a/python/pyspark/sql/tests/connect/test_parity_dataframe.py
+++ b/python/pyspark/sql/tests/connect/test_parity_dataframe.py
@@ -30,6 +30,10 @@ class DataFrameParityTests(DataFrameTestsMixin, ReusedConnectTestCase):
     def test_toDF_with_schema_string(self):
         super().test_toDF_with_schema_string()
 
+    @unittest.skip("Spark Connect does not support RDD but the tests depend on them.")
+    def test_rdd_conversion_propagates_sql_conf(self):
+        super().test_rdd_conversion_propagates_sql_conf()
+
     @unittest.skip("This test is dedicated for classic mode.")
     def test_query_execution_unsupported_in_classic(self):
         pass
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 70c52404fc4f..0393cb1b6164 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -1228,6 +1228,37 @@ class DataFrameTestsMixin:
             self.assertIsInstance(df, DataFrame)
             self.assertEqual(df.select("value").count(), 10)
 
+    def test_rdd_conversion_propagates_sql_conf(self):
+        # Converting a DataFrame to an RDD must run under a tracked SQL execution so
+        # that session SQLConfs are propagated to the executors, mirroring the classic
+        # Scala Dataset.rdd behavior (SPARK-50994). Otherwise the non-vectorized parquet
+        # reader on the executor side would not see spark.sql.caseSensitive and would
+        # resolve the two same-named but differently-cased columns wrongly, failing to
+        # read the file. Accessing `.rdd` eagerly materializes the shuffle map stage
+        # (the parquet scan) under adaptive execution; disabling shuffle-file cleanup
+        # (off by default outside of tests) lets `collect` reuse that materialized
+        # output instead of recomputing the scan without the propagated conf.
+        with self.sql_conf(
+            {
+                "spark.sql.caseSensitive": True,
+                "spark.sql.parquet.enableVectorizedReader": False,
+                "spark.sql.classic.shuffleDependency.fileCleanup.enabled": False,
+            }
+        ):
+            with tempfile.TemporaryDirectory(prefix="test_rdd_conversion_sql_conf") as d:
+                self.spark.createDataFrame(
+                    [(1, 1.0), (2, 2.0), (3, 3.0), (1, 1.0)], ["a", "A"]
+                ).write.format("parquet").mode("overwrite").save(d)
+
+                deduplicated = self.spark.read.parquet(d).dropDuplicates(["a"])
+
+                rows = deduplicated.rdd.collect()
+
+                self.assertEqual(
+                    sorted((row[0], row[1]) for row in rows),
+                    [(1, 1.0), (2, 2.0), (3, 3.0)],
+                )
+
     def test_zip_with_index(self):
         df = self.spark.createDataFrame([("a", 1), ("b", 2), ("c", 3)], ["letter", "number"])
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
index e635484f6ed1..8f03d91dc55b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
@@ -2132,7 +2132,10 @@ class Dataset[T] private[sql](
    * Converts a JavaRDD to a PythonRDD.
    */
   private[sql] def javaToPython: JavaRDD[Array[Byte]] = {
-    javaToPython(queryExecution.withRegularShuffle)
+    val qe = queryExecution.withRegularShuffle
+    withNewRDDExecutionId("javaToPython", qe) {
+      javaToPython(qe)
+    }
   }
 
   private def javaToPython(qe: QueryExecution): JavaRDD[Array[Byte]] = {
@@ -2314,9 +2317,11 @@ class Dataset[T] private[sql](
    * them with an execution. Before performing the action, the metrics of the executed plan will be
    * reset.
    */
-  private def withNewRDDExecutionId[U](name: String)(body: => U): U = {
-    SQLExecution.withNewExecutionId(rddQueryExecution, Some(name)) {
-      rddQueryExecution.executedPlan.resetMetrics()
+  private def withNewRDDExecutionId[U](
+      name: String,
+      qe: QueryExecution = rddQueryExecution)(body: => U): U = {
+    SQLExecution.withNewExecutionId(qe, Some(name)) {
+      qe.executedPlan.resetMetrics()
       body
     }
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org