You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/09/21 05:37:43 UTC
(spark) branch master updated: [SPARK-59520][PYTHON] Perform PySpark RDD conversion under tracked execution
This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9de1c68dfe68 [SPARK-59520][PYTHON] Perform PySpark RDD conversion under tracked execution
9de1c68dfe68 is described below
commit 9de1c68dfe686f15373f03a4290b7cbb8765af93
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon Sep 21 14:37:23 2026 +0900
[SPARK-59520][PYTHON] Perform PySpark RDD conversion under tracked execution
### What changes were proposed in this pull request?
`DataFrame.rdd` in PySpark calls `Dataset.javaToPython` on the JVM side to build the Python RDD. Unlike the classic Scala `Dataset.rdd`, which was wrapped in `withNewRDDExecutionId` by SPARK-50994, the no-arg `javaToPython` executed the plan without an execution context. This PR wraps it in `withNewRDDExecutionId` so the RDD conversion runs under a tracked SQL execution, reusing the existing helper (generalized to accept the `QueryExecution` to track).
### Why are the changes needed?
Without a tracked execution, the thread-local Spark properties carrying the session SQL configs are not propagated to the executors during the RDD conversion. Some operations depend on these being available in the `TaskContext` on the executor side, for example the non-vectorized Parquet reader reads `spark.sql.caseSensitive` from `SQLConf`. When the conf is not propagated, the read resolves columns incorrectly, producing wrong results or failing the read. This is the PySpark counterp [...]
### Does this PR introduce _any_ user-facing change?
Yes. Session SQL configs are now correctly propagated to the executors when a PySpark `DataFrame` is converted to an RDD, so operations that read configs on the executor side (e.g. the non-vectorized Parquet reader honoring `spark.sql.caseSensitive`) behave correctly.
### How was this patch tested?
Added a test in `pyspark.sql.tests.test_dataframe` mirroring the SPARK-50994 scenario: with `spark.sql.caseSensitive=true` and the non-vectorized Parquet reader, a `DataFrame` with two same-named but differently-cased columns is written to Parquet, deduplicated, and converted to an RDD. Without the fix the executor-side read does not see the conf and fails; with the fix the RDD yields the correct deduplicated rows.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Isaac
This pull request and its description were written by Isaac.
Closes #58808 from HyukjinKwon/SPARK-59520.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <hy...@databricks.com>
---
.../sql/tests/connect/test_parity_dataframe.py | 4 +++
python/pyspark/sql/tests/test_dataframe.py | 31 ++++++++++++++++++++++
.../org/apache/spark/sql/classic/Dataset.scala | 13 ++++++---
3 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_dataframe.py b/python/pyspark/sql/tests/connect/test_parity_dataframe.py
index decb027f8e21..fe38894e0776 100644
--- a/python/pyspark/sql/tests/connect/test_parity_dataframe.py
+++ b/python/pyspark/sql/tests/connect/test_parity_dataframe.py
@@ -30,6 +30,10 @@ class DataFrameParityTests(DataFrameTestsMixin, ReusedConnectTestCase):
def test_toDF_with_schema_string(self):
super().test_toDF_with_schema_string()
+ @unittest.skip("Spark Connect does not support RDD but the tests depend on them.")
+ def test_rdd_conversion_propagates_sql_conf(self):
+ super().test_rdd_conversion_propagates_sql_conf()
+
@unittest.skip("This test is dedicated for classic mode.")
def test_query_execution_unsupported_in_classic(self):
pass
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 70c52404fc4f..0393cb1b6164 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -1228,6 +1228,37 @@ class DataFrameTestsMixin:
self.assertIsInstance(df, DataFrame)
self.assertEqual(df.select("value").count(), 10)
+ def test_rdd_conversion_propagates_sql_conf(self):
+ # Converting a DataFrame to an RDD must run under a tracked SQL execution so
+ # that session SQLConfs are propagated to the executors, mirroring the classic
+ # Scala Dataset.rdd behavior (SPARK-50994). Otherwise the non-vectorized parquet
+ # reader on the executor side would not see spark.sql.caseSensitive and would
+ # resolve the two same-named but differently-cased columns wrongly, failing to
+ # read the file. Accessing `.rdd` eagerly materializes the shuffle map stage
+ # (the parquet scan) under adaptive execution; disabling shuffle-file cleanup
+ # (off by default outside of tests) lets `collect` reuse that materialized
+ # output instead of recomputing the scan without the propagated conf.
+ with self.sql_conf(
+ {
+ "spark.sql.caseSensitive": True,
+ "spark.sql.parquet.enableVectorizedReader": False,
+ "spark.sql.classic.shuffleDependency.fileCleanup.enabled": False,
+ }
+ ):
+ with tempfile.TemporaryDirectory(prefix="test_rdd_conversion_sql_conf") as d:
+ self.spark.createDataFrame(
+ [(1, 1.0), (2, 2.0), (3, 3.0), (1, 1.0)], ["a", "A"]
+ ).write.format("parquet").mode("overwrite").save(d)
+
+ deduplicated = self.spark.read.parquet(d).dropDuplicates(["a"])
+
+ rows = deduplicated.rdd.collect()
+
+ self.assertEqual(
+ sorted((row[0], row[1]) for row in rows),
+ [(1, 1.0), (2, 2.0), (3, 3.0)],
+ )
+
def test_zip_with_index(self):
df = self.spark.createDataFrame([("a", 1), ("b", 2), ("c", 3)], ["letter", "number"])
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
index e635484f6ed1..8f03d91dc55b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
@@ -2132,7 +2132,10 @@ class Dataset[T] private[sql](
* Converts a JavaRDD to a PythonRDD.
*/
private[sql] def javaToPython: JavaRDD[Array[Byte]] = {
- javaToPython(queryExecution.withRegularShuffle)
+ val qe = queryExecution.withRegularShuffle
+ withNewRDDExecutionId("javaToPython", qe) {
+ javaToPython(qe)
+ }
}
private def javaToPython(qe: QueryExecution): JavaRDD[Array[Byte]] = {
@@ -2314,9 +2317,11 @@ class Dataset[T] private[sql](
* them with an execution. Before performing the action, the metrics of the executed plan will be
* reset.
*/
- private def withNewRDDExecutionId[U](name: String)(body: => U): U = {
- SQLExecution.withNewExecutionId(rddQueryExecution, Some(name)) {
- rddQueryExecution.executedPlan.resetMetrics()
+ private def withNewRDDExecutionId[U](
+ name: String,
+ qe: QueryExecution = rddQueryExecution)(body: => U): U = {
+ SQLExecution.withNewExecutionId(qe, Some(name)) {
+ qe.executedPlan.resetMetrics()
body
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org