You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/04/24 07:30:20 UTC
(spark) branch master updated: [SPARK-56607][PYTHON] Remove unnecessary __new__ and __init__
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d325970c3777 [SPARK-56607][PYTHON] Remove unnecessary __new__ and __init__
d325970c3777 is described below
commit d325970c3777d70576ebfa4f02ac764c16dce692
Author: Tian Gao <ga...@hotmail.com>
AuthorDate: Fri Apr 24 15:30:07 2026 +0800
[SPARK-56607][PYTHON] Remove unnecessary __new__ and __init__
### What changes were proposed in this pull request?
* Removed `__init__` call in `__new__` because it's unnecessary and could be unsound.
* Removed unnecessary `__new__` method when it just does the default thing.
* Kept the weird behavior of `DataFrame` where the base class calls the derived class `__new__` method, but made it cleaner.
### Why are the changes needed?
Remove some unnecessary and potentially buggy code. Also less mypy ignores.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #55526 from gaogaotiantian/remove-unnecessary-new.
Authored-by: Tian Gao <ga...@hotmail.com>
Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
---
python/pyspark/sql/classic/dataframe.py | 10 +++++-----
python/pyspark/sql/classic/window.py | 1 -
python/pyspark/sql/connect/dataframe.py | 10 +++++-----
python/pyspark/sql/connect/window.py | 1 -
python/pyspark/sql/dataframe.py | 2 +-
python/pyspark/sql/tests/connect/test_connect_plan.py | 5 -----
python/pyspark/sql/window.py | 2 +-
python/pyspark/testing/mlutils.py | 8 ++------
8 files changed, 14 insertions(+), 25 deletions(-)
diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py
index f80b7ca7f019..3501f01135d8 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -104,12 +104,12 @@ if TYPE_CHECKING:
class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
def __new__(
cls,
- jdf: "JavaObject",
- sql_ctx: Union["SQLContext", "SparkSession"],
+ *args: Any,
+ **kwargs: Any,
) -> "DataFrame":
- self = object.__new__(cls)
- self.__init__(jdf, sql_ctx) # type: ignore[misc]
- return self
+ # ParentDataFrame by default calls DataFrame.__new__ for backward compatibility.
+ # We have to do an explicit object.__new__ to avoid infinite recursion.
+ return object.__new__(cls)
def __init__(
self,
diff --git a/python/pyspark/sql/classic/window.py b/python/pyspark/sql/classic/window.py
index 3b3c8785cb4c..0a9e0b3c96c6 100644
--- a/python/pyspark/sql/classic/window.py
+++ b/python/pyspark/sql/classic/window.py
@@ -95,7 +95,6 @@ class Window(ParentWindow):
class WindowSpec(ParentWindowSpec):
def __new__(cls, jspec: "JavaObject") -> "WindowSpec":
self = object.__new__(cls)
- self.__init__(jspec) # type: ignore[misc]
return self
def __init__(self, jspec: "JavaObject") -> None:
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 1722c5feab87..c6602e08fac4 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -112,12 +112,12 @@ if TYPE_CHECKING:
class DataFrame(ParentDataFrame):
def __new__(
cls,
- plan: plan.LogicalPlan,
- session: "SparkSession",
+ *args: Any,
+ **kwargs: Any,
) -> "DataFrame":
- self = object.__new__(cls)
- self.__init__(plan, session) # type: ignore[misc]
- return self
+ # ParentDataFrame by default creates classic DataFrame for backward compatibility.
+ # We have to do an explicit object.__new__ to avoid that.
+ return object.__new__(cls)
def __init__(
self,
diff --git a/python/pyspark/sql/connect/window.py b/python/pyspark/sql/connect/window.py
index 0fe118ff189f..55a43c2158a3 100644
--- a/python/pyspark/sql/connect/window.py
+++ b/python/pyspark/sql/connect/window.py
@@ -62,7 +62,6 @@ class WindowSpec(ParentWindowSpec):
frame: Optional[WindowFrame],
) -> "WindowSpec":
self = object.__new__(cls)
- self.__init__(partitionSpec, orderSpec, frame) # type: ignore[misc]
return self
def __getnewargs__(self) -> Tuple[Any, ...]:
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index b87eefd9ca4e..b2586a2d7a18 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -147,7 +147,7 @@ class DataFrame:
) -> "DataFrame":
from pyspark.sql.classic.dataframe import DataFrame
- return DataFrame.__new__(DataFrame, jdf, sql_ctx)
+ return DataFrame(jdf, sql_ctx)
@property
def sparkSession(self) -> "SparkSession":
diff --git a/python/pyspark/sql/tests/connect/test_connect_plan.py b/python/pyspark/sql/tests/connect/test_connect_plan.py
index a0ea44c8e531..097f2dc18231 100644
--- a/python/pyspark/sql/tests/connect/test_connect_plan.py
+++ b/python/pyspark/sql/tests/connect/test_connect_plan.py
@@ -343,11 +343,6 @@ class SparkConnectPlanTests(PlanOnlyTestFixture):
from pyspark.sql.connect.observation import Observation
class MockDF(DataFrame):
- def __new__(cls, df: DataFrame) -> "DataFrame":
- self = object.__new__(cls)
- self.__init__(df) # type: ignore[misc]
- return self
-
def __init__(self, df: DataFrame):
super().__init__(df._plan, df._session)
diff --git a/python/pyspark/sql/window.py b/python/pyspark/sql/window.py
index 73b1d22431d5..0b2687a03886 100644
--- a/python/pyspark/sql/window.py
+++ b/python/pyspark/sql/window.py
@@ -340,7 +340,7 @@ class WindowSpec:
"""
def __new__(cls, jspec: "JavaObject") -> "WindowSpec":
- from pyspark.sql.classic.WindowSpec import WindowSpec # type: ignore[import-not-found]
+ from pyspark.sql.classic.window import WindowSpec
return WindowSpec.__new__(WindowSpec, jspec)
diff --git a/python/pyspark/testing/mlutils.py b/python/pyspark/testing/mlutils.py
index 491c539acc0e..8c1c7c3b1242 100644
--- a/python/pyspark/testing/mlutils.py
+++ b/python/pyspark/testing/mlutils.py
@@ -25,7 +25,8 @@ from pyspark.ml.param.shared import HasMaxIter, HasRegParam
from pyspark.ml.classification import Classifier, ClassificationModel
from pyspark.ml.util import DefaultParamsReadable, DefaultParamsWritable
from pyspark.ml.wrapper import _java2py
-from pyspark.sql import DataFrame, SparkSession
+from pyspark.sql import SparkSession
+from pyspark.sql.classic.dataframe import DataFrame
from pyspark.sql.types import DoubleType
from pyspark.testing.utils import ReusedPySparkTestCase as PySparkTestCase
@@ -99,11 +100,6 @@ class SparkSessionTestCase(PySparkTestCase):
class MockDataset(DataFrame):
- def __new__(cls) -> "DataFrame":
- self = object.__new__(cls)
- self.__init__()
- return self
-
def __init__(self):
self.index = 0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org