You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/06/06 11:16:25 UTC
(spark) branch master updated: [SPARK-48550][PS] Directly use the parent Window class
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8cb78a7811f3 [SPARK-48550][PS] Directly use the parent Window class
8cb78a7811f3 is described below
commit 8cb78a7811f337309a72ff0ef29eefbb31d8bdc4
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Jun 6 20:16:14 2024 +0900
[SPARK-48550][PS] Directly use the parent Window class
### What changes were proposed in this pull request?
Directly use the parent Window class
### Why are the changes needed?
the `get_window_class` method is no longer needed
### Does this PR introduce _any_ user-facing change?
no, refactor only
### How was this patch tested?
CI
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #46892 from zhengruifeng/del_get_win_class.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/series.py | 17 ++++++++++-------
python/pyspark/sql/utils.py | 13 -------------
2 files changed, 10 insertions(+), 20 deletions(-)
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index ac11f327258a..4e0f3ca34917 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -54,7 +54,12 @@ from pandas.api.types import ( # type: ignore[attr-defined]
)
from pandas.tseries.frequencies import DateOffset
-from pyspark.sql import functions as F, Column as PySparkColumn, DataFrame as SparkDataFrame
+from pyspark.sql import (
+ functions as F,
+ Column as PySparkColumn,
+ DataFrame as SparkDataFrame,
+ Window as PySparkWindow,
+)
from pyspark.sql.types import (
ArrayType,
BooleanType,
@@ -70,7 +75,6 @@ from pyspark.sql.types import (
NullType,
)
from pyspark.sql.window import Window
-from pyspark.sql.utils import get_window_class
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark.pandas._typing import Axis, Dtype, Label, Name, Scalar, T
from pyspark.pandas.accessors import PandasOnSparkSeriesMethods
@@ -2257,15 +2261,14 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
last_non_null = F.last(scol, True)
null_index = SF.null_index(scol)
- Window = get_window_class()
- window_forward = Window.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(
- Window.unboundedPreceding, Window.currentRow
+ window_forward = PySparkWindow.orderBy(NATURAL_ORDER_COLUMN_NAME).rowsBetween(
+ PySparkWindow.unboundedPreceding, PySparkWindow.currentRow
)
last_non_null_forward = last_non_null.over(window_forward)
null_index_forward = null_index.over(window_forward)
- window_backward = Window.orderBy(F.desc(NATURAL_ORDER_COLUMN_NAME)).rowsBetween(
- Window.unboundedPreceding, Window.currentRow
+ window_backward = PySparkWindow.orderBy(F.desc(NATURAL_ORDER_COLUMN_NAME)).rowsBetween(
+ PySparkWindow.unboundedPreceding, PySparkWindow.currentRow
)
last_non_null_backward = last_non_null.over(window_backward)
null_index_backward = null_index.over(window_backward)
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index ba1832ca36c9..76227851f9fe 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -27,7 +27,6 @@ from typing import (
cast,
TypeVar,
Union,
- Type,
)
# For backward compatibility.
@@ -57,7 +56,6 @@ if TYPE_CHECKING:
from pyspark import SparkContext
from pyspark.sql.session import SparkSession
from pyspark.sql.dataframe import DataFrame
- from pyspark.sql.window import Window
from pyspark.pandas._typing import IndexOpsLike, SeriesOrIndex
has_numpy: bool = False
@@ -417,17 +415,6 @@ def pyspark_column_op(
return result.fillna(fillna) if fillna is not None else result
-def get_window_class() -> Type["Window"]:
- from pyspark.sql.window import Window as PySparkWindow
-
- if is_remote():
- from pyspark.sql.connect.window import Window as ConnectWindow
-
- return ConnectWindow
- else:
- return PySparkWindow
-
-
def get_lit_sql_str(val: str) -> str:
# Equivalent to `lit(val)._jc.expr().sql()` for string typed val
# See `sql` definition in `sql/catalyst/src/main/scala/org/apache/spark/
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org