You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/12/22 12:51:26 UTC
[spark] branch master updated: [SPARK-41649][CONNECT] Deduplicate docstrings in pyspark.sql.connect.window
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 261dc6f4e8a [SPARK-41649][CONNECT] Deduplicate docstrings in pyspark.sql.connect.window
261dc6f4e8a is described below
commit 261dc6f4e8ac1cc7feb48e91ecd1b5020c6dcd81
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Thu Dec 22 21:51:13 2022 +0900
[SPARK-41649][CONNECT] Deduplicate docstrings in pyspark.sql.connect.window
### What changes were proposed in this pull request?
This PR proposes to deduplicate docstrings in `pyspark.sql.connect.window`.
### Why are the changes needed?
For easier maintenance
### Does this PR introduce _any_ user-facing change?
No, dev-only. There're mintor doc changes that mention about remote support in Apache Spark 3.4.
### How was this patch tested?
No test. It has to be manually verified until we resolve SPARK-41653.
Closes #39180 from HyukjinKwon/SPARK-41649.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/window.py | 369 +++--------------------------------
python/pyspark/sql/window.py | 8 +-
2 files changed, 29 insertions(+), 348 deletions(-)
diff --git a/python/pyspark/sql/connect/window.py b/python/pyspark/sql/connect/window.py
index 12043433824..c421a0eb2e1 100644
--- a/python/pyspark/sql/connect/window.py
+++ b/python/pyspark/sql/connect/window.py
@@ -26,6 +26,7 @@ from pyspark.sql.connect.column import (
JVM_LONG_MIN,
JVM_LONG_MAX,
)
+from pyspark.sql.window import Window as PySparkWindow, WindowSpec as PySparkWindowSpec
if TYPE_CHECKING:
@@ -58,16 +59,6 @@ class WindowFrame:
class WindowSpec:
-
- """
- A window specification that defines the partitioning, ordering,
- and frame boundaries.
-
- Use the static methods in :class:`Window` to create a :class:`WindowSpec`.
-
- .. versionadded:: 3.4.0
- """
-
def __init__(
self,
partitionSpec: Sequence[Expression],
@@ -90,17 +81,6 @@ class WindowSpec:
self._frame = frame
def partitionBy(self, *cols: Union["ColumnOrName", List["ColumnOrName"]]) -> "WindowSpec":
- """
- Defines the partitioning columns in a :class:`WindowSpec`.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : str, :class:`Column` or list
- names of columns or expressions
- """
-
_cols: List[ColumnOrName] = []
for col in cols:
if isinstance(col, (str, Column)):
@@ -131,17 +111,9 @@ class WindowSpec:
frame=self._frame,
)
- def orderBy(self, *cols: Union["ColumnOrName", List["ColumnOrName"]]) -> "WindowSpec":
- """
- Defines the ordering columns in a :class:`WindowSpec`.
+ partitionBy.__doc__ = PySparkWindowSpec.partitionBy.__doc__
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : str, :class:`Column` or list
- names of columns or expressions
- """
+ def orderBy(self, *cols: Union["ColumnOrName", List["ColumnOrName"]]) -> "WindowSpec":
_cols: List[ColumnOrName] = []
for col in cols:
if isinstance(col, (str, Column)):
@@ -175,32 +147,9 @@ class WindowSpec:
frame=self._frame,
)
- def rowsBetween(self, start: int, end: int) -> "WindowSpec":
- """
- Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).
-
- Both `start` and `end` are relative positions from the current row.
- For example, "0" means "current row", while "-1" means the row before
- the current row, and "5" means the fifth row after the current row.
-
- We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
- and ``Window.currentRow`` to specify special boundary values, rather than using integral
- values directly.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- start : int
- boundary start, inclusive.
- The frame is unbounded if this is ``Window.unboundedPreceding``, or
- any value less than or equal to max(-sys.maxsize, -9223372036854775808).
- end : int
- boundary end, inclusive.
- The frame is unbounded if this is ``Window.unboundedFollowing``, or
- any value greater than or equal to min(sys.maxsize, 9223372036854775807).
- """
+ orderBy.__doc__ = PySparkWindowSpec.orderBy.__doc__
+ def rowsBetween(self, start: int, end: int) -> "WindowSpec":
if not isinstance(start, int):
raise TypeError(f"start must be a int, but got {type(start).__name__}")
if not isinstance(end, int):
@@ -217,32 +166,9 @@ class WindowSpec:
frame=WindowFrame(isRowFrame=True, start=start, end=end),
)
- def rangeBetween(self, start: int, end: int) -> "WindowSpec":
- """
- Defines the frame boundaries, from `start` (inclusive) to `end` (inclusive).
-
- Both `start` and `end` are relative from the current row. For example,
- "0" means "current row", while "-1" means one off before the current row,
- and "5" means the five off after the current row.
-
- We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
- and ``Window.currentRow`` to specify special boundary values, rather than using integral
- values directly.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- start : int
- boundary start, inclusive.
- The frame is unbounded if this is ``Window.unboundedPreceding``, or
- any value less than or equal to max(-sys.maxsize, -9223372036854775808).
- end : int
- boundary end, inclusive.
- The frame is unbounded if this is ``Window.unboundedFollowing``, or
- any value greater than or equal to min(sys.maxsize, 9223372036854775807).
- """
+ rowsBetween.__doc__ = PySparkWindowSpec.rowsBetween.__doc__
+ def rangeBetween(self, start: int, end: int) -> "WindowSpec":
if not isinstance(start, int):
raise TypeError(f"start must be a int, but got {type(start).__name__}")
if not isinstance(end, int):
@@ -259,6 +185,8 @@ class WindowSpec:
frame=WindowFrame(isRowFrame=False, start=start, end=end),
)
+ rangeBetween.__doc__ = PySparkWindowSpec.rangeBetween.__doc__
+
def __repr__(self) -> str:
strs: List[str] = []
if len(self._partitionSpec) > 0:
@@ -272,27 +200,10 @@ class WindowSpec:
return "WindowSpec(" + ", ".join(strs) + ")"
-class Window:
- """
- Utility functions for defining window in DataFrames.
-
- .. versionadded:: 3.4
-
- Notes
- -----
- When ordering is not defined, an unbounded window frame (rowFrame,
- unboundedPreceding, unboundedFollowing) is used by default. When ordering is defined,
- a growing window frame (rangeFrame, unboundedPreceding, currentRow) is used by default.
-
- Examples
- --------
- >>> # ORDER BY date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
- >>> window = Window.orderBy("date").rowsBetween(Window.unboundedPreceding, Window.currentRow)
+WindowSpec.__doc__ = PySparkWindow.__doc__
- >>> # PARTITION BY country ORDER BY date RANGE BETWEEN 3 PRECEDING AND 3 FOLLOWING
- >>> window = Window.orderBy("date").partitionBy("country").rangeBetween(-3, 3)
- """
+class Window:
_PRECEDING_THRESHOLD = max(-sys.maxsize, JVM_LONG_MIN)
_FOLLOWING_THRESHOLD = min(sys.maxsize, JVM_LONG_MAX)
@@ -306,263 +217,27 @@ class Window:
@staticmethod
def partitionBy(*cols: Union["ColumnOrName", List["ColumnOrName"]]) -> "WindowSpec":
- """
- Creates a :class:`WindowSpec` with the partitioning defined.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : str, :class:`Column` or list
- names of columns or expressions
-
- Returns
- -------
- :class: `WindowSpec`
- A :class:`WindowSpec` with the partitioning defined.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> from pyspark.sql.functions import row_number
- >>> df = spark.createDataFrame(
- ... [(1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b")], ["id", "category"])
- >>> df.show()
- +---+--------+
- | id|category|
- +---+--------+
- | 1| a|
- | 1| a|
- | 2| a|
- | 1| b|
- | 2| b|
- | 3| b|
- +---+--------+
-
- Show row number order by ``id`` in partition ``category``.
-
- >>> window = Window.partitionBy("category").orderBy("id")
- >>> df.withColumn("row_number", row_number().over(window)).show()
- +---+--------+----------+
- | id|category|row_number|
- +---+--------+----------+
- | 1| a| 1|
- | 1| a| 2|
- | 2| a| 3|
- | 1| b| 1|
- | 2| b| 2|
- | 3| b| 3|
- +---+--------+----------+
- """
-
return Window._spec.partitionBy(*cols)
+ partitionBy.__doc__ = PySparkWindow.partitionBy.__doc__
+
@staticmethod
def orderBy(*cols: Union["ColumnOrName", List["ColumnOrName"]]) -> "WindowSpec":
- """
- Creates a :class:`WindowSpec` with the ordering defined.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- cols : str, :class:`Column` or list
- names of columns or expressions
-
- Returns
- -------
- :class: `WindowSpec`
- A :class:`WindowSpec` with the ordering defined.
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> from pyspark.sql.functions import row_number
- >>> df = spark.createDataFrame(
- ... [(1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b")], ["id", "category"])
- >>> df.show()
- +---+--------+
- | id|category|
- +---+--------+
- | 1| a|
- | 1| a|
- | 2| a|
- | 1| b|
- | 2| b|
- | 3| b|
- +---+--------+
-
- Show row number order by ``category`` in partition ``id``.
-
- >>> window = Window.partitionBy("id").orderBy("category")
- >>> df.withColumn("row_number", row_number().over(window)).show()
- +---+--------+----------+
- | id|category|row_number|
- +---+--------+----------+
- | 1| a| 1|
- | 1| a| 2|
- | 1| b| 3|
- | 2| a| 1|
- | 2| b| 2|
- | 3| b| 1|
- +---+--------+----------+
- """
-
return Window._spec.orderBy(*cols)
+ orderBy.__doc__ = PySparkWindow.orderBy.__doc__
+
@staticmethod
def rowsBetween(start: int, end: int) -> "WindowSpec":
- """
- Creates a :class:`WindowSpec` with the frame boundaries defined,
- from `start` (inclusive) to `end` (inclusive).
-
- Both `start` and `end` are relative positions from the current row.
- For example, "0" means "current row", while "-1" means the row before
- the current row, and "5" means the fifth row after the current row.
-
- We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
- and ``Window.currentRow`` to specify special boundary values, rather than using integral
- values directly.
-
- A row based boundary is based on the position of the row within the partition.
- An offset indicates the number of rows above or below the current row, the frame for the
- current row starts or ends. For instance, given a row based sliding frame with a lower bound
- offset of -1 and a upper bound offset of +2. The frame for row with index 5 would range from
- index 4 to index 7.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- start : int
- boundary start, inclusive.
- The frame is unbounded if this is ``Window.unboundedPreceding``, or
- any value less than or equal to -9223372036854775808.
- end : int
- boundary end, inclusive.
- The frame is unbounded if this is ``Window.unboundedFollowing``, or
- any value greater than or equal to 9223372036854775807.
-
- Returns
- -------
- :class: `WindowSpec`
- A :class:`WindowSpec` with the frame boundaries defined,
- from `start` (inclusive) to `end` (inclusive).
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> from pyspark.sql import functions as func
- >>> df = spark.createDataFrame(
- ... [(1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b")], ["id", "category"])
- >>> df.show()
- +---+--------+
- | id|category|
- +---+--------+
- | 1| a|
- | 1| a|
- | 2| a|
- | 1| b|
- | 2| b|
- | 3| b|
- +---+--------+
-
- Calculate sum of ``id`` in the range from currentRow to currentRow + 1
- in partition ``category``
-
- >>> window = Window.partitionBy("category").orderBy("id").rowsBetween(Window.currentRow, 1)
- >>> df.withColumn("sum", func.sum("id").over(window)).sort("id", "category", "sum").show()
- +---+--------+---+
- | id|category|sum|
- +---+--------+---+
- | 1| a| 2|
- | 1| a| 3|
- | 1| b| 3|
- | 2| a| 2|
- | 2| b| 5|
- | 3| b| 3|
- +---+--------+---+
-
- """
-
return Window._spec.rowsBetween(start, end)
+ rowsBetween.__doc__ = PySparkWindow.rowsBetween.__doc__
+
@staticmethod
def rangeBetween(start: int, end: int) -> "WindowSpec":
- """
- Creates a :class:`WindowSpec` with the frame boundaries defined,
- from `start` (inclusive) to `end` (inclusive).
-
- Both `start` and `end` are relative from the current row. For example,
- "0" means "current row", while "-1" means one off before the current row,
- and "5" means the five off after the current row.
-
- We recommend users use ``Window.unboundedPreceding``, ``Window.unboundedFollowing``,
- and ``Window.currentRow`` to specify special boundary values, rather than using integral
- values directly.
-
- A range-based boundary is based on the actual value of the ORDER BY
- expression(s). An offset is used to alter the value of the ORDER BY expression, for
- instance if the current ORDER BY expression has a value of 10 and the lower bound offset
- is -3, the resulting lower bound for the current row will be 10 - 3 = 7. This however puts a
- number of constraints on the ORDER BY expressions: there can be only one expression and this
- expression must have a numerical data type. An exception can be made when the offset is
- unbounded, because no value modification is needed, in this case multiple and non-numeric
- ORDER BY expression are allowed.
-
- .. versionadded:: 3.4.0
-
- Parameters
- ----------
- start : int
- boundary start, inclusive.
- The frame is unbounded if this is ``Window.unboundedPreceding``, or
- any value less than or equal to max(-sys.maxsize, -9223372036854775808).
- end : int
- boundary end, inclusive.
- The frame is unbounded if this is ``Window.unboundedFollowing``, or
- any value greater than or equal to min(sys.maxsize, 9223372036854775807).
-
- Returns
- -------
- :class: `WindowSpec`
- A :class:`WindowSpec` with the frame boundaries defined,
- from `start` (inclusive) to `end` (inclusive).
-
- Examples
- --------
- >>> from pyspark.sql import Window
- >>> from pyspark.sql import functions as func
- >>> df = spark.createDataFrame(
- ... [(1, "a"), (1, "a"), (2, "a"), (1, "b"), (2, "b"), (3, "b")], ["id", "category"])
- >>> df.show()
- +---+--------+
- | id|category|
- +---+--------+
- | 1| a|
- | 1| a|
- | 2| a|
- | 1| b|
- | 2| b|
- | 3| b|
- +---+--------+
-
- Calculate sum of ``id`` in the range from ``id`` of currentRow to ``id`` of currentRow + 1
- in partition ``category``
-
- >>> window = Window.partitionBy("category").orderBy("id").rangeBetween(Window.currentRow, 1)
- >>> df.withColumn("sum", func.sum("id").over(window)).sort("id", "category").show()
- +---+--------+---+
- | id|category|sum|
- +---+--------+---+
- | 1| a| 4|
- | 1| a| 4|
- | 1| b| 3|
- | 2| a| 2|
- | 2| b| 5|
- | 3| b| 3|
- +---+--------+---+
-
- """
-
return Window._spec.rangeBetween(start, end)
+
+ rangeBetween.__doc__ = PySparkWindow.rangeBetween.__doc__
+
+
+Window.__doc__ = Window.__doc__
diff --git a/python/pyspark/sql/window.py b/python/pyspark/sql/window.py
index e9ea1089526..6a06345d61b 100644
--- a/python/pyspark/sql/window.py
+++ b/python/pyspark/sql/window.py
@@ -41,7 +41,10 @@ class Window:
"""
Utility functions for defining window in DataFrames.
- .. versionadded:: 1.4
+ .. versionadded:: 1.4.0
+
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
Notes
-----
@@ -361,6 +364,9 @@ class WindowSpec:
Use the static methods in :class:`Window` to create a :class:`WindowSpec`.
.. versionadded:: 1.4.0
+
+ .. versionchanged:: 3.4.0
+ Support Spark Connect.
"""
def __init__(self, jspec: JavaObject) -> None:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org