You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/12/21 09:46:36 UTC

[spark] branch master updated: [SPARK-41643][CONNECT] Deduplicate docstrings in pyspark.sql.connect.column

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new d94642625bc [SPARK-41643][CONNECT] Deduplicate docstrings in pyspark.sql.connect.column
d94642625bc is described below

commit d94642625bc701f2bf76a0e1d97aa29212d961bd
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Wed Dec 21 18:46:19 2022 +0900

    [SPARK-41643][CONNECT] Deduplicate docstrings in pyspark.sql.connect.column
    
    ### What changes were proposed in this pull request?
    
    This PR proposes to deduplicate docstrings in `pyspark.sql.connect.column`.
    
    ### Why are the changes needed?
    
    For easier maintenance
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, dev-only. There're mintor doc changes that mention about remote support in Apache Spark 3.4.
    
    ### How was this patch tested?
    
    CI in PR builds should test.
    
    Closes #39150 from HyukjinKwon/SPARK-41643.
    
    Authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 python/pyspark/sql/column.py         |  57 +++++
 python/pyspark/sql/connect/column.py | 451 ++++-------------------------------
 2 files changed, 97 insertions(+), 411 deletions(-)

diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 26a1cda61c6..bb43b0af57c 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -189,6 +189,9 @@ class Column:
 
     .. versionadded:: 1.3.0
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Examples
     --------
     Column instances can be created by
@@ -282,6 +285,9 @@ class Column:
 
     .. versionadded:: 2.3.0
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Parameters
     ----------
     other
@@ -358,6 +364,9 @@ class Column:
     _bitwiseOR_doc = """
     Compute bitwise OR of this expression with another expression.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Parameters
     ----------
     other
@@ -374,6 +383,9 @@ class Column:
     _bitwiseAND_doc = """
     Compute bitwise AND of this expression with another expression.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Parameters
     ----------
     other
@@ -390,6 +402,9 @@ class Column:
     _bitwiseXOR_doc = """
     Compute bitwise XOR of this expression with another expression.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Parameters
     ----------
     other
@@ -625,6 +640,9 @@ class Column:
     _contains_doc = """
     Contains the other element. Returns a boolean :class:`Column` based on a string match.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Parameters
     ----------
     other
@@ -645,6 +663,9 @@ class Column:
     other : :class:`Column` or str
         string at start of line (do not use a regex `^`)
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Examples
     --------
     >>> df = spark.createDataFrame(
@@ -657,6 +678,9 @@ class Column:
     _endswith_doc = """
     String ends with. Returns a boolean :class:`Column` based on a string match.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Parameters
     ----------
     other : :class:`Column` or str
@@ -680,6 +704,9 @@ class Column:
         """
         SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         other : str
@@ -710,6 +737,9 @@ class Column:
         SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex
         match.
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         other : str
@@ -738,6 +768,9 @@ class Column:
 
         .. versionadded:: 3.3.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         other : str
@@ -777,6 +810,9 @@ class Column:
 
         .. versionadded:: 1.3.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         startPos : :class:`Column` or int
@@ -819,6 +855,9 @@ class Column:
 
         .. versionadded:: 1.5.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         cols
@@ -938,6 +977,9 @@ class Column:
     _isNull_doc = """
     True if the current expression is null.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Examples
     --------
     >>> from pyspark.sql import Row
@@ -948,6 +990,9 @@ class Column:
     _isNotNull_doc = """
     True if the current expression is NOT null.
 
+    .. versionchanged:: 3.4.0
+        Support Spark Connect.
+
     Examples
     --------
     >>> from pyspark.sql import Row
@@ -966,6 +1011,9 @@ class Column:
 
         .. versionadded:: 1.3.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         alias : str
@@ -1021,6 +1069,9 @@ class Column:
 
         .. versionadded:: 1.3.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         dataType : :class:`DataType` or str
@@ -1100,6 +1151,9 @@ class Column:
 
         .. versionadded:: 1.4.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         condition : :class:`Column`
@@ -1142,6 +1196,9 @@ class Column:
 
         .. versionadded:: 1.4.0
 
+        .. versionchanged:: 3.4.0
+            Support Spark Connect.
+
         Parameters
         ----------
         value
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index adce180eaa3..7216862f25d 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -50,13 +50,13 @@ from pyspark.sql.types import (
     DayTimeIntervalType,
 )
 
+from pyspark.sql.column import Column as PySparkColumn
 import pyspark.sql.connect.proto as proto
 from pyspark.sql.connect.types import pyspark_types_to_proto_types
 
 if TYPE_CHECKING:
-    from pyspark.sql.connect._typing import ColumnOrName, PrimitiveType
+    from pyspark.sql.connect._typing import ColumnOrName
     from pyspark.sql.connect.client import SparkConnectClient
-    import pyspark.sql.connect.proto as proto
 
 JVM_BYTE_MIN = -(1 << 7)
 JVM_BYTE_MAX = (1 << 7) - 1
@@ -68,17 +68,18 @@ JVM_LONG_MIN = -(1 << 63)
 JVM_LONG_MAX = (1 << 63) - 1
 
 
-def _func_op(name: str, doc: str = "") -> Callable[["Column"], "Column"]:
-    def _(self: "Column") -> "Column":
+def _func_op(name: str, doc: Optional[str] = "") -> Callable[["Column"], "Column"]:
+    def wrapped(self: "Column") -> "Column":
         return scalar_function(name, self)
 
-    return _
+    wrapped.__doc__ = doc
+    return wrapped
 
 
 def _bin_op(
-    name: str, doc: str = "binary function", reverse: bool = False
+    name: str, doc: Optional[str] = "binary function", reverse: bool = False
 ) -> Callable[["Column", Any], "Column"]:
-    def _(self: "Column", other: Any) -> "Column":
+    def wrapped(self: "Column", other: Any) -> "Column":
         from pyspark.sql.connect._typing import PrimitiveType
         from pyspark.sql.connect.functions import lit
 
@@ -89,14 +90,16 @@ def _bin_op(
         else:
             return scalar_function(name, other, self)
 
-    return _
+    wrapped.__doc__ = doc
+    return wrapped
 
 
-def _unary_op(name: str, doc: str = "unary function") -> Callable[["Column"], "Column"]:
-    def _(self: "Column") -> "Column":
+def _unary_op(name: str, doc: Optional[str] = "unary function") -> Callable[["Column"], "Column"]:
+    def wrapped(self: "Column") -> "Column":
         return scalar_function(name, self)
 
-    return _
+    wrapped.__doc__ = doc
+    return wrapped
 
 
 def scalar_function(op: str, *args: "Column") -> "Column":
@@ -122,42 +125,12 @@ class Expression:
         ...
 
     def alias(self, *alias: str, **kwargs: Any) -> "ColumnAlias":
-        """
-        Returns this column aliased with a new name or names (in the case of expressions that
-        return more than one column, such as explode).
-
-        .. versionadded:: 3.4.0
-
-        Parameters
-        ----------
-        alias : str
-            desired column names (collects all positional arguments passed)
-
-        Other Parameters
-        ----------------
-        metadata: dict
-            a dict of information to be stored in ``metadata`` attribute of the
-            corresponding :class:`StructField <pyspark.sql.types.StructField>` (optional, keyword
-            only argument)
-
-        Returns
-        -------
-        :class:`Column`
-            Column representing whether each element of Column is aliased with new name or names.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.select(df.age.alias("age2")).collect()
-        [Row(age2=2), Row(age2=5)]
-        >>> df.select(df.age.alias("age3", metadata={'max': 99})).schema['age3'].metadata['max']
-        99
-        """
         metadata = kwargs.pop("metadata", None)
         assert not kwargs, "Unexpected kwargs where passed: %s" % kwargs
         return ColumnAlias(self, list(alias), metadata)
 
+    alias.__doc__ = PySparkColumn.alias.__doc__
+
     def name(self) -> str:
         ...
 
@@ -569,14 +542,6 @@ class LambdaFunction(Expression):
 
 
 class Column:
-    """
-    A column in a DataFrame. Column can refer to different things based on the
-    wrapped expression. Some common examples include attribute references, functions,
-    literals, etc.
-
-    .. versionadded:: 3.4.0
-    """
-
     def __init__(self, expr: Expression) -> None:
         if not isinstance(expr, Expression):
             raise TypeError(
@@ -602,64 +567,7 @@ class Column:
     __ge__ = _bin_op(">=")
     __le__ = _bin_op("<=")
 
-    _eqNullSafe_doc = """
-        Equality test that is safe for null values.
-
-        Parameters
-        ----------
-        other
-            a value or :class:`Column`
-
-        Examples
-        --------
-        >>> from pyspark.sql import Row
-        >>> df1 = spark.createDataFrame([
-        ...     Row(id=1, value='foo'),
-        ...     Row(id=2, value=None)
-        ... ])
-        >>> df1.select(
-        ...     df1['value'] == 'foo',
-        ...     df1['value'].eqNullSafe('foo'),
-        ...     df1['value'].eqNullSafe(None)
-        ... ).show()
-        +-------------+---------------+----------------+
-        |(value = foo)|(value <=> foo)|(value <=> NULL)|
-        +-------------+---------------+----------------+
-        |         true|           true|           false|
-        |         null|          false|            true|
-        +-------------+---------------+----------------+
-        >>> df2 = spark.createDataFrame([
-        ...     Row(value = 'bar'),
-        ...     Row(value = None)
-        ... ])
-        >>> df1.join(df2, df1["value"] == df2["value"]).count()
-        0
-        >>> df1.join(df2, df1["value"].eqNullSafe(df2["value"])).count()
-        1
-        >>> df2 = spark.createDataFrame([
-        ...     Row(id=1, value=float('NaN')),
-        ...     Row(id=2, value=42.0),
-        ...     Row(id=3, value=None)
-        ... ])
-        >>> df2.select(
-        ...     df2['value'].eqNullSafe(None),
-        ...     df2['value'].eqNullSafe(float('NaN')),
-        ...     df2['value'].eqNullSafe(42.0)
-        ... ).show()
-        +----------------+---------------+----------------+
-        |(value <=> NULL)|(value <=> NaN)|(value <=> 42.0)|
-        +----------------+---------------+----------------+
-        |           false|           true|           false|
-        |           false|          false|            true|
-        |            true|          false|           false|
-        +----------------+---------------+----------------+
-        Notes
-        -----
-        Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the
-        `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_
-        for details.
-        """
-    eqNullSafe = _bin_op("eqNullSafe", _eqNullSafe_doc)
+    eqNullSafe = _bin_op("eqNullSafe", PySparkColumn.eqNullSafe.__doc__)
 
     __neg__ = _func_op("negate")
 
@@ -672,68 +580,12 @@ class Column:
     __ror__ = _bin_op("or")
 
     # bitwise operators
-    _bitwiseOR_doc = """
-        Compute bitwise OR of this expression with another expression.
-
-        Parameters
-        ----------
-        other
-            a value or :class:`Column` to calculate bitwise or(|) with
-            this :class:`Column`.
-
-        Examples
-        --------
-        >>> from pyspark.sql import Row
-        >>> df = spark.createDataFrame([Row(a=170, b=75)])
-        >>> df.select(df.a.bitwiseOR(df.b)).collect()
-        [Row((a | b)=235)]
-        """
-    _bitwiseAND_doc = """
-        Compute bitwise AND of this expression with another expression.
-
-        Parameters
-        ----------
-        other
-            a value or :class:`Column` to calculate bitwise and(&) with
-            this :class:`Column`.
-
-        Examples
-        --------
-        >>> from pyspark.sql import Row
-        >>> df = spark.createDataFrame([Row(a=170, b=75)])
-        >>> df.select(df.a.bitwiseAND(df.b)).collect()
-        [Row((a & b)=10)]
-        """
-    _bitwiseXOR_doc = """
-        Compute bitwise XOR of this expression with another expression.
-
-        Parameters
-        ----------
-        other
-            a value or :class:`Column` to calculate bitwise xor(^) with
-            this :class:`Column`.
-
-        Examples
-        --------
-        >>> from pyspark.sql import Row
-        >>> df = spark.createDataFrame([Row(a=170, b=75)])
-        >>> df.select(df.a.bitwiseXOR(df.b)).collect()
-        [Row((a ^ b)=225)]
-        """
-
-    bitwiseOR = _bin_op("bitwiseOR", _bitwiseOR_doc)
-    bitwiseAND = _bin_op("bitwiseAND", _bitwiseAND_doc)
-    bitwiseXOR = _bin_op("bitwiseXOR", _bitwiseXOR_doc)
+    bitwiseOR = _bin_op("bitwiseOR", PySparkColumn.bitwiseOR.__doc__)
+    bitwiseAND = _bin_op("bitwiseAND", PySparkColumn.bitwiseAND.__doc__)
+    bitwiseXOR = _bin_op("bitwiseXOR", PySparkColumn.bitwiseXOR.__doc__)
 
-    _isNull_doc = """
-    True if the current expression is null.
-    """
-    _isNotNull_doc = """
-    True if the current expression is NOT null.
-    """
-
-    isNull = _unary_op("isNull", _isNull_doc)
-    isNotNull = _unary_op("isNotNull", _isNotNull_doc)
+    isNull = _unary_op("isNull", PySparkColumn.isNull.__doc__)
+    isNotNull = _unary_op("isNotNull", PySparkColumn.isNotNull.__doc__)
 
     def __ne__(  # type: ignore[override]
         self,
@@ -743,79 +595,11 @@ class Column:
         return _func_op("not")(_bin_op("==")(self, other))
 
     # string methods
-    def contains(self, other: Union["PrimitiveType", "Column"]) -> "Column":
-        """
-        Contains the other element. Returns a boolean :class:`Column` based on a string match.
-
-        Parameters
-        ----------
-        other
-            string in line. A value as a literal or a :class:`Column`.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.filter(df.name.contains('o')).collect()
-        [Row(age=5, name='Bob')]
-        """
-        return _bin_op("contains")(self, other)
-
-    _startswith_doc = """
-    String starts with. Returns a boolean :class:`Column` based on a string match.
-
-    Parameters
-    ----------
-    other : :class:`Column` or str
-        string at start of line (do not use a regex `^`)
-    """
-    _endswith_doc = """
-    String ends with. Returns a boolean :class:`Column` based on a string match.
-
-    Parameters
-    ----------
-    other : :class:`Column` or str
-        string at end of line (do not use a regex `$`)
-    """
-    startswith = _bin_op("startsWith", _startswith_doc)
-    endswith = _bin_op("endsWith", _endswith_doc)
+    contains = _bin_op("contains", PySparkColumn.contains.__doc__)
+    startswith = _bin_op("startsWith", PySparkColumn.startswith.__doc__)
+    endswith = _bin_op("endsWith", PySparkColumn.endswith.__doc__)
 
     def when(self, condition: "Column", value: Any) -> "Column":
-        """
-        Evaluates a list of conditions and returns one of multiple possible result expressions.
-        If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.
-
-        .. versionadded:: 3.4.0
-
-        Parameters
-        ----------
-        condition : :class:`Column`
-            a boolean :class:`Column` expression.
-        value
-            a literal value, or a :class:`Column` expression.
-
-        Returns
-        -------
-        :class:`Column`
-            Column representing whether each element of Column is in conditions.
-
-        Examples
-        --------
-        >>> from pyspark.sql import functions as F
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.select(df.name, F.when(df.age > 4, 1).when(df.age < 3, -1).otherwise(0)).show()
-        +-----+------------------------------------------------------------+
-        | name|CASE WHEN (age > 4) THEN 1 WHEN (age < 3) THEN -1 ELSE 0 END|
-        +-----+------------------------------------------------------------+
-        |Alice|                                                          -1|
-        |  Bob|                                                           1|
-        +-----+------------------------------------------------------------+
-
-        See Also
-        --------
-        pyspark.sql.functions.when
-        """
         if not isinstance(condition, Column):
             raise TypeError("condition should be a Column")
 
@@ -836,40 +620,9 @@ class Column:
 
         return Column(CaseWhen(branches=_branches, else_value=None))
 
+    when.__doc__ = PySparkColumn.when.__doc__
+
     def otherwise(self, value: Any) -> "Column":
-        """
-        Evaluates a list of conditions and returns one of multiple possible result expressions.
-        If :func:`Column.otherwise` is not invoked, None is returned for unmatched conditions.
-
-        .. versionadded:: 3.4.0
-
-        Parameters
-        ----------
-        value
-            a literal value, or a :class:`Column` expression.
-
-        Returns
-        -------
-        :class:`Column`
-            Column representing whether each element of Column is unmatched conditions.
-
-        Examples
-        --------
-        >>> from pyspark.sql import functions as F
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.select(df.name, F.when(df.age > 3, 1).otherwise(0)).show()
-        +-----+-------------------------------------+
-        | name|CASE WHEN (age > 3) THEN 1 ELSE 0 END|
-        +-----+-------------------------------------+
-        |Alice|                                    0|
-        |  Bob|                                    1|
-        +-----+-------------------------------------+
-
-        See Also
-        --------
-        pyspark.sql.functions.when
-        """
         if not isinstance(self._expr, CaseWhen):
             raise TypeError(
                 "otherwise() can only be applied on a Column previously generated by when()"
@@ -887,82 +640,11 @@ class Column:
 
         return Column(CaseWhen(branches=self._expr._branches, else_value=_value))
 
-    def like(self: "Column", other: str) -> "Column":
-        """
-        SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.
-
-        Parameters
-        ----------
-        other : str
-            a SQL LIKE pattern
-        See Also
-        --------
-        pyspark.sql.Column.rlike
-        Returns
-        -------
-        :class:`Column`
-            Column of booleans showing whether each element
-            in the Column is matched by SQL LIKE pattern.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.filter(df.name.like('Al%')).collect()
-        [Row(age=2, name='Alice')]
-        """
-        return _bin_op("like")(self, other)
-
-    def rlike(self: "Column", other: str) -> "Column":
-        """
-        SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex
-        match.
-
-        Parameters
-        ----------
-        other : str
-            an extended regex expression
-        Returns
-        -------
-        :class:`Column`
-            Column of booleans showing whether each element
-            in the Column is matched by extended regex expression.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.filter(df.name.rlike('ice$')).collect()
-        [Row(age=2, name='Alice')]
-        """
-        return _bin_op("like")(self, other)
+    otherwise.__doc__ = PySparkColumn.otherwise.__doc__
 
-    def ilike(self: "Column", other: str) -> "Column":
-        """
-        SQL ILIKE expression (case insensitive LIKE). Returns a boolean :class:`Column`
-        based on a case insensitive match.
-
-        Parameters
-        ----------
-        other : str
-            a SQL LIKE pattern
-        See Also
-        --------
-        pyspark.sql.Column.rlike
-        Returns
-        -------
-        :class:`Column`
-            Column of booleans showing whether each element
-            in the Column is matched by SQL LIKE pattern.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.filter(df.name.ilike('%Ice')).collect()
-        [Row(age=2, name='Alice')]
-        """
-        return _bin_op("ilike")(self, other)
+    like = _bin_op("like", PySparkColumn.like.__doc__)
+    rlike = _bin_op("rlike", PySparkColumn.rlike.__doc__)
+    ilike = _bin_op("ilike", PySparkColumn.ilike.__doc__)
 
     @overload
     def substr(self, startPos: int, length: int) -> "Column":
@@ -973,27 +655,6 @@ class Column:
         ...
 
     def substr(self, startPos: Union[int, "Column"], length: Union[int, "Column"]) -> "Column":
-        """
-        Return a :class:`Column` which is a substring of the column.
-
-        Parameters
-        ----------
-        startPos : :class:`Column` or int
-            start position
-        length : :class:`Column` or int
-            length of the substring
-        Returns
-        -------
-        :class:`Column`
-            Column representing whether each element of Column is substr of origin Column.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df.select(df.name.substr(1, 3).alias("col")).collect()
-        [Row(col='Ali'), Row(col='Bob')]
-        """
         from pyspark.sql.connect.function_builder import functions as F
         from pyspark.sql.connect.functions import lit
 
@@ -1020,6 +681,8 @@ class Column:
 
         return F.substr(self, start_exp, length_exp)
 
+    substr.__doc__ = PySparkColumn.substr.__doc__
+
     def __eq__(self, other: Any) -> "Column":  # type: ignore[override]
         """Returns a binary expression with the current column as the left
         side and the other expression as the right side.
@@ -1059,27 +722,13 @@ class Column:
         return self._expr.name()
 
     def cast(self, dataType: Union[DataType, str]) -> "Column":
-        """
-        Casts the column into type ``dataType``.
-
-        .. versionadded:: 3.4.0
-
-        Parameters
-        ----------
-        dataType : :class:`DataType` or str
-            a DataType or Python string literal with a DDL-formatted string
-            to use when parsing the column to the same type.
-
-        Returns
-        -------
-        :class:`Column`
-            Column representing whether each element of Column is cast into new type.
-        """
         if isinstance(dataType, (DataType, str)):
             return Column(CastExpression(col=self, data_type=dataType))
         else:
             raise TypeError("unexpected type: %s" % type(dataType))
 
+    cast.__doc__ = PySparkColumn.cast.__doc__
+
     def __repr__(self) -> str:
         return "Column<'%s'>" % self._expr.__repr__()
 
@@ -1087,31 +736,6 @@ class Column:
         raise NotImplementedError("over() is not yet implemented.")
 
     def isin(self, *cols: Any) -> "Column":
-        """
-        A boolean expression that is evaluated to true if the value of this
-        expression is contained by the evaluated values of the arguments.
-
-        .. versionadded:: 3.4.0
-
-        Parameters
-        ----------
-        cols
-            The result will only be true at a location if any value matches in the Column.
-
-        Returns
-        -------
-        :class:`Column`
-            Column of booleans showing whether each element in the Column is contained in cols.
-
-        Examples
-        --------
-        >>> df = spark.createDataFrame(
-        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-        >>> df[df.name.isin("Bob", "Mike")].collect()
-        [Row(age=5, name='Bob')]
-        >>> df[df.age.isin([1, 2, 3])].collect()
-        [Row(age=2, name='Alice')]
-        """
         from pyspark.sql.connect.functions import lit
 
         if len(cols) == 1 and isinstance(cols[0], (list, set)):
@@ -1121,6 +745,8 @@ class Column:
 
         return Column(UnresolvedFunction("in", [self._expr] + [lit(c)._expr for c in _cols]))
 
+    isin.__doc__ = PySparkColumn.isin.__doc__
+
     def getItem(self, *args: Any, **kwargs: Any) -> None:
         raise NotImplementedError("getItem() is not yet implemented.")
 
@@ -1144,3 +770,6 @@ class Column:
 
     def __iter__(self) -> None:
         raise TypeError("Column is not iterable")
+
+
+Column.__doc__ = PySparkColumn.__doc__


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org