You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/04/23 05:13:23 UTC
(spark) branch master updated: [SPARK-47933][PYTHON] Parent Column class for Spark Connect and Spark Classic
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e01ac581f46a [SPARK-47933][PYTHON] Parent Column class for Spark Connect and Spark Classic
e01ac581f46a is described below
commit e01ac581f46aa595e66daf33fe92b56d1328bc78
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Tue Apr 23 14:13:09 2024 +0900
[SPARK-47933][PYTHON] Parent Column class for Spark Connect and Spark Classic
### What changes were proposed in this pull request?
Same as https://github.com/apache/spark/pull/46129 but for `Column` class.
### Why are the changes needed?
Same as https://github.com/apache/spark/pull/46129
### Does this PR introduce _any_ user-facing change?
Same as https://github.com/apache/spark/pull/46129
### How was this patch tested?
Manually tested, and CI should verify them.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46155 from HyukjinKwon/SPARK-47933.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
dev/sparktestsupport/modules.py | 2 +-
python/pyspark/ml/connect/functions.py | 2 +-
python/pyspark/ml/functions.py | 2 +-
python/pyspark/ml/stat.py | 3 +-
python/pyspark/pandas/internal.py | 2 +-
python/pyspark/pandas/spark/functions.py | 44 +-
python/pyspark/sql/avro/functions.py | 4 +-
python/pyspark/sql/classic/column.py | 637 ++++++++++
python/pyspark/sql/classic/dataframe.py | 3 +-
python/pyspark/sql/column.py | 1211 +++++++++-----------
python/pyspark/sql/connect/_typing.py | 2 +-
python/pyspark/sql/connect/avro/functions.py | 3 +-
python/pyspark/sql/connect/column.py | 404 ++++---
python/pyspark/sql/connect/dataframe.py | 24 +-
python/pyspark/sql/connect/functions/builtin.py | 55 +-
.../pyspark/sql/connect/functions/partitioning.py | 2 +-
python/pyspark/sql/connect/group.py | 8 +-
python/pyspark/sql/connect/observation.py | 2 +-
python/pyspark/sql/connect/plan.py | 5 +-
python/pyspark/sql/connect/protobuf/functions.py | 2 +-
python/pyspark/sql/connect/udf.py | 5 +-
python/pyspark/sql/connect/udtf.py | 5 +-
python/pyspark/sql/connect/window.py | 6 +-
python/pyspark/sql/functions/builtin.py | 158 ++-
python/pyspark/sql/functions/partitioning.py | 4 +-
python/pyspark/sql/group.py | 6 +-
python/pyspark/sql/observation.py | 7 +-
python/pyspark/sql/protobuf/functions.py | 4 +-
python/pyspark/sql/readwriter.py | 16 +-
python/pyspark/sql/session.py | 2 +-
python/pyspark/sql/streaming/readwriter.py | 3 +-
.../sql/tests/pandas/test_pandas_udf_scalar.py | 2 +-
python/pyspark/sql/tests/test_column.py | 9 +-
python/pyspark/sql/udf.py | 4 +-
python/pyspark/sql/udtf.py | 3 +-
python/pyspark/sql/utils.py | 29 +-
python/pyspark/sql/window.py | 3 +-
37 files changed, 1766 insertions(+), 917 deletions(-)
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 593cae5902c8..8a94187bfced 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -476,7 +476,7 @@ pyspark_sql = Module(
"pyspark.sql.session",
"pyspark.sql.conf",
"pyspark.sql.catalog",
- "pyspark.sql.column",
+ "pyspark.sql.classic.column",
"pyspark.sql.classic.dataframe",
"pyspark.sql.datasource",
"pyspark.sql.group",
diff --git a/python/pyspark/ml/connect/functions.py b/python/pyspark/ml/connect/functions.py
index b305c04519ae..6597e6c4118a 100644
--- a/python/pyspark/ml/connect/functions.py
+++ b/python/pyspark/ml/connect/functions.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
from pyspark.ml import functions as PyMLFunctions
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col, lit
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index 466d94ccc888..25bbc290a3e9 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -28,7 +28,7 @@ except ImportError:
pass # Let it throw a better error message later when the API is invoked.
from pyspark.sql.functions import pandas_udf
-from pyspark.sql.column import Column, _to_java_column
+from pyspark.sql.classic.column import Column, _to_java_column
from pyspark.sql.types import (
ArrayType,
ByteType,
diff --git a/python/pyspark/ml/stat.py b/python/pyspark/ml/stat.py
index ec5da94079ea..d6020607aff2 100644
--- a/python/pyspark/ml/stat.py
+++ b/python/pyspark/ml/stat.py
@@ -22,7 +22,8 @@ from pyspark import since
from pyspark.ml.common import _java2py, _py2java
from pyspark.ml.linalg import Matrix, Vector
from pyspark.ml.wrapper import JavaWrapper, _jvm
-from pyspark.sql.column import Column, _to_seq
+from pyspark.sql.column import Column
+from pyspark.sql.classic.column import _to_seq
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.functions import lit
diff --git a/python/pyspark/pandas/internal.py b/python/pyspark/pandas/internal.py
index 2966db073d0c..767ec9a57f9b 100644
--- a/python/pyspark/pandas/internal.py
+++ b/python/pyspark/pandas/internal.py
@@ -959,7 +959,7 @@ class InternalFrame:
return sdf.select(
ConnectColumn(DistributedSequenceID()).alias(column_name),
- "*", # type: ignore[call-overload]
+ "*",
)
else:
return PySparkDataFrame(
diff --git a/python/pyspark/pandas/spark/functions.py b/python/pyspark/pandas/spark/functions.py
index a598ff37c47f..91602ae2b2b8 100644
--- a/python/pyspark/pandas/spark/functions.py
+++ b/python/pyspark/pandas/spark/functions.py
@@ -25,9 +25,9 @@ def product(col: Column, dropna: bool) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_product",
- col, # type: ignore[arg-type]
+ col,
lit(dropna),
)
@@ -42,9 +42,9 @@ def stddev(col: Column, ddof: int) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_stddev",
- col, # type: ignore[arg-type]
+ col,
lit(ddof),
)
@@ -59,9 +59,9 @@ def var(col: Column, ddof: int) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_var",
- col, # type: ignore[arg-type]
+ col,
lit(ddof),
)
@@ -76,9 +76,9 @@ def skew(col: Column) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_skew",
- col, # type: ignore[arg-type]
+ col,
)
else:
@@ -92,9 +92,9 @@ def kurt(col: Column) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_kurt",
- col, # type: ignore[arg-type]
+ col,
)
else:
@@ -108,9 +108,9 @@ def mode(col: Column, dropna: bool) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_mode",
- col, # type: ignore[arg-type]
+ col,
lit(dropna),
)
@@ -125,10 +125,10 @@ def covar(col1: Column, col2: Column, ddof: int) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"pandas_covar",
- col1, # type: ignore[arg-type]
- col2, # type: ignore[arg-type]
+ col1,
+ col2,
lit(ddof),
)
@@ -143,9 +143,9 @@ def ewm(col: Column, alpha: float, ignore_na: bool) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"ewm",
- col, # type: ignore[arg-type]
+ col,
lit(alpha),
lit(ignore_na),
)
@@ -161,9 +161,9 @@ def null_index(col: Column) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"null_index",
- col, # type: ignore[arg-type]
+ col,
)
else:
@@ -177,11 +177,11 @@ def timestampdiff(unit: str, start: Column, end: Column) -> Column:
if is_remote():
from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
- return _invoke_function_over_columns( # type: ignore[return-value]
+ return _invoke_function_over_columns(
"timestampdiff",
lit(unit),
- start, # type: ignore[arg-type]
- end, # type: ignore[arg-type]
+ start,
+ end,
)
else:
diff --git a/python/pyspark/sql/avro/functions.py b/python/pyspark/sql/avro/functions.py
index 5cebfa384045..40a4ce337db0 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -22,7 +22,7 @@ A collections of builtin avro functions
from typing import Dict, Optional, TYPE_CHECKING, cast
-from pyspark.sql.column import Column, _to_java_column
+from pyspark.sql.column import Column
from pyspark.sql.utils import get_active_spark_context, try_remote_avro_functions
from pyspark.util import _print_missing_jar
@@ -78,6 +78,7 @@ def from_avro(
[Row(value=Row(avro=Row(age=2, name='Alice')))]
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_java_column
sc = get_active_spark_context()
try:
@@ -128,6 +129,7 @@ def to_avro(data: "ColumnOrName", jsonFormatSchema: str = "") -> Column:
[Row(suite=bytearray(b'\\x02\\x00'))]
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_java_column
sc = get_active_spark_context()
try:
diff --git a/python/pyspark/sql/classic/column.py b/python/pyspark/sql/classic/column.py
new file mode 100644
index 000000000000..dc8a9a1c93b4
--- /dev/null
+++ b/python/pyspark/sql/classic/column.py
@@ -0,0 +1,637 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import json
+import warnings
+from typing import (
+ cast,
+ overload,
+ Any,
+ Callable,
+ Iterable,
+ List,
+ Optional,
+ Tuple,
+ TYPE_CHECKING,
+ Union,
+)
+
+from pyspark.sql.column import Column as ParentColumn
+from pyspark.errors import PySparkAttributeError, PySparkTypeError, PySparkValueError
+from pyspark.errors.utils import with_origin_to_class
+from pyspark.sql.types import DataType
+from pyspark.sql.utils import get_active_spark_context
+
+if TYPE_CHECKING:
+ from py4j.java_gateway import JavaObject
+ from pyspark.core.context import SparkContext
+ from pyspark.sql._typing import ColumnOrName, LiteralType, DecimalLiteral, DateTimeLiteral
+ from pyspark.sql.window import WindowSpec
+
+__all__ = ["Column"]
+
+
+def _create_column_from_literal(
+ literal: Union["LiteralType", "DecimalLiteral", "DateTimeLiteral", "ParentColumn"]
+) -> "JavaObject":
+ from py4j.java_gateway import JVMView
+
+ sc = get_active_spark_context()
+ return cast(JVMView, sc._jvm).functions.lit(literal)
+
+
+def _create_column_from_name(name: str) -> "JavaObject":
+ from py4j.java_gateway import JVMView
+
+ sc = get_active_spark_context()
+ return cast(JVMView, sc._jvm).functions.col(name)
+
+
+def _to_java_column(col: "ColumnOrName") -> "JavaObject":
+ if isinstance(col, Column):
+ jcol = col._jc
+ elif isinstance(col, str):
+ jcol = _create_column_from_name(col)
+ else:
+ raise PySparkTypeError(
+ error_class="NOT_COLUMN_OR_STR",
+ message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
+ )
+ return jcol
+
+
+def _to_java_expr(col: "ColumnOrName") -> "JavaObject":
+ return _to_java_column(col).expr()
+
+
+@overload
+def _to_seq(sc: "SparkContext", cols: Iterable["JavaObject"]) -> "JavaObject":
+ ...
+
+
+@overload
+def _to_seq(
+ sc: "SparkContext",
+ cols: Iterable["ColumnOrName"],
+ converter: Optional[Callable[["ColumnOrName"], "JavaObject"]],
+) -> "JavaObject":
+ ...
+
+
+def _to_seq(
+ sc: "SparkContext",
+ cols: Union[Iterable["ColumnOrName"], Iterable["JavaObject"]],
+ converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
+) -> "JavaObject":
+ """
+ Convert a list of Columns (or names) into a JVM Seq of Column.
+
+ An optional `converter` could be used to convert items in `cols`
+ into JVM Column objects.
+ """
+ if converter:
+ cols = [converter(c) for c in cols]
+ assert sc._jvm is not None
+ return sc._jvm.PythonUtils.toSeq(cols)
+
+
+def _to_list(
+ sc: "SparkContext",
+ cols: List["ColumnOrName"],
+ converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
+) -> "JavaObject":
+ """
+ Convert a list of Columns (or names) into a JVM (Scala) List of Columns.
+
+ An optional `converter` could be used to convert items in `cols`
+ into JVM Column objects.
+ """
+ if converter:
+ cols = [converter(c) for c in cols]
+ assert sc._jvm is not None
+ return sc._jvm.PythonUtils.toList(cols)
+
+
+def _unary_op(name: str, self: ParentColumn) -> ParentColumn:
+ """Create a method for given unary operator"""
+
+ jc = getattr(self._jc, name)()
+ return Column(jc)
+
+
+def _func_op(name: str, self: ParentColumn) -> ParentColumn:
+ from py4j.java_gateway import JVMView
+
+ sc = get_active_spark_context()
+ jc = getattr(cast(JVMView, sc._jvm).functions, name)(self._jc)
+ return Column(jc)
+
+
+def _bin_func_op(
+ name: str,
+ self: ParentColumn,
+ other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+ reverse: bool = False,
+) -> ParentColumn:
+ from py4j.java_gateway import JVMView
+
+ sc = get_active_spark_context()
+ fn = getattr(cast(JVMView, sc._jvm).functions, name)
+ jc = other._jc if isinstance(other, ParentColumn) else _create_column_from_literal(other)
+ njc = fn(self._jc, jc) if not reverse else fn(jc, self._jc)
+ return Column(njc)
+
+
+def _bin_op(
+ name: str,
+ self: ParentColumn,
+ other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+) -> ParentColumn:
+ """Create a method for given binary operator"""
+ jc = other._jc if isinstance(other, ParentColumn) else other
+ njc = getattr(self._jc, name)(jc)
+ return Column(njc)
+
+
+def _reverse_op(
+ name: str,
+ self: ParentColumn,
+ other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+) -> ParentColumn:
+ """Create a method for binary operator (this object is on right side)"""
+ jother = _create_column_from_literal(other)
+ jc = getattr(jother, name)(self._jc)
+ return Column(jc)
+
+
+@with_origin_to_class
+class Column(ParentColumn):
+ def __new__(
+ cls,
+ jc: "JavaObject",
+ ) -> "Column":
+ self = object.__new__(cls)
+ self.__init__(jc) # type: ignore[misc]
+ return self
+
+ def __init__(self, jc: "JavaObject") -> None:
+ self._jc = jc
+
+ # arithmetic operators
+ def __neg__(self) -> ParentColumn:
+ return _func_op("negate", self)
+
+ def __add__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("plus", self, other)
+
+ def __sub__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("minus", self, other)
+
+ def __mul__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("multiply", self, other)
+
+ def __div__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("divide", self, other)
+
+ def __truediv__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("divide", self, other)
+
+ def __mod__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("mod", self, other)
+
+ def __radd__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("plus", self, other)
+
+ def __rsub__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _reverse_op("minus", self, other)
+
+ def __rmul__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("multiply", self, other)
+
+ def __rdiv__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _reverse_op("divide", self, other)
+
+ def __rtruediv__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _reverse_op("divide", self, other)
+
+ def __rmod__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _reverse_op("mod", self, other)
+
+ def __pow__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_func_op("pow", self, other)
+
+ def __rpow__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_func_op("pow", self, other, reverse=True)
+
+ # logistic operators
+ def __eq__( # type: ignore[override]
+ self,
+ other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+ ) -> ParentColumn:
+ return _bin_op("equalTo", self, other)
+
+ def __ne__( # type: ignore[override]
+ self,
+ other: Any,
+ ) -> ParentColumn:
+ return _bin_op("notEqual", self, other)
+
+ def __lt__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("lt", self, other)
+
+ def __le__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("leq", self, other)
+
+ def __ge__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("geq", self, other)
+
+ def __gt__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("gt", self, other)
+
+ def eqNullSafe(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("eqNullSafe", self, other)
+
+ # `and`, `or`, `not` cannot be overloaded in Python,
+ # so use bitwise operators as boolean operators
+ def __and__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("and", self, other)
+
+ def __or__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("or", self, other)
+
+ def __invert__(self) -> ParentColumn:
+ return _func_op("not", self)
+
+ def __rand__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("and", self, other)
+
+ def __ror__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("or", self, other)
+
+ # container operators
+ def __contains__(self, item: Any) -> None:
+ raise PySparkValueError(
+ error_class="CANNOT_APPLY_IN_FOR_COLUMN",
+ message_parameters={},
+ )
+
+ # bitwise operators
+ def bitwiseOR(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("bitwiseOR", self, other)
+
+ def bitwiseAND(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("bitwiseAND", self, other)
+
+ def bitwiseXOR(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("bitwiseXOR", self, other)
+
+ def getItem(self, key: Any) -> ParentColumn:
+ if isinstance(key, Column):
+ warnings.warn(
+ "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
+ "be supported in the future release. Use `column[key]` or `column.key` syntax "
+ "instead.",
+ FutureWarning,
+ )
+ return self[key]
+
+ def getField(self, name: Any) -> ParentColumn:
+ if isinstance(name, Column):
+ warnings.warn(
+ "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
+ "be supported in the future release. Use `column[name]` or `column.name` syntax "
+ "instead.",
+ FutureWarning,
+ )
+ return self[name]
+
+ def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:
+ if not isinstance(fieldName, str):
+ raise PySparkTypeError(
+ error_class="NOT_STR",
+ message_parameters={"arg_name": "fieldName", "arg_type": type(fieldName).__name__},
+ )
+
+ if not isinstance(col, Column):
+ raise PySparkTypeError(
+ error_class="NOT_COLUMN",
+ message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
+ )
+
+ return Column(self._jc.withField(fieldName, col._jc))
+
+ def dropFields(self, *fieldNames: str) -> ParentColumn:
+ sc = get_active_spark_context()
+ jc = self._jc.dropFields(_to_seq(sc, fieldNames))
+ return Column(jc)
+
+ def __getattr__(self, item: Any) -> ParentColumn:
+ if item.startswith("__"):
+ raise PySparkAttributeError(
+ error_class="CANNOT_ACCESS_TO_DUNDER",
+ message_parameters={},
+ )
+ return self[item]
+
+ def __getitem__(self, k: Any) -> ParentColumn:
+ if isinstance(k, slice):
+ if k.step is not None:
+ raise PySparkValueError(
+ error_class="SLICE_WITH_STEP",
+ message_parameters={},
+ )
+ return self.substr(k.start, k.stop)
+ else:
+ return _bin_op("apply", self, k)
+
+ def __iter__(self) -> None:
+ raise PySparkTypeError(
+ error_class="NOT_ITERABLE", message_parameters={"objectName": "Column"}
+ )
+
+ # string methods
+ def contains(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("contains", self, other)
+
+ def startswith(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("startsWith", self, other)
+
+ def endswith(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("endsWith", self, other)
+
+ def like(self: ParentColumn, other: str) -> ParentColumn:
+ njc = getattr(self._jc, "like")(other)
+ return Column(njc)
+
+ def rlike(self: ParentColumn, other: str) -> ParentColumn:
+ njc = getattr(self._jc, "rlike")(other)
+ return Column(njc)
+
+ def ilike(self: ParentColumn, other: str) -> ParentColumn:
+ njc = getattr(self._jc, "ilike")(other)
+ return Column(njc)
+
+ def substr(
+ self, startPos: Union[int, ParentColumn], length: Union[int, ParentColumn]
+ ) -> ParentColumn:
+ if type(startPos) != type(length):
+ raise PySparkTypeError(
+ error_class="NOT_SAME_TYPE",
+ message_parameters={
+ "arg_name1": "startPos",
+ "arg_name2": "length",
+ "arg_type1": type(startPos).__name__,
+ "arg_type2": type(length).__name__,
+ },
+ )
+ if isinstance(startPos, int):
+ jc = self._jc.substr(startPos, length)
+ elif isinstance(startPos, Column):
+ jc = self._jc.substr(startPos._jc, cast(ParentColumn, length)._jc)
+ else:
+ raise PySparkTypeError(
+ error_class="NOT_COLUMN_OR_INT",
+ message_parameters={"arg_name": "startPos", "arg_type": type(startPos).__name__},
+ )
+ return Column(jc)
+
+ def isin(self, *cols: Any) -> ParentColumn:
+ if len(cols) == 1 and isinstance(cols[0], (list, set)):
+ cols = cast(Tuple, cols[0])
+ cols = cast(
+ Tuple,
+ [c._jc if isinstance(c, Column) else _create_column_from_literal(c) for c in cols],
+ )
+ sc = get_active_spark_context()
+ jc = getattr(self._jc, "isin")(_to_seq(sc, cols))
+ return Column(jc)
+
+ # order
+ def asc(self) -> ParentColumn:
+ return _unary_op("asc", self)
+
+ def asc_nulls_first(self) -> ParentColumn:
+ return _unary_op("asc_nulls_first", self)
+
+ def asc_nulls_last(self) -> ParentColumn:
+ return _unary_op("asc_nulls_last", self)
+
+ def desc(self) -> ParentColumn:
+ return _unary_op("desc", self)
+
+ def desc_nulls_first(self) -> ParentColumn:
+ return _unary_op("desc_nulls_first", self)
+
+ def desc_nulls_last(self) -> ParentColumn:
+ return _unary_op("desc_nulls_last", self)
+
+ def isNull(self) -> ParentColumn:
+ return _unary_op("isNull", self)
+
+ def isNotNull(self) -> ParentColumn:
+ return _unary_op("isNotNull", self)
+
+ def isNaN(self) -> ParentColumn:
+ return _unary_op("isNaN", self)
+
+ def alias(self, *alias: str, **kwargs: Any) -> ParentColumn:
+ metadata = kwargs.pop("metadata", None)
+ assert not kwargs, "Unexpected kwargs where passed: %s" % kwargs
+
+ sc = get_active_spark_context()
+ if len(alias) == 1:
+ if metadata:
+ assert sc._jvm is not None
+ jmeta = sc._jvm.org.apache.spark.sql.types.Metadata.fromJson(json.dumps(metadata))
+ return Column(getattr(self._jc, "as")(alias[0], jmeta))
+ else:
+ return Column(getattr(self._jc, "as")(alias[0]))
+ else:
+ if metadata is not None:
+ raise PySparkValueError(
+ error_class="ONLY_ALLOWED_FOR_SINGLE_COLUMN",
+ message_parameters={"arg_name": "metadata"},
+ )
+ return Column(getattr(self._jc, "as")(_to_seq(sc, list(alias))))
+
+ def name(self, *alias: str, **kwargs: Any) -> ParentColumn:
+ return self.alias(*alias, **kwargs)
+
+ def cast(self, dataType: Union[DataType, str]) -> ParentColumn:
+ if isinstance(dataType, str):
+ jc = self._jc.cast(dataType)
+ elif isinstance(dataType, DataType):
+ from pyspark.sql import SparkSession
+
+ spark = SparkSession._getActiveSessionOrCreate()
+ jdt = spark._jsparkSession.parseDataType(dataType.json())
+ jc = self._jc.cast(jdt)
+ else:
+ raise PySparkTypeError(
+ error_class="NOT_DATATYPE_OR_STR",
+ message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+ )
+ return Column(jc)
+
+ def try_cast(self, dataType: Union[DataType, str]) -> ParentColumn:
+ if isinstance(dataType, str):
+ jc = self._jc.try_cast(dataType)
+ elif isinstance(dataType, DataType):
+ from pyspark.sql import SparkSession
+
+ spark = SparkSession._getActiveSessionOrCreate()
+ jdt = spark._jsparkSession.parseDataType(dataType.json())
+ jc = self._jc.try_cast(jdt)
+ else:
+ raise PySparkTypeError(
+ error_class="NOT_DATATYPE_OR_STR",
+ message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+ )
+ return Column(jc)
+
+ def astype(self, dataType: Union[DataType, str]) -> ParentColumn:
+ return self.cast(dataType)
+
+ def between(
+ self,
+ lowerBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+ upperBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+ ) -> ParentColumn:
+ return (self >= lowerBound) & (self <= upperBound)
+
+ def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
+ if not isinstance(condition, Column):
+ raise PySparkTypeError(
+ error_class="NOT_COLUMN",
+ message_parameters={"arg_name": "condition", "arg_type": type(condition).__name__},
+ )
+ v = value._jc if isinstance(value, Column) else value
+ jc = self._jc.when(condition._jc, v)
+ return Column(jc)
+
+ def otherwise(self, value: Any) -> ParentColumn:
+ v = value._jc if isinstance(value, Column) else value
+ jc = self._jc.otherwise(v)
+ return Column(jc)
+
+ def over(self, window: "WindowSpec") -> ParentColumn:
+ from pyspark.sql.window import WindowSpec
+
+ if not isinstance(window, WindowSpec):
+ raise PySparkTypeError(
+ error_class="NOT_WINDOWSPEC",
+ message_parameters={"arg_name": "window", "arg_type": type(window).__name__},
+ )
+ jc = self._jc.over(window._jspec)
+ return Column(jc)
+
+ def __nonzero__(self) -> None:
+ raise PySparkValueError(
+ error_class="CANNOT_CONVERT_COLUMN_INTO_BOOL",
+ message_parameters={},
+ )
+
+ __bool__ = __nonzero__
+
+ def __repr__(self) -> str:
+ return "Column<'%s'>" % self._jc.toString()
+
+
+def _test() -> None:
+ import doctest
+ from pyspark.sql import SparkSession
+ import pyspark.sql.column
+
+ # It inherits docstrings but doctests cannot detect them so we run
+ # the parent classe's doctests here directly.
+ globs = pyspark.sql.column.__dict__.copy()
+ spark = (
+ SparkSession.builder.master("local[4]").appName("sql.classic.column tests").getOrCreate()
+ )
+ globs["spark"] = spark
+
+ (failure_count, test_count) = doctest.testmod(
+ pyspark.sql.column,
+ globs=globs,
+ optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
+ )
+ spark.stop()
+ if failure_count:
+ sys.exit(-1)
+
+
+if __name__ == "__main__":
+ _test()
diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py
index b8b07356bfe9..db9f22517dda 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -54,7 +54,8 @@ from pyspark.util import (
from pyspark.serializers import BatchedSerializer, CPickleSerializer, UTF8Deserializer
from pyspark.storagelevel import StorageLevel
from pyspark.traceback_utils import SCCallSiteSync
-from pyspark.sql.column import Column, _to_seq, _to_list, _to_java_column
+from pyspark.sql.column import Column
+from pyspark.sql.classic.column import _to_seq, _to_list, _to_java_column
from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2
from pyspark.sql.streaming import DataStreamWriter
from pyspark.sql.types import (
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 2e79e30285ba..4ea621b626bb 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -15,197 +15,28 @@
# limitations under the License.
#
+# mypy: disable-error-code="empty-body"
+
import sys
-import json
-import warnings
from typing import (
- cast,
overload,
Any,
- Callable,
- Iterable,
- List,
- Optional,
- Tuple,
TYPE_CHECKING,
Union,
)
-from pyspark.errors import PySparkAttributeError, PySparkTypeError, PySparkValueError
-from pyspark.errors.utils import with_origin_to_class
+from pyspark.sql.utils import dispatch_col_method
from pyspark.sql.types import DataType
-from pyspark.sql.utils import get_active_spark_context
+from pyspark.errors import PySparkValueError
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
- from pyspark.core.context import SparkContext
- from pyspark.sql._typing import ColumnOrName, LiteralType, DecimalLiteral, DateTimeLiteral
+ from pyspark.sql._typing import LiteralType, DecimalLiteral, DateTimeLiteral
from pyspark.sql.window import WindowSpec
__all__ = ["Column"]
-def _create_column_from_literal(literal: Union["LiteralType", "DecimalLiteral"]) -> "Column":
- from py4j.java_gateway import JVMView
-
- sc = get_active_spark_context()
- return cast(JVMView, sc._jvm).functions.lit(literal)
-
-
-def _create_column_from_name(name: str) -> "Column":
- from py4j.java_gateway import JVMView
-
- sc = get_active_spark_context()
- return cast(JVMView, sc._jvm).functions.col(name)
-
-
-def _to_java_column(col: "ColumnOrName") -> "JavaObject":
- if isinstance(col, Column):
- jcol = col._jc
- elif isinstance(col, str):
- jcol = _create_column_from_name(col)
- else:
- raise PySparkTypeError(
- error_class="NOT_COLUMN_OR_STR",
- message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
- )
- return jcol
-
-
-def _to_java_expr(col: "ColumnOrName") -> "JavaObject":
- return _to_java_column(col).expr()
-
-
-@overload
-def _to_seq(sc: "SparkContext", cols: Iterable["JavaObject"]) -> "JavaObject":
- ...
-
-
-@overload
-def _to_seq(
- sc: "SparkContext",
- cols: Iterable["ColumnOrName"],
- converter: Optional[Callable[["ColumnOrName"], "JavaObject"]],
-) -> "JavaObject":
- ...
-
-
-def _to_seq(
- sc: "SparkContext",
- cols: Union[Iterable["ColumnOrName"], Iterable["JavaObject"]],
- converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
-) -> "JavaObject":
- """
- Convert a list of Columns (or names) into a JVM Seq of Column.
-
- An optional `converter` could be used to convert items in `cols`
- into JVM Column objects.
- """
- if converter:
- cols = [converter(c) for c in cols]
- assert sc._jvm is not None
- return sc._jvm.PythonUtils.toSeq(cols)
-
-
-def _to_list(
- sc: "SparkContext",
- cols: List["ColumnOrName"],
- converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
-) -> "JavaObject":
- """
- Convert a list of Columns (or names) into a JVM (Scala) List of Columns.
-
- An optional `converter` could be used to convert items in `cols`
- into JVM Column objects.
- """
- if converter:
- cols = [converter(c) for c in cols]
- assert sc._jvm is not None
- return sc._jvm.PythonUtils.toList(cols)
-
-
-def _unary_op(
- name: str,
- doc: str = "unary operator",
-) -> Callable[["Column"], "Column"]:
- """Create a method for given unary operator"""
-
- def _(self: "Column") -> "Column":
- jc = getattr(self._jc, name)()
- return Column(jc)
-
- _.__doc__ = doc
- return _
-
-
-def _func_op(name: str, doc: str = "") -> Callable[["Column"], "Column"]:
- def _(self: "Column") -> "Column":
- from py4j.java_gateway import JVMView
-
- sc = get_active_spark_context()
- jc = getattr(cast(JVMView, sc._jvm).functions, name)(self._jc)
- return Column(jc)
-
- _.__doc__ = doc
- return _
-
-
-def _bin_func_op(
- name: str,
- reverse: bool = False,
- doc: str = "binary function",
-) -> Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"]:
- def _(self: "Column", other: Union["Column", "LiteralType", "DecimalLiteral"]) -> "Column":
- from py4j.java_gateway import JVMView
-
- sc = get_active_spark_context()
- fn = getattr(cast(JVMView, sc._jvm).functions, name)
- jc = other._jc if isinstance(other, Column) else _create_column_from_literal(other)
- njc = fn(self._jc, jc) if not reverse else fn(jc, self._jc)
- return Column(njc)
-
- _.__doc__ = doc
- return _
-
-
-def _bin_op(
- name: str,
- doc: str = "binary operator",
-) -> Callable[
- ["Column", Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]], "Column"
-]:
- """Create a method for given binary operator"""
-
- def _(
- self: "Column",
- other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
- ) -> "Column":
- jc = other._jc if isinstance(other, Column) else other
- njc = getattr(self._jc, name)(jc)
- return Column(njc)
-
- _.__doc__ = doc
- _.__name__ = name
- return _
-
-
-def _reverse_op(
- name: str,
- doc: str = "binary operator",
-) -> Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"]:
- """Create a method for binary operator (this object is on right side)"""
-
- def _(self: "Column", other: Union["LiteralType", "DecimalLiteral"]) -> "Column":
- jother = _create_column_from_literal(other)
- jc = getattr(jother, name)(self._jc)
- return Column(jc)
-
- _.__doc__ = doc
- _.__name__ = name
- return _
-
-
-@with_origin_to_class
class Column:
"""
@@ -237,146 +68,251 @@ class Column:
Column<...>
"""
+ # HACK ALERT!! this is to reduce the backward compatibility concern, and returns
+ # Spark Classic Column by default. This is NOT an API, and NOT supposed to
+ # be directly invoked. DO NOT use this constructor.
+ def __new__(
+ cls,
+ jc: "JavaObject",
+ ) -> "Column":
+ from pyspark.sql.classic.column import Column
+
+ return Column.__new__(Column, jc)
+
def __init__(self, jc: "JavaObject") -> None:
self._jc = jc
# arithmetic operators
- __neg__ = _func_op("negate")
- __add__ = cast(
- Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
- _bin_op("plus"),
- )
- __sub__ = cast(
- Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
- _bin_op("minus"),
- )
- __mul__ = cast(
- Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
- _bin_op("multiply"),
- )
- __div__ = cast(
- Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
- _bin_op("divide"),
- )
- __truediv__ = cast(
- Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
- _bin_op("divide"),
- )
- __mod__ = cast(
- Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
- _bin_op("mod"),
- )
- __radd__ = cast(
- Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"], _bin_op("plus")
- )
- __rsub__ = _reverse_op("minus")
- __rmul__ = cast(
- Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"], _bin_op("multiply")
- )
- __rdiv__ = _reverse_op("divide")
- __rtruediv__ = _reverse_op("divide")
- __rmod__ = _reverse_op("mod")
-
- __pow__ = _bin_func_op("pow")
- __rpow__ = cast(
- Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"],
- _bin_func_op("pow", reverse=True),
- )
+ @dispatch_col_method
+ def __neg__(self) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __add__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __sub__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __mul__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __div__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __truediv__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __mod__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __radd__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rsub__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rmul__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rdiv__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rtruediv__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rmod__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __pow__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rpow__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
# logistic operators
+ @dispatch_col_method
def __eq__( # type: ignore[override]
self,
other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
) -> "Column":
"""binary function"""
- return _bin_op("equalTo")(self, other)
+ ...
+ @dispatch_col_method
def __ne__( # type: ignore[override]
self,
other: Any,
) -> "Column":
"""binary function"""
- return _bin_op("notEqual")(self, other)
+ ...
- __lt__ = _bin_op("lt")
- __le__ = _bin_op("leq")
- __ge__ = _bin_op("geq")
- __gt__ = _bin_op("gt")
+ @dispatch_col_method
+ def __lt__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
- _eqNullSafe_doc = """
- Equality test that is safe for null values.
+ @dispatch_col_method
+ def __le__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
- .. versionadded:: 2.3.0
+ @dispatch_col_method
+ def __ge__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ @dispatch_col_method
+ def __gt__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def eqNullSafe(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """
+ Equality test that is safe for null values.
- Parameters
- ----------
- other
- a value or :class:`Column`
+ .. versionadded:: 2.3.0
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df1 = spark.createDataFrame([
- ... Row(id=1, value='foo'),
- ... Row(id=2, value=None)
- ... ])
- >>> df1.select(
- ... df1['value'] == 'foo',
- ... df1['value'].eqNullSafe('foo'),
- ... df1['value'].eqNullSafe(None)
- ... ).show()
- +-------------+---------------+----------------+
- |(value = foo)|(value <=> foo)|(value <=> NULL)|
- +-------------+---------------+----------------+
- | true| true| false|
- | NULL| false| true|
- +-------------+---------------+----------------+
- >>> df2 = spark.createDataFrame([
- ... Row(value = 'bar'),
- ... Row(value = None)
- ... ])
- >>> df1.join(df2, df1["value"] == df2["value"]).count()
- 0
- >>> df1.join(df2, df1["value"].eqNullSafe(df2["value"])).count()
- 1
- >>> df2 = spark.createDataFrame([
- ... Row(id=1, value=float('NaN')),
- ... Row(id=2, value=42.0),
- ... Row(id=3, value=None)
- ... ])
- >>> df2.select(
- ... df2['value'].eqNullSafe(None),
- ... df2['value'].eqNullSafe(float('NaN')),
- ... df2['value'].eqNullSafe(42.0)
- ... ).show()
- +----------------+---------------+----------------+
- |(value <=> NULL)|(value <=> NaN)|(value <=> 42.0)|
- +----------------+---------------+----------------+
- | false| true| false|
- | false| false| true|
- | true| false| false|
- +----------------+---------------+----------------+
-
- Notes
- -----
- Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the
- `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_
- for details.
- """
- eqNullSafe = _bin_op("eqNullSafe", _eqNullSafe_doc)
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
+
+ Parameters
+ ----------
+ other
+ a value or :class:`Column`
+
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df1 = spark.createDataFrame([
+ ... Row(id=1, value='foo'),
+ ... Row(id=2, value=None)
+ ... ])
+ >>> df1.select(
+ ... df1['value'] == 'foo',
+ ... df1['value'].eqNullSafe('foo'),
+ ... df1['value'].eqNullSafe(None)
+ ... ).show()
+ +-------------+---------------+----------------+
+ |(value = foo)|(value <=> foo)|(value <=> NULL)|
+ +-------------+---------------+----------------+
+ | true| true| false|
+ | NULL| false| true|
+ +-------------+---------------+----------------+
+ >>> df2 = spark.createDataFrame([
+ ... Row(value = 'bar'),
+ ... Row(value = None)
+ ... ])
+ >>> df1.join(df2, df1["value"] == df2["value"]).count()
+ 0
+ >>> df1.join(df2, df1["value"].eqNullSafe(df2["value"])).count()
+ 1
+ >>> df2 = spark.createDataFrame([
+ ... Row(id=1, value=float('NaN')),
+ ... Row(id=2, value=42.0),
+ ... Row(id=3, value=None)
+ ... ])
+ >>> df2.select(
+ ... df2['value'].eqNullSafe(None),
+ ... df2['value'].eqNullSafe(float('NaN')),
+ ... df2['value'].eqNullSafe(42.0)
+ ... ).show()
+ +----------------+---------------+----------------+
+ |(value <=> NULL)|(value <=> NaN)|(value <=> 42.0)|
+ +----------------+---------------+----------------+
+ | false| true| false|
+ | false| false| true|
+ | true| false| false|
+ +----------------+---------------+----------------+
+
+ Notes
+ -----
+ Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the
+ `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_
+ for details.
+ """
+ ...
# `and`, `or`, `not` cannot be overloaded in Python,
# so use bitwise operators as boolean operators
- __and__ = _bin_op("and")
- __or__ = _bin_op("or")
- __invert__ = _func_op("not")
- __rand__ = _bin_op("and")
- __ror__ = _bin_op("or")
+ @dispatch_col_method
+ def __and__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __or__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __invert__(self) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __rand__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
+
+ @dispatch_col_method
+ def __ror__(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ ...
# container operators
+ @dispatch_col_method
def __contains__(self, item: Any) -> None:
raise PySparkValueError(
error_class="CANNOT_APPLY_IN_FOR_COLUMN",
@@ -384,68 +320,82 @@ class Column:
)
# bitwise operators
- _bitwiseOR_doc = """
- Compute bitwise OR of this expression with another expression.
+ @dispatch_col_method
+ def bitwiseOR(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """ "
+ Compute bitwise OR of this expression with another expression.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- Parameters
- ----------
- other
- a value or :class:`Column` to calculate bitwise or(|) with
- this :class:`Column`.
+ Parameters
+ ----------
+ other
+ a value or :class:`Column` to calculate bitwise or(|) with
+ this :class:`Column`.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(a=170, b=75)])
- >>> df.select(df.a.bitwiseOR(df.b)).collect()
- [Row((a | b)=235)]
- """
- _bitwiseAND_doc = """
- Compute bitwise AND of this expression with another expression.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([Row(a=170, b=75)])
+ >>> df.select(df.a.bitwiseOR(df.b)).collect()
+ [Row((a | b)=235)]
+ """
+ ...
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ @dispatch_col_method
+ def bitwiseAND(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """
+ Compute bitwise AND of this expression with another expression.
- Parameters
- ----------
- other
- a value or :class:`Column` to calculate bitwise and(&) with
- this :class:`Column`.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(a=170, b=75)])
- >>> df.select(df.a.bitwiseAND(df.b)).collect()
- [Row((a & b)=10)]
- """
- _bitwiseXOR_doc = """
- Compute bitwise XOR of this expression with another expression.
+ Parameters
+ ----------
+ other
+ a value or :class:`Column` to calculate bitwise and(&) with
+ this :class:`Column`.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([Row(a=170, b=75)])
+ >>> df.select(df.a.bitwiseAND(df.b)).collect()
+ [Row((a & b)=10)]
+ """
+ ...
- Parameters
- ----------
- other
- a value or :class:`Column` to calculate bitwise xor(^) with
- this :class:`Column`.
+ @dispatch_col_method
+ def bitwiseXOR(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """
+ Compute bitwise XOR of this expression with another expression.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(a=170, b=75)])
- >>> df.select(df.a.bitwiseXOR(df.b)).collect()
- [Row((a ^ b)=225)]
- """
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- bitwiseOR = _bin_op("bitwiseOR", _bitwiseOR_doc)
- bitwiseAND = _bin_op("bitwiseAND", _bitwiseAND_doc)
- bitwiseXOR = _bin_op("bitwiseXOR", _bitwiseXOR_doc)
+ Parameters
+ ----------
+ other
+ a value or :class:`Column` to calculate bitwise xor(^) with
+ this :class:`Column`.
+
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([Row(a=170, b=75)])
+ >>> df.select(df.a.bitwiseXOR(df.b)).collect()
+ [Row((a ^ b)=225)]
+ """
+ ...
+ @dispatch_col_method
def getItem(self, key: Any) -> "Column":
"""
An expression that gets an item at position ``ordinal`` out of a list,
@@ -480,15 +430,9 @@ class Column:
| 1| value|
+----+------+
"""
- if isinstance(key, Column):
- warnings.warn(
- "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
- "be supported in the future release. Use `column[key]` or `column.key` syntax "
- "instead.",
- FutureWarning,
- )
- return self[key]
+ ...
+ @dispatch_col_method
def getField(self, name: Any) -> "Column":
"""
An expression that gets a field by name in a :class:`StructType`.
@@ -528,15 +472,9 @@ class Column:
| 1|
+---+
"""
- if isinstance(name, Column):
- warnings.warn(
- "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
- "be supported in the future release. Use `column[name]` or `column.name` syntax "
- "instead.",
- FutureWarning,
- )
- return self[name]
+ ...
+ @dispatch_col_method
def withField(self, fieldName: str, col: "Column") -> "Column":
"""
An expression that adds/replaces a field in :class:`StructType` by name.
@@ -578,20 +516,9 @@ class Column:
| 4|
+---+
"""
- if not isinstance(fieldName, str):
- raise PySparkTypeError(
- error_class="NOT_STR",
- message_parameters={"arg_name": "fieldName", "arg_type": type(fieldName).__name__},
- )
-
- if not isinstance(col, Column):
- raise PySparkTypeError(
- error_class="NOT_COLUMN",
- message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
- )
-
- return Column(self._jc.withField(fieldName, col._jc))
+ ...
+ @dispatch_col_method
def dropFields(self, *fieldNames: str) -> "Column":
"""
An expression that drops fields in :class:`StructType` by name.
@@ -656,10 +583,9 @@ class Column:
+--------------+
"""
- sc = get_active_spark_context()
- jc = self._jc.dropFields(_to_seq(sc, fieldNames))
- return Column(jc)
+ ...
+ @dispatch_col_method
def __getattr__(self, item: Any) -> "Column":
"""
An expression that gets an item at position ``ordinal`` out of a list,
@@ -690,13 +616,9 @@ class Column:
| value|
+------+
"""
- if item.startswith("__"):
- raise PySparkAttributeError(
- error_class="CANNOT_ACCESS_TO_DUNDER",
- message_parameters={},
- )
- return self[item]
+ ...
+ @dispatch_col_method
def __getitem__(self, k: Any) -> "Column":
"""
An expression that gets an item at position ``ordinal`` out of a list,
@@ -728,85 +650,90 @@ class Column:
| abc| value|
+---------------+------+
"""
- if isinstance(k, slice):
- if k.step is not None:
- raise PySparkValueError(
- error_class="SLICE_WITH_STEP",
- message_parameters={},
- )
- return self.substr(k.start, k.stop)
- else:
- return _bin_op("apply")(self, k)
+ ...
+ @dispatch_col_method
def __iter__(self) -> None:
- raise PySparkTypeError(
- error_class="NOT_ITERABLE", message_parameters={"objectName": "Column"}
- )
+ ...
# string methods
- _contains_doc = """
- Contains the other element. Returns a boolean :class:`Column` based on a string match.
+ @dispatch_col_method
+ def contains(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """
+ Contains the other element. Returns a boolean :class:`Column` based on a string match.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- Parameters
- ----------
- other
- string in line. A value as a literal or a :class:`Column`.
+ Parameters
+ ----------
+ other
+ string in line. A value as a literal or a :class:`Column`.
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(2, "Alice"), (5, "Bob")], ["age", "name"])
- >>> df.filter(df.name.contains('o')).collect()
- [Row(age=5, name='Bob')]
- """
- _startswith_doc = """
- String starts with. Returns a boolean :class:`Column` based on a string match.
+ Examples
+ --------
+ >>> df = spark.createDataFrame(
+ ... [(2, "Alice"), (5, "Bob")], ["age", "name"])
+ >>> df.filter(df.name.contains('o')).collect()
+ [Row(age=5, name='Bob')]
+ """
+ ...
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ @dispatch_col_method
+ def startswith(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """
+ String starts with. Returns a boolean :class:`Column` based on a string match.
- Parameters
- ----------
- other : :class:`Column` or str
- string at start of line (do not use a regex `^`)
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(2, "Alice"), (5, "Bob")], ["age", "name"])
- >>> df.filter(df.name.startswith('Al')).collect()
- [Row(age=2, name='Alice')]
- >>> df.filter(df.name.startswith('^Al')).collect()
- []
- """
- _endswith_doc = """
- String ends with. Returns a boolean :class:`Column` based on a string match.
+ Parameters
+ ----------
+ other : :class:`Column` or str
+ string at start of line (do not use a regex `^`)
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ Examples
+ --------
+ >>> df = spark.createDataFrame(
+ ... [(2, "Alice"), (5, "Bob")], ["age", "name"])
+ >>> df.filter(df.name.startswith('Al')).collect()
+ [Row(age=2, name='Alice')]
+ >>> df.filter(df.name.startswith('^Al')).collect()
+ []
+ """
+ ...
- Parameters
- ----------
- other : :class:`Column` or str
- string at end of line (do not use a regex `$`)
+ @dispatch_col_method
+ def endswith(
+ self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> "Column":
+ """
+ String ends with. Returns a boolean :class:`Column` based on a string match.
- Examples
- --------
- >>> df = spark.createDataFrame(
- ... [(2, "Alice"), (5, "Bob")], ["age", "name"])
- >>> df.filter(df.name.endswith('ice')).collect()
- [Row(age=2, name='Alice')]
- >>> df.filter(df.name.endswith('ice$')).collect()
- []
- """
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- contains = _bin_op("contains", _contains_doc)
- startswith = _bin_op("startsWith", _startswith_doc)
- endswith = _bin_op("endsWith", _endswith_doc)
+ Parameters
+ ----------
+ other : :class:`Column` or str
+ string at end of line (do not use a regex `$`)
+ Examples
+ --------
+ >>> df = spark.createDataFrame(
+ ... [(2, "Alice"), (5, "Bob")], ["age", "name"])
+ >>> df.filter(df.name.endswith('ice')).collect()
+ [Row(age=2, name='Alice')]
+ >>> df.filter(df.name.endswith('ice$')).collect()
+ []
+ """
+ ...
+
+ @dispatch_col_method
def like(self: "Column", other: str) -> "Column":
"""
SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.
@@ -836,9 +763,9 @@ class Column:
>>> df.filter(df.name.like('Al%')).collect()
[Row(age=2, name='Alice')]
"""
- njc = getattr(self._jc, "like")(other)
- return Column(njc)
+ ...
+ @dispatch_col_method
def rlike(self: "Column", other: str) -> "Column":
"""
SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex
@@ -865,9 +792,9 @@ class Column:
>>> df.filter(df.name.rlike('ice$')).collect()
[Row(age=2, name='Alice')]
"""
- njc = getattr(self._jc, "rlike")(other)
- return Column(njc)
+ ...
+ @dispatch_col_method
def ilike(self: "Column", other: str) -> "Column":
"""
SQL ILIKE expression (case insensitive LIKE). Returns a boolean :class:`Column`
@@ -900,8 +827,7 @@ class Column:
>>> df.filter(df.name.ilike('%Ice')).collect()
[Row(age=2, name='Alice')]
"""
- njc = getattr(self._jc, "ilike")(other)
- return Column(njc)
+ ...
@overload
def substr(self, startPos: int, length: int) -> "Column":
@@ -911,6 +837,7 @@ class Column:
def substr(self, startPos: "Column", length: "Column") -> "Column":
...
+ @dispatch_col_method
def substr(self, startPos: Union[int, "Column"], length: Union[int, "Column"]) -> "Column":
"""
Return a :class:`Column` which is a substring of the column.
@@ -949,27 +876,9 @@ class Column:
>>> df.select(df.name.substr(df.sidx, df.eidx).alias("col")).collect()
[Row(col='ice'), Row(col='ob')]
"""
- if type(startPos) != type(length):
- raise PySparkTypeError(
- error_class="NOT_SAME_TYPE",
- message_parameters={
- "arg_name1": "startPos",
- "arg_name2": "length",
- "arg_type1": type(startPos).__name__,
- "arg_type2": type(length).__name__,
- },
- )
- if isinstance(startPos, int):
- jc = self._jc.substr(startPos, length)
- elif isinstance(startPos, Column):
- jc = self._jc.substr(startPos._jc, cast("Column", length)._jc)
- else:
- raise PySparkTypeError(
- error_class="NOT_COLUMN_OR_INT",
- message_parameters={"arg_name": "startPos", "arg_type": type(startPos).__name__},
- )
- return Column(jc)
+ ...
+ @dispatch_col_method
def isin(self, *cols: Any) -> "Column":
"""
A boolean expression that is evaluated to true if the value of this
@@ -1023,164 +932,184 @@ class Column:
| 8|Mike|
+---+----+
"""
- if len(cols) == 1 and isinstance(cols[0], (list, set)):
- cols = cast(Tuple, cols[0])
- cols = cast(
- Tuple,
- [c._jc if isinstance(c, Column) else _create_column_from_literal(c) for c in cols],
- )
- sc = get_active_spark_context()
- jc = getattr(self._jc, "isin")(_to_seq(sc, cols))
- return Column(jc)
+ ...
# order
- _asc_doc = """
- Returns a sort expression based on the ascending order of the column.
+ @dispatch_col_method
+ def asc(self) -> "Column":
+ """
+ Returns a sort expression based on the ascending order of the column.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
- >>> df.select(df.name).orderBy(df.name.asc()).collect()
- [Row(name='Alice'), Row(name='Tom')]
- """
- _asc_nulls_first_doc = """
- Returns a sort expression based on ascending order of the column, and null values
- return before non-null values.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
+ >>> df.select(df.name).orderBy(df.name.asc()).collect()
+ [Row(name='Alice'), Row(name='Tom')]
+ """
+ ...
- .. versionadded:: 2.4.0
+ @dispatch_col_method
+ def asc_nulls_first(self) -> "Column":
+ """
+ Returns a sort expression based on ascending order of the column, and null values
+ return before non-null values.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ .. versionadded:: 2.4.0
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
- >>> df.select(df.name).orderBy(df.name.asc_nulls_first()).collect()
- [Row(name=None), Row(name='Alice'), Row(name='Tom')]
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- """
- _asc_nulls_last_doc = """
- Returns a sort expression based on ascending order of the column, and null values
- appear after non-null values.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame(
+ ... [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+ >>> df.select(df.name).orderBy(df.name.asc_nulls_first()).collect()
+ [Row(name=None), Row(name='Alice'), Row(name='Tom')]
- .. versionadded:: 2.4.0
+ """
+ ...
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ @dispatch_col_method
+ def asc_nulls_last(self) -> "Column":
+ """
+ Returns a sort expression based on ascending order of the column, and null values
+ appear after non-null values.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
- >>> df.select(df.name).orderBy(df.name.asc_nulls_last()).collect()
- [Row(name='Alice'), Row(name='Tom'), Row(name=None)]
+ .. versionadded:: 2.4.0
- """
- _desc_doc = """
- Returns a sort expression based on the descending order of the column.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
+
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame(
+ ... [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+ >>> df.select(df.name).orderBy(df.name.asc_nulls_last()).collect()
+ [Row(name='Alice'), Row(name='Tom'), Row(name=None)]
- .. versionadded:: 2.4.0
+ """
+ ...
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ @dispatch_col_method
+ def desc(self) -> "Column":
+ """
+ Returns a sort expression based on the descending order of the column.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
- >>> df.select(df.name).orderBy(df.name.desc()).collect()
- [Row(name='Tom'), Row(name='Alice')]
- """
- _desc_nulls_first_doc = """
- Returns a sort expression based on the descending order of the column, and null values
- appear before non-null values.
+ .. versionadded:: 2.4.0
- .. versionadded:: 2.4.0
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
+ >>> df.select(df.name).orderBy(df.name.desc()).collect()
+ [Row(name='Tom'), Row(name='Alice')]
+ """
+ ...
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
- >>> df.select(df.name).orderBy(df.name.desc_nulls_first()).collect()
- [Row(name=None), Row(name='Tom'), Row(name='Alice')]
+ @dispatch_col_method
+ def desc_nulls_first(self) -> "Column":
+ """
+ Returns a sort expression based on the descending order of the column, and null values
+ appear before non-null values.
- """
- _desc_nulls_last_doc = """
- Returns a sort expression based on the descending order of the column, and null values
- appear after non-null values.
+ .. versionadded:: 2.4.0
- .. versionadded:: 2.4.0
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame(
+ ... [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+ >>> df.select(df.name).orderBy(df.name.desc_nulls_first()).collect()
+ [Row(name=None), Row(name='Tom'), Row(name='Alice')]
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
- >>> df.select(df.name).orderBy(df.name.desc_nulls_last()).collect()
- [Row(name='Tom'), Row(name='Alice'), Row(name=None)]
- """
+ """
+ ...
- asc = _unary_op("asc", _asc_doc)
- asc_nulls_first = _unary_op("asc_nulls_first", _asc_nulls_first_doc)
- asc_nulls_last = _unary_op("asc_nulls_last", _asc_nulls_last_doc)
- desc = _unary_op("desc", _desc_doc)
- desc_nulls_first = _unary_op("desc_nulls_first", _desc_nulls_first_doc)
- desc_nulls_last = _unary_op("desc_nulls_last", _desc_nulls_last_doc)
+ @dispatch_col_method
+ def desc_nulls_last(self) -> "Column":
+ """
+ Returns a sort expression based on the descending order of the column, and null values
+ appear after non-null values.
- _isNull_doc = """
- True if the current expression is null.
+ .. versionadded:: 2.4.0
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
- >>> df.filter(df.height.isNull()).collect()
- [Row(name='Alice', height=None)]
- """
- _isNotNull_doc = """
- True if the current expression is NOT null.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame(
+ ... [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+ >>> df.select(df.name).orderBy(df.name.desc_nulls_last()).collect()
+ [Row(name='Tom'), Row(name='Alice'), Row(name=None)]
+ """
+ ...
- .. versionchanged:: 3.4.0
- Supports Spark Connect.
+ @dispatch_col_method
+ def isNull(self) -> "Column":
+ """
+ True if the current expression is null.
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
- >>> df.filter(df.height.isNotNull()).collect()
- [Row(name='Tom', height=80)]
- """
- _isNaN_doc = """
- True if the current expression is NaN.
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
- .. versionadded:: 4.0.0
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
+ >>> df.filter(df.height.isNull()).collect()
+ [Row(name='Alice', height=None)]
+ """
+ ...
- Examples
- --------
- >>> from pyspark.sql import Row
- >>> df = spark.createDataFrame(
- ... [Row(name='Tom', height=80.0), Row(name='Alice', height=float('nan'))])
- >>> df.filter(df.height.isNaN()).collect()
- [Row(name='Alice', height=nan)]
- """
+ @dispatch_col_method
+ def isNotNull(self) -> "Column":
+ """
+ True if the current expression is NOT null.
- isNull = _unary_op("isNull", _isNull_doc)
- isNotNull = _unary_op("isNotNull", _isNotNull_doc)
- isNaN = _unary_op("isNaN", _isNaN_doc)
+ .. versionchanged:: 3.4.0
+ Supports Spark Connect.
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
+ >>> df.filter(df.height.isNotNull()).collect()
+ [Row(name='Tom', height=80)]
+ """
+ ...
+
+ @dispatch_col_method
+ def isNaN(self) -> "Column":
+ """
+ True if the current expression is NaN.
+
+ .. versionadded:: 4.0.0
+
+ Examples
+ --------
+ >>> from pyspark.sql import Row
+ >>> df = spark.createDataFrame(
+ ... [Row(name='Tom', height=80.0), Row(name='Alice', height=float('nan'))])
+ >>> df.filter(df.height.isNaN()).collect()
+ [Row(name='Alice', height=nan)]
+ """
+ ...
+
+ @dispatch_col_method
def alias(self, *alias: str, **kwargs: Any) -> "Column":
"""
Returns this column aliased with a new name or names (in the case of expressions that
@@ -1220,34 +1149,18 @@ class Column:
>>> df.select(df.age.alias("age3", metadata={'max': 99})).schema['age3'].metadata['max']
99
"""
+ ...
- metadata = kwargs.pop("metadata", None)
- assert not kwargs, "Unexpected kwargs where passed: %s" % kwargs
-
- sc = get_active_spark_context()
- if len(alias) == 1:
- if metadata:
- assert sc._jvm is not None
- jmeta = sc._jvm.org.apache.spark.sql.types.Metadata.fromJson(json.dumps(metadata))
- return Column(getattr(self._jc, "as")(alias[0], jmeta))
- else:
- return Column(getattr(self._jc, "as")(alias[0]))
- else:
- if metadata is not None:
- raise PySparkValueError(
- error_class="ONLY_ALLOWED_FOR_SINGLE_COLUMN",
- message_parameters={"arg_name": "metadata"},
- )
- return Column(getattr(self._jc, "as")(_to_seq(sc, list(alias))))
-
+ @dispatch_col_method
def name(self, *alias: str, **kwargs: Any) -> "Column":
"""
:func:`name` is an alias for :func:`alias`.
.. versionadded:: 2.0.0
"""
- return self.alias(*alias, **kwargs)
+ ...
+ @dispatch_col_method
def cast(self, dataType: Union[DataType, str]) -> "Column":
"""
Casts the column into type ``dataType``.
@@ -1278,21 +1191,9 @@ class Column:
>>> df.select(df.age.cast(StringType()).alias('ages')).collect()
[Row(ages='2'), Row(ages='5')]
"""
- if isinstance(dataType, str):
- jc = self._jc.cast(dataType)
- elif isinstance(dataType, DataType):
- from pyspark.sql import SparkSession
-
- spark = SparkSession._getActiveSessionOrCreate()
- jdt = spark._jsparkSession.parseDataType(dataType.json())
- jc = self._jc.cast(jdt)
- else:
- raise PySparkTypeError(
- error_class="NOT_DATATYPE_OR_STR",
- message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
- )
- return Column(jc)
+ ...
+ @dispatch_col_method
def try_cast(self, dataType: Union[DataType, str]) -> "Column":
"""
This is a special version of `cast` that performs the same operation, but returns a NULL
@@ -1340,29 +1241,18 @@ class Column:
| NULL|
+-----+
"""
- if isinstance(dataType, str):
- jc = self._jc.try_cast(dataType)
- elif isinstance(dataType, DataType):
- from pyspark.sql import SparkSession
-
- spark = SparkSession._getActiveSessionOrCreate()
- jdt = spark._jsparkSession.parseDataType(dataType.json())
- jc = self._jc.try_cast(jdt)
- else:
- raise PySparkTypeError(
- error_class="NOT_DATATYPE_OR_STR",
- message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
- )
- return Column(jc)
+ ...
+ @dispatch_col_method
def astype(self, dataType: Union[DataType, str]) -> "Column":
"""
:func:`astype` is an alias for :func:`cast`.
.. versionadded:: 1.4.0
"""
- return self.cast(dataType)
+ ...
+ @dispatch_col_method
def between(
self,
lowerBound: Union["Column", "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
@@ -1470,8 +1360,9 @@ class Column:
| Bob| true|
+-----+------------------------------------------------------------------+
"""
- return (self >= lowerBound) & (self <= upperBound)
+ ...
+ @dispatch_col_method
def when(self, condition: "Column", value: Any) -> "Column":
"""
Evaluates a list of conditions and returns one of multiple possible result expressions.
@@ -1545,15 +1436,9 @@ class Column:
--------
pyspark.sql.functions.when
"""
- if not isinstance(condition, Column):
- raise PySparkTypeError(
- error_class="NOT_COLUMN",
- message_parameters={"arg_name": "condition", "arg_type": type(condition).__name__},
- )
- v = value._jc if isinstance(value, Column) else value
- jc = self._jc.when(condition._jc, v)
- return Column(jc)
+ ...
+ @dispatch_col_method
def otherwise(self, value: Any) -> "Column":
"""
Evaluates a list of conditions and returns one of multiple possible result expressions.
@@ -1591,10 +1476,9 @@ class Column:
--------
pyspark.sql.functions.when
"""
- v = value._jc if isinstance(value, Column) else value
- jc = self._jc.otherwise(v)
- return Column(jc)
+ ...
+ @dispatch_col_method
def over(self, window: "WindowSpec") -> "Column":
"""
Define a windowing column.
@@ -1635,26 +1519,19 @@ class Column:
| 2|Alice| 1| 2|
+---+-----+----+---+
"""
- from pyspark.sql.window import WindowSpec
-
- if not isinstance(window, WindowSpec):
- raise PySparkTypeError(
- error_class="NOT_WINDOWSPEC",
- message_parameters={"arg_name": "window", "arg_type": type(window).__name__},
- )
- jc = self._jc.over(window._jspec)
- return Column(jc)
+ ...
+ @dispatch_col_method
def __nonzero__(self) -> None:
- raise PySparkValueError(
- error_class="CANNOT_CONVERT_COLUMN_INTO_BOOL",
- message_parameters={},
- )
+ ...
- __bool__ = __nonzero__
+ @dispatch_col_method
+ def __bool__(self) -> None:
+ ...
+ @dispatch_col_method
def __repr__(self) -> str:
- return "Column<'%s'>" % self._jc.toString()
+ ...
def _test() -> None:
diff --git a/python/pyspark/sql/connect/_typing.py b/python/pyspark/sql/connect/_typing.py
index 1b8516427dbd..efb3e0e8eb50 100644
--- a/python/pyspark/sql/connect/_typing.py
+++ b/python/pyspark/sql/connect/_typing.py
@@ -22,7 +22,7 @@ import decimal
import pyarrow
from pandas.core.frame import DataFrame as PandasDataFrame
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.types import DataType
from pyspark.sql.streaming.state import GroupState
diff --git a/python/pyspark/sql/connect/avro/functions.py b/python/pyspark/sql/connect/avro/functions.py
index f153b17acf58..7e46bc8d33be 100644
--- a/python/pyspark/sql/connect/avro/functions.py
+++ b/python/pyspark/sql/connect/avro/functions.py
@@ -26,8 +26,7 @@ check_dependencies(__name__)
from typing import Dict, Optional, TYPE_CHECKING
from pyspark.sql.avro import functions as PyAvroFunctions
-
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col, _options_to_col, lit
if TYPE_CHECKING:
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index 4436b36907a9..146c517d1b1a 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -24,16 +24,14 @@ import warnings
from typing import (
TYPE_CHECKING,
- Callable,
Any,
Union,
- overload,
Optional,
)
+from pyspark.sql.column import Column as ParentColumn
from pyspark.errors import PySparkTypeError, PySparkAttributeError, PySparkValueError
from pyspark.sql.types import DataType
-from pyspark.sql.column import Column as PySparkColumn
import pyspark.sql.connect.proto as proto
from pyspark.sql.connect.expressions import (
@@ -60,53 +58,52 @@ if TYPE_CHECKING:
from pyspark.sql.connect.window import WindowSpec
-def _func_op(name: str, doc: Optional[str] = "") -> Callable[["Column"], "Column"]:
- def wrapped(self: "Column") -> "Column":
- return Column(UnresolvedFunction(name, [self._expr]))
-
- wrapped.__doc__ = doc
- return wrapped
+def _func_op(name: str, self: ParentColumn) -> ParentColumn:
+ return Column(UnresolvedFunction(name, [self._expr])) # type: ignore[list-item]
def _bin_op(
- name: str, doc: Optional[str] = "binary function", reverse: bool = False
-) -> Callable[["Column", Any], "Column"]:
- def wrapped(self: "Column", other: Any) -> "Column":
- if other is None or isinstance(
- other,
- (
- bool,
- float,
- int,
- str,
- datetime.datetime,
- datetime.date,
- decimal.Decimal,
- datetime.timedelta,
- ),
- ):
- other_expr = LiteralExpression._from_value(other)
- else:
- other_expr = other._expr
-
- if not reverse:
- return Column(UnresolvedFunction(name, [self._expr, other_expr]))
- else:
- return Column(UnresolvedFunction(name, [other_expr, self._expr]))
-
- wrapped.__doc__ = doc
- return wrapped
-
-
-def _unary_op(name: str, doc: Optional[str] = "unary function") -> Callable[["Column"], "Column"]:
- def wrapped(self: "Column") -> "Column":
- return Column(UnresolvedFunction(name, [self._expr]))
-
- wrapped.__doc__ = doc
- return wrapped
-
+ name: str,
+ self: ParentColumn,
+ other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+ reverse: bool = False,
+) -> ParentColumn:
+ if other is None or isinstance(
+ other,
+ (
+ bool,
+ float,
+ int,
+ str,
+ datetime.datetime,
+ datetime.date,
+ decimal.Decimal,
+ datetime.timedelta,
+ ),
+ ):
+ other_expr = LiteralExpression._from_value(other)
+ else:
+ other_expr = other._expr # type: ignore[assignment]
+
+ if not reverse:
+ return Column(UnresolvedFunction(name, [self._expr, other_expr])) # type: ignore[list-item]
+ else:
+ return Column(UnresolvedFunction(name, [other_expr, self._expr])) # type: ignore[list-item]
+
+
+def _unary_op(name: str, self: ParentColumn) -> ParentColumn:
+ return Column(UnresolvedFunction(name, [self._expr])) # type: ignore[list-item]
+
+
+class Column(ParentColumn):
+ def __new__(
+ cls,
+ expr: "Expression",
+ ) -> "Column":
+ self = object.__new__(cls)
+ self.__init__(expr) # type: ignore[misc]
+ return self
-class Column:
def __init__(self, expr: "Expression") -> None:
if not isinstance(expr, Expression):
raise PySparkTypeError(
@@ -115,36 +112,128 @@ class Column:
)
self._expr = expr
- __gt__ = _bin_op(">")
- __lt__ = _bin_op("<")
- __add__ = _bin_op("+")
- __sub__ = _bin_op("-")
- __mul__ = _bin_op("*")
- __div__ = _bin_op("/")
- __truediv__ = _bin_op("/")
- __mod__ = _bin_op("%")
- __radd__ = _bin_op("+", reverse=True)
- __rsub__ = _bin_op("-", reverse=True)
- __rmul__ = _bin_op("*", reverse=True)
- __rdiv__ = _bin_op("/", reverse=True)
- __rtruediv__ = _bin_op("/", reverse=True)
- __rmod__ = _bin_op("%", reverse=True)
- __pow__ = _bin_op("power")
- __rpow__ = _bin_op("power", reverse=True)
- __ge__ = _bin_op(">=")
- __le__ = _bin_op("<=")
-
- eqNullSafe = _bin_op("<=>", PySparkColumn.eqNullSafe.__doc__)
-
- __neg__ = _func_op("negative")
+ def __gt__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op(">", self, other)
+
+ def __lt__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("<", self, other)
+
+ def __add__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("+", self, other)
+
+ def __sub__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("-", self, other)
+
+ def __mul__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("*", self, other)
+
+ def __div__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("/", self, other)
+
+ def __truediv__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("/", self, other)
+
+ def __mod__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("%", self, other)
+
+ def __radd__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("+", self, other, reverse=True)
+
+ def __rsub__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("-", self, other, reverse=True)
+
+ def __rmul__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("*", self, other, reverse=True)
+
+ def __rdiv__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("/", self, other, reverse=True)
+
+ def __rtruediv__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("/", self, other, reverse=True)
+
+ def __rmod__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("%", self, other, reverse=True)
+
+ def __pow__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("power", self, other)
+
+ def __rpow__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("power", self, other, reverse=True)
+
+ def __ge__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op(">=", self, other)
+
+ def __le__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("<=", self, other)
+
+ def eqNullSafe(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("<=>", self, other)
+
+ def __neg__(self) -> ParentColumn:
+ return _func_op("negative", self)
# `and`, `or`, `not` cannot be overloaded in Python,
# so use bitwise operators as boolean operators
- __and__ = _bin_op("and")
- __or__ = _bin_op("or")
- __invert__ = _func_op("not")
- __rand__ = _bin_op("and")
- __ror__ = _bin_op("or")
+ def __and__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("and", self, other)
+
+ def __or__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("or", self, other)
+
+ def __invert__(self) -> ParentColumn:
+ return _func_op("not", self)
+
+ def __rand__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("and", self, other)
+
+ def __ror__(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("or", self, other)
# container operators
def __contains__(self, item: Any) -> None:
@@ -154,27 +243,53 @@ class Column:
)
# bitwise operators
- bitwiseOR = _bin_op("|", PySparkColumn.bitwiseOR.__doc__)
- bitwiseAND = _bin_op("&", PySparkColumn.bitwiseAND.__doc__)
- bitwiseXOR = _bin_op("^", PySparkColumn.bitwiseXOR.__doc__)
+ def bitwiseOR(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("|", self, other)
+
+ def bitwiseAND(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("&", self, other)
- isNull = _unary_op("isnull", PySparkColumn.isNull.__doc__)
- isNotNull = _unary_op("isnotnull", PySparkColumn.isNotNull.__doc__)
- isNaN = _unary_op("isNaN", PySparkColumn.isNaN.__doc__)
+ def bitwiseXOR(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("^", self, other)
+
+ def isNull(self) -> ParentColumn:
+ return _unary_op("isNull", self)
+
+ def isNotNull(self) -> ParentColumn:
+ return _unary_op("isNotNull", self)
+
+ def isNaN(self) -> ParentColumn:
+ return _unary_op("isNaN", self)
def __ne__( # type: ignore[override]
self,
other: Any,
- ) -> "Column":
- """binary function"""
- return _func_op("not")(_bin_op("==")(self, other))
+ ) -> ParentColumn:
+ return _func_op("not", _bin_op("==", self, other))
# string methods
- contains = _bin_op("contains", PySparkColumn.contains.__doc__)
- startswith = _bin_op("startswith", PySparkColumn.startswith.__doc__)
- endswith = _bin_op("endswith", PySparkColumn.endswith.__doc__)
-
- def when(self, condition: "Column", value: Any) -> "Column":
+ def contains(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("contains", self, other)
+
+ def startswith(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("startsWith", self, other)
+
+ def endswith(
+ self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+ ) -> ParentColumn:
+ return _bin_op("endsWith", self, other)
+
+ def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
if not isinstance(condition, Column):
raise PySparkTypeError(
error_class="NOT_COLUMN",
@@ -202,9 +317,7 @@ class Column:
return Column(CaseWhen(branches=_branches, else_value=None))
- when.__doc__ = PySparkColumn.when.__doc__
-
- def otherwise(self, value: Any) -> "Column":
+ def otherwise(self, value: Any) -> ParentColumn:
if not isinstance(self._expr, CaseWhen):
raise PySparkTypeError(
"otherwise() can only be applied on a Column previously generated by when()"
@@ -222,21 +335,18 @@ class Column:
return Column(CaseWhen(branches=self._expr._branches, else_value=_value))
- otherwise.__doc__ = PySparkColumn.otherwise.__doc__
+ def like(self: ParentColumn, other: str) -> ParentColumn:
+ return _bin_op("like", self, other)
- like = _bin_op("like", PySparkColumn.like.__doc__)
- rlike = _bin_op("rlike", PySparkColumn.rlike.__doc__)
- ilike = _bin_op("ilike", PySparkColumn.ilike.__doc__)
+ def rlike(self: ParentColumn, other: str) -> ParentColumn:
+ return _bin_op("rlike", self, other)
- @overload
- def substr(self, startPos: int, length: int) -> "Column":
- ...
+ def ilike(self: ParentColumn, other: str) -> ParentColumn:
+ return _bin_op("ilike", self, other)
- @overload
- def substr(self, startPos: "Column", length: "Column") -> "Column":
- ...
-
- def substr(self, startPos: Union[int, "Column"], length: Union[int, "Column"]) -> "Column":
+ def substr(
+ self, startPos: Union[int, ParentColumn], length: Union[int, ParentColumn]
+ ) -> ParentColumn:
if type(startPos) != type(length):
raise PySparkTypeError(
error_class="NOT_SAME_TYPE",
@@ -259,14 +369,13 @@ class Column:
error_class="NOT_COLUMN_OR_INT",
message_parameters={"arg_name": "startPos", "arg_type": type(length).__name__},
)
- return Column(UnresolvedFunction("substr", [self._expr, start_expr, length_expr]))
-
- substr.__doc__ = PySparkColumn.substr.__doc__
+ return Column(
+ UnresolvedFunction(
+ "substr", [self._expr, start_expr, length_expr] # type: ignore[list-item]
+ )
+ )
- def __eq__(self, other: Any) -> "Column": # type: ignore[override]
- """Returns a binary expression with the current column as the left
- side and the other expression as the right side.
- """
+ def __eq__(self, other: Any) -> ParentColumn: # type: ignore[override]
if other is None or isinstance(
other, (bool, float, int, str, datetime.datetime, datetime.date, decimal.Decimal)
):
@@ -279,46 +388,30 @@ class Column:
def to_plan(self, session: "SparkConnectClient") -> proto.Expression:
return self._expr.to_plan(session)
- def alias(self, *alias: str, **kwargs: Any) -> "Column":
+ def alias(self, *alias: str, **kwargs: Any) -> ParentColumn:
return Column(self._expr.alias(*alias, **kwargs))
- alias.__doc__ = PySparkColumn.alias.__doc__
-
name = alias
- name.__doc__ = PySparkColumn.name.__doc__
-
- def asc(self) -> "Column":
+ def asc(self) -> ParentColumn:
return self.asc_nulls_first()
- asc.__doc__ = PySparkColumn.asc.__doc__
-
- def asc_nulls_first(self) -> "Column":
+ def asc_nulls_first(self) -> ParentColumn:
return Column(SortOrder(self._expr, ascending=True, nullsFirst=True))
- asc_nulls_first.__doc__ = PySparkColumn.asc_nulls_first.__doc__
-
- def asc_nulls_last(self) -> "Column":
+ def asc_nulls_last(self) -> ParentColumn:
return Column(SortOrder(self._expr, ascending=True, nullsFirst=False))
- asc_nulls_last.__doc__ = PySparkColumn.asc_nulls_last.__doc__
-
- def desc(self) -> "Column":
+ def desc(self) -> ParentColumn:
return self.desc_nulls_last()
- desc.__doc__ = PySparkColumn.desc.__doc__
-
- def desc_nulls_first(self) -> "Column":
+ def desc_nulls_first(self) -> ParentColumn:
return Column(SortOrder(self._expr, ascending=False, nullsFirst=True))
- desc_nulls_first.__doc__ = PySparkColumn.desc_nulls_first.__doc__
-
- def desc_nulls_last(self) -> "Column":
+ def desc_nulls_last(self) -> ParentColumn:
return Column(SortOrder(self._expr, ascending=False, nullsFirst=False))
- desc_nulls_last.__doc__ = PySparkColumn.desc_nulls_last.__doc__
-
- def cast(self, dataType: Union[DataType, str]) -> "Column":
+ def cast(self, dataType: Union[DataType, str]) -> ParentColumn:
if isinstance(dataType, (DataType, str)):
return Column(CastExpression(expr=self._expr, data_type=dataType))
else:
@@ -327,11 +420,9 @@ class Column:
message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
)
- cast.__doc__ = PySparkColumn.cast.__doc__
-
astype = cast
- def try_cast(self, dataType: Union[DataType, str]) -> "Column":
+ def try_cast(self, dataType: Union[DataType, str]) -> ParentColumn:
if isinstance(dataType, (DataType, str)):
return Column(
CastExpression(
@@ -346,12 +437,10 @@ class Column:
message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
)
- try_cast.__doc__ = PySparkColumn.try_cast.__doc__
-
def __repr__(self) -> str:
return "Column<'%s'>" % self._expr.__repr__()
- def over(self, window: "WindowSpec") -> "Column":
+ def over(self, window: "WindowSpec") -> ParentColumn: # type: ignore[override]
from pyspark.sql.connect.window import WindowSpec
if not isinstance(window, WindowSpec):
@@ -362,9 +451,7 @@ class Column:
return Column(WindowExpression(windowFunction=self._expr, windowSpec=window))
- over.__doc__ = PySparkColumn.over.__doc__
-
- def isin(self, *cols: Any) -> "Column":
+ def isin(self, *cols: Any) -> ParentColumn:
if len(cols) == 1 and isinstance(cols[0], (list, set)):
_cols = list(cols[0])
else:
@@ -379,18 +466,14 @@ class Column:
return Column(UnresolvedFunction("in", _exprs))
- isin.__doc__ = PySparkColumn.isin.__doc__
-
def between(
self,
- lowerBound: Union["Column", "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
- upperBound: Union["Column", "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
- ) -> "Column":
+ lowerBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+ upperBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+ ) -> ParentColumn:
return (self >= lowerBound) & (self <= upperBound)
- between.__doc__ = PySparkColumn.between.__doc__
-
- def getItem(self, key: Any) -> "Column":
+ def getItem(self, key: Any) -> ParentColumn:
if isinstance(key, Column):
warnings.warn(
"A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
@@ -400,9 +483,7 @@ class Column:
)
return self[key]
- getItem.__doc__ = PySparkColumn.getItem.__doc__
-
- def getField(self, name: Any) -> "Column":
+ def getField(self, name: Any) -> ParentColumn:
if isinstance(name, Column):
warnings.warn(
"A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
@@ -412,9 +493,7 @@ class Column:
)
return self[name]
- getField.__doc__ = PySparkColumn.getField.__doc__
-
- def withField(self, fieldName: str, col: "Column") -> "Column":
+ def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:
if not isinstance(fieldName, str):
raise PySparkTypeError(
error_class="NOT_STR",
@@ -429,9 +508,7 @@ class Column:
return Column(WithField(self._expr, fieldName, col._expr))
- withField.__doc__ = PySparkColumn.withField.__doc__
-
- def dropFields(self, *fieldNames: str) -> "Column":
+ def dropFields(self, *fieldNames: str) -> ParentColumn:
dropField: Optional[DropField] = None
for fieldName in fieldNames:
if not isinstance(fieldName, str):
@@ -458,9 +535,7 @@ class Column:
return Column(dropField)
- dropFields.__doc__ = PySparkColumn.dropFields.__doc__
-
- def __getattr__(self, item: Any) -> "Column":
+ def __getattr__(self, item: Any) -> ParentColumn:
if item == "_jc":
raise PySparkAttributeError(
error_class="JVM_ATTRIBUTE_NOT_SUPPORTED", message_parameters={"attr_name": "_jc"}
@@ -471,7 +546,7 @@ class Column:
)
return self[item]
- def __getitem__(self, k: Any) -> "Column":
+ def __getitem__(self, k: Any) -> ParentColumn:
if isinstance(k, slice):
if k.step is not None:
raise PySparkValueError(
@@ -499,9 +574,6 @@ class Column:
__bool__ = __nonzero__
-Column.__doc__ = PySparkColumn.__doc__
-
-
def _test() -> None:
import os
import sys
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 98a9cd4875c6..c1d6b4f81c3c 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -73,7 +73,7 @@ from pyspark.sql.connect.conversion import ArrowTableToRowsConversion
from pyspark.sql.connect.group import GroupedData
from pyspark.sql.connect.readwriter import DataFrameWriter, DataFrameWriterV2
from pyspark.sql.connect.streaming.readwriter import DataStreamWriter
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.expressions import (
ColumnReference,
UnresolvedRegex,
@@ -255,12 +255,14 @@ class DataFrame(ParentDataFrame):
return DataFrame(plan.SubqueryAlias(self._plan, alias), session=self._session)
def colRegex(self, colName: str) -> Column:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
if not isinstance(colName, str):
raise PySparkTypeError(
error_class="NOT_STR",
message_parameters={"arg_name": "colName", "arg_type": type(colName).__name__},
)
- return Column(UnresolvedRegex(colName, self._plan._plan_id))
+ return ConnectColumn(UnresolvedRegex(colName, self._plan._plan_id))
@property
def dtypes(self) -> List[Tuple[str, str]]:
@@ -750,7 +752,7 @@ class DataFrame(ParentDataFrame):
session=self._session,
)
- orderBy = sort # type: ignore[assignment]
+ orderBy = sort
def sortWithinPartitions(
self,
@@ -959,7 +961,7 @@ class DataFrame(ParentDataFrame):
self._session,
)
- melt = unpivot # type: ignore[assignment]
+ melt = unpivot
def withWatermark(self, eventTime: str, delayThreshold: str) -> ParentDataFrame:
# TODO: reuse error handling code in sql.DataFrame.withWatermark()
@@ -1703,9 +1705,11 @@ class DataFrame(ParentDataFrame):
def __getitem__(
self, item: Union[int, str, Column, List, Tuple]
) -> Union[Column, ParentDataFrame]:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
if isinstance(item, str):
if item == "*":
- return Column(
+ return ConnectColumn(
UnresolvedStar(
unparsed_target=None,
plan_id=self._plan._plan_id,
@@ -1716,7 +1720,7 @@ class DataFrame(ParentDataFrame):
# if (sparkSession.sessionState.conf.supportQuotedRegexColumnName) {
# colRegex(colName)
# } else {
- # Column(addDataFrameIdToCol(resolve(colName)))
+ # ConnectColumn(addDataFrameIdToCol(resolve(colName)))
# }
# validate the column name
@@ -1742,7 +1746,9 @@ class DataFrame(ParentDataFrame):
)
def _col(self, name: str) -> Column:
- return Column(
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
+ return ConnectColumn(
ColumnReference(
unparsed_identifier=name,
plan_id=self._plan._plan_id,
@@ -2037,9 +2043,7 @@ class DataFrame(ParentDataFrame):
def foreach_func(row: Any) -> None:
f(row)
- self.select( # type: ignore[call-overload]
- F.struct(*self.schema.fieldNames()).alias("row")
- ).select(
+ self.select(F.struct(*self.schema.fieldNames()).alias("row")).select(
F.udf(foreach_func, StructType())("row") # type: ignore[arg-type]
).collect()
diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py
index 4a3f634d21bb..519e53c3a13f 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -43,7 +43,7 @@ import numpy as np
from pyspark.errors import PySparkTypeError, PySparkValueError
from pyspark.sql.dataframe import DataFrame as ParentDataFrame
-from pyspark.sql.connect.column import Column
+from pyspark.sql import Column
from pyspark.sql.connect.expressions import (
CaseWhen,
SortOrder,
@@ -112,14 +112,16 @@ def _invoke_function(name: str, *args: Union[Column, Expression]) -> Column:
-------
:class:`Column`
"""
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
expressions: List[Expression] = []
for arg in args:
assert isinstance(arg, (Column, Expression))
if isinstance(arg, Column):
- expressions.append(arg._expr)
+ expressions.append(arg._expr) # type: ignore[arg-type]
else:
expressions.append(arg)
- return Column(UnresolvedFunction(name, expressions))
+ return ConnectColumn(UnresolvedFunction(name, expressions))
def _invoke_function_over_columns(name: str, *cols: "ColumnOrName") -> Column:
@@ -180,6 +182,8 @@ def _create_lambda(f: Callable) -> LambdaFunction:
- (Column, Column) -> Column: ...
- (Column, Column, Column) -> Column: ...
"""
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
parameters = _get_lambda_parameters(f)
arg_names = ["x", "y", "z"][: len(parameters)]
@@ -187,7 +191,7 @@ def _create_lambda(f: Callable) -> LambdaFunction:
UnresolvedNamedLambdaVariable([UnresolvedNamedLambdaVariable.fresh_var_name(arg_name)])
for arg_name in arg_names
]
- arg_cols = [Column(arg_expr) for arg_expr in arg_exprs]
+ arg_cols = [ConnectColumn(arg_expr) for arg_expr in arg_exprs]
result = f(*arg_cols)
@@ -197,7 +201,7 @@ def _create_lambda(f: Callable) -> LambdaFunction:
message_parameters={"func_name": f.__name__, "return_type": type(result).__name__},
)
- return LambdaFunction(result._expr, arg_exprs)
+ return LambdaFunction(result._expr, arg_exprs) # type: ignore[arg-type]
def _invoke_higher_order_function(
@@ -234,12 +238,14 @@ def _options_to_col(options: Dict[str, Any]) -> Column:
def col(col: str) -> Column:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
if col == "*":
- return Column(UnresolvedStar(unparsed_target=None))
+ return ConnectColumn(UnresolvedStar(unparsed_target=None))
elif col.endswith(".*"):
- return Column(UnresolvedStar(unparsed_target=col))
+ return ConnectColumn(UnresolvedStar(unparsed_target=col))
else:
- return Column(ColumnReference(unparsed_identifier=col))
+ return ConnectColumn(ColumnReference(unparsed_identifier=col))
col.__doc__ = pysparkfuncs.col.__doc__
@@ -249,6 +255,8 @@ column = col
def lit(col: Any) -> Column:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
if isinstance(col, Column):
return col
elif isinstance(col, list):
@@ -272,7 +280,7 @@ def lit(col: Any) -> Column:
return array(*[lit(c) for c in col])
else:
- return Column(LiteralExpression._from_value(col))
+ return ConnectColumn(LiteralExpression._from_value(col))
lit.__doc__ = pysparkfuncs.lit.__doc__
@@ -336,7 +344,9 @@ coalesce.__doc__ = pysparkfuncs.coalesce.__doc__
def expr(str: str) -> Column:
- return Column(SQLExpression(str))
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
+ return ConnectColumn(SQLExpression(str))
expr.__doc__ = pysparkfuncs.expr.__doc__
@@ -429,6 +439,8 @@ spark_partition_id.__doc__ = pysparkfuncs.spark_partition_id.__doc__
def when(condition: Column, value: Any) -> Column:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
# Explicitly not using ColumnOrName type here to make reading condition less opaque
if not isinstance(condition, Column):
raise PySparkTypeError(
@@ -438,7 +450,12 @@ def when(condition: Column, value: Any) -> Column:
value_col = value if isinstance(value, Column) else lit(value)
- return Column(CaseWhen(branches=[(condition._expr, value_col._expr)], else_value=None))
+ return ConnectColumn(
+ CaseWhen(
+ branches=[(condition._expr, value_col._expr)], # type: ignore[list-item]
+ else_value=None,
+ )
+ )
when.__doc__ = pysparkfuncs.when.__doc__
@@ -1045,8 +1062,12 @@ countDistinct.__doc__ = pysparkfuncs.countDistinct.__doc__
def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
_exprs = [_to_col(c)._expr for c in [col] + list(cols)]
- return Column(UnresolvedFunction("count", _exprs, is_distinct=True))
+ return ConnectColumn(
+ UnresolvedFunction("count", _exprs, is_distinct=True) # type: ignore[arg-type]
+ )
count_distinct.__doc__ = pysparkfuncs.count_distinct.__doc__
@@ -1293,7 +1314,11 @@ sumDistinct.__doc__ = pysparkfuncs.sumDistinct.__doc__
def sum_distinct(col: "ColumnOrName") -> Column:
- return Column(UnresolvedFunction("sum", [_to_col(col)._expr], is_distinct=True))
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
+ return ConnectColumn(
+ UnresolvedFunction("sum", [_to_col(col)._expr], is_distinct=True) # type: ignore[list-item]
+ )
sum_distinct.__doc__ = pysparkfuncs.sum_distinct.__doc__
@@ -4101,8 +4126,10 @@ udtf.__doc__ = pysparkfuncs.udtf.__doc__
def call_function(funcName: str, *cols: "ColumnOrName") -> Column:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
expressions = [_to_col(c)._expr for c in cols]
- return Column(CallFunction(funcName, expressions))
+ return ConnectColumn(CallFunction(funcName, expressions)) # type: ignore[arg-type]
call_function.__doc__ = pysparkfuncs.call_function.__doc__
diff --git a/python/pyspark/sql/connect/functions/partitioning.py b/python/pyspark/sql/connect/functions/partitioning.py
index bfeddad7d568..5d2dd58313bb 100644
--- a/python/pyspark/sql/connect/functions/partitioning.py
+++ b/python/pyspark/sql/connect/functions/partitioning.py
@@ -22,7 +22,7 @@ from typing import Union, TYPE_CHECKING
from pyspark.errors import PySparkTypeError
from pyspark.sql import functions as pysparkfuncs
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.functions.builtin import _to_col, _invoke_function_over_columns
from pyspark.sql.connect.functions.builtin import lit, _invoke_function
diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py
index b866f61efe4a..699dce76c4a1 100644
--- a/python/pyspark/sql/connect/group.py
+++ b/python/pyspark/sql/connect/group.py
@@ -38,7 +38,7 @@ from pyspark.sql.types import NumericType
from pyspark.sql.types import StructType
import pyspark.sql.connect.plan as plan
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.functions import builtin as F
from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
@@ -61,10 +61,10 @@ class GroupedData:
self,
df: "DataFrame",
group_type: str,
- grouping_cols: Sequence["Column"],
- pivot_col: Optional["Column"] = None,
+ grouping_cols: Sequence[Column],
+ pivot_col: Optional[Column] = None,
pivot_values: Optional[Sequence["LiteralType"]] = None,
- grouping_sets: Optional[Sequence[Sequence["Column"]]] = None,
+ grouping_sets: Optional[Sequence[Sequence[Column]]] = None,
) -> None:
from pyspark.sql.connect.dataframe import DataFrame
diff --git a/python/pyspark/sql/connect/observation.py b/python/pyspark/sql/connect/observation.py
index 4fefb8aac41f..2471cf04cfbe 100644
--- a/python/pyspark/sql/connect/observation.py
+++ b/python/pyspark/sql/connect/observation.py
@@ -23,7 +23,7 @@ from pyspark.errors import (
IllegalArgumentException,
PySparkAssertionError,
)
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.dataframe import DataFrame
from pyspark.sql.observation import Observation as PySparkObservation
import pyspark.sql.connect.plan as plan
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
index 239ee23c2061..4ac4946745f5 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -14,6 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+
+# mypy: disable-error-code="operator"
+
from pyspark.resource import ResourceProfile
from pyspark.sql.connect.utils import check_dependencies
@@ -45,8 +48,8 @@ from pyspark.storagelevel import StorageLevel
from pyspark.sql.types import DataType
import pyspark.sql.connect.proto as proto
+from pyspark.sql.column import Column
from pyspark.sql.connect.conversion import storage_level_to_proto
-from pyspark.sql.connect.column import Column
from pyspark.sql.connect.expressions import Expression
from pyspark.sql.connect.types import pyspark_types_to_proto_types, UnparsedDataType
from pyspark.errors import (
diff --git a/python/pyspark/sql/connect/protobuf/functions.py b/python/pyspark/sql/connect/protobuf/functions.py
index 058925942fa0..07e9b4b8c686 100644
--- a/python/pyspark/sql/connect/protobuf/functions.py
+++ b/python/pyspark/sql/connect/protobuf/functions.py
@@ -27,7 +27,7 @@ from typing import Dict, Optional, TYPE_CHECKING
from pyspark.sql.protobuf import functions as PyProtobufFunctions
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col, _options_to_col, lit
if TYPE_CHECKING:
diff --git a/python/pyspark/sql/connect/udf.py b/python/pyspark/sql/connect/udf.py
index f3aa719b2bb6..a5faf9446fff 100644
--- a/python/pyspark/sql/connect/udf.py
+++ b/python/pyspark/sql/connect/udf.py
@@ -161,7 +161,10 @@ class UserDefinedFunction:
self, *args: "ColumnOrName", **kwargs: "ColumnOrName"
) -> CommonInlineUserDefinedFunction:
def to_expr(col: "ColumnOrName") -> Expression:
- return col._expr if isinstance(col, Column) else ColumnReference(col)
+ if isinstance(col, Column):
+ return col._expr
+ else:
+ return ColumnReference(col) # type: ignore[arg-type]
arg_exprs: List[Expression] = [to_expr(arg) for arg in args] + [
NamedArgumentExpression(key, to_expr(value)) for key, value in kwargs.items()
diff --git a/python/pyspark/sql/connect/udtf.py b/python/pyspark/sql/connect/udtf.py
index 4ee39dc89b8e..739289d72a3b 100644
--- a/python/pyspark/sql/connect/udtf.py
+++ b/python/pyspark/sql/connect/udtf.py
@@ -141,7 +141,10 @@ class UserDefinedTableFunction:
self, *args: "ColumnOrName", **kwargs: "ColumnOrName"
) -> CommonInlineUserDefinedTableFunction:
def to_expr(col: "ColumnOrName") -> Expression:
- return col._expr if isinstance(col, Column) else ColumnReference(col)
+ if isinstance(col, Column):
+ return col._expr
+ else:
+ return ColumnReference(col) # type: ignore[arg-type]
arg_exprs: List[Expression] = [to_expr(arg) for arg in args] + [
NamedArgumentExpression(key, to_expr(value)) for key, value in kwargs.items()
diff --git a/python/pyspark/sql/connect/window.py b/python/pyspark/sql/connect/window.py
index e30a5b7d7a9e..04cf4c91d320 100644
--- a/python/pyspark/sql/connect/window.py
+++ b/python/pyspark/sql/connect/window.py
@@ -21,7 +21,7 @@ check_dependencies(__name__)
import sys
from typing import TYPE_CHECKING, Union, Sequence, List, Optional
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
from pyspark.sql.connect.expressions import (
ColumnReference,
Expression,
@@ -107,7 +107,7 @@ class WindowSpec:
newPartitionSpec: List[Expression] = []
for c in _cols:
if isinstance(c, Column):
- newPartitionSpec.append(c._expr)
+ newPartitionSpec.append(c._expr) # type: ignore[arg-type]
else:
newPartitionSpec.append(ColumnReference(c))
@@ -143,7 +143,7 @@ class WindowSpec:
if isinstance(c._expr, SortOrder):
newOrderSpec.append(c._expr)
else:
- newOrderSpec.append(SortOrder(c._expr))
+ newOrderSpec.append(SortOrder(c._expr)) # type: ignore[arg-type]
else:
newOrderSpec.append(SortOrder(ColumnReference(c)))
diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py
index cef6eeed4fbc..96be5de0180b 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -40,7 +40,7 @@ from typing import (
)
from pyspark.errors import PySparkTypeError, PySparkValueError
-from pyspark.sql.column import Column, _to_java_column, _to_seq, _create_column_from_literal
+from pyspark.sql.column import Column
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.types import ArrayType, DataType, StringType, StructType, _from_numpy_type
@@ -106,6 +106,8 @@ def _invoke_function_over_columns(name: str, *cols: "ColumnOrName") -> Column:
Invokes n-ary JVM function identified by name
and wraps the result with :class:`~pyspark.sql.Column`.
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function(name, *(_to_java_column(col) for col in cols))
@@ -114,6 +116,8 @@ def _invoke_function_over_seq_of_columns(name: str, cols: "Iterable[ColumnOrName
Invokes unary JVM function identified by name with
and wraps the result with :class:`~pyspark.sql.Column`.
"""
+ from pyspark.sql.classic.column import _to_java_column, _to_seq
+
sc = _get_active_spark_context()
return _invoke_function(name, _to_seq(sc, cols, _to_java_column))
@@ -123,6 +127,7 @@ def _invoke_binary_math_function(name: str, col1: Any, col2: Any) -> Column:
Invokes binary JVM math function identified by name
and wraps the result with :class:`~pyspark.sql.Column`.
"""
+ from pyspark.sql.classic.column import _to_java_column, _create_column_from_literal
# For legacy reasons, the arguments here can be implicitly converted into column
cols = [
@@ -1005,6 +1010,8 @@ def mode(col: "ColumnOrName", deterministic: bool = False) -> Column:
| -10|
+---------------------------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("mode", _to_java_column(col), deterministic)
@@ -5123,6 +5130,8 @@ def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> C
| 95546| 102065|
+----------------+------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if rsd is None:
return _invoke_function_over_columns("approx_count_distinct", col)
else:
@@ -5386,6 +5395,8 @@ def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
| 2|
+------------------------------+
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function(
"count_distinct", _to_java_column(col), _to_seq(sc, cols, _to_java_column)
@@ -5443,6 +5454,8 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
| Bob| 5|
+-----+----------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("first", _to_java_column(col), ignorenulls)
@@ -5714,6 +5727,8 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
| Bob| 5|
+-----+---------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("last", _to_java_column(col), ignorenulls)
@@ -5847,6 +5862,8 @@ def percentile(
| 2| 19.967859769284075|
+---+--------------------+
"""
+ from pyspark.sql.classic.column import _to_seq, _create_column_from_literal, _to_java_column
+
sc = _get_active_spark_context()
if isinstance(percentage, (list, tuple)):
@@ -5924,6 +5941,8 @@ def percentile_approx(
|-- key: long (nullable = true)
|-- median: double (nullable = true)
"""
+ from pyspark.sql.classic.column import _to_seq, _create_column_from_literal, _to_java_column
+
sc = _get_active_spark_context()
if isinstance(percentage, (list, tuple)):
@@ -5998,6 +6017,8 @@ def approx_percentile(
|-- key: long (nullable = true)
|-- approx_percentile(value, 0.5, 1000000): double (nullable = true)
"""
+ from pyspark.sql.classic.column import _to_seq, _create_column_from_literal, _to_java_column
+
sc = _get_active_spark_context()
if isinstance(percentage, (list, tuple)):
@@ -6279,6 +6300,8 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column:
>>> spark.createDataFrame([(21,)], ['a']).select(shiftleft('a', 1).alias('r')).collect()
[Row(r=42)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("shiftleft", _to_java_column(col), numBits)
@@ -6324,6 +6347,8 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column:
>>> spark.createDataFrame([(42,)], ['a']).select(shiftright('a', 1).alias('r')).collect()
[Row(r=21)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("shiftright", _to_java_column(col), numBits)
@@ -6370,6 +6395,8 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column:
>>> df.select(shiftrightunsigned('a', 1).alias('r')).collect()
[Row(r=9223372036854775787)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("shiftrightunsigned", _to_java_column(col), numBits)
@@ -6684,6 +6711,8 @@ def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = Non
|1.3862943611198906|
+------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if arg2 is None:
return _invoke_function_over_columns("log", cast("ColumnOrName", arg1))
else:
@@ -6781,6 +6810,8 @@ def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column:
>>> df.select(conv(df.n, 2, 16).alias('hex')).collect()
[Row(hex='15')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("conv", _to_java_column(col), fromBase, toBase)
@@ -6894,6 +6925,8 @@ def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) ->
| b| 8| -1|
+---+---+-------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("lag", _to_java_column(col), offset, default)
@@ -6975,6 +7008,8 @@ def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) ->
| b| 8| -1|
+---+---+----------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("lead", _to_java_column(col), offset, default)
@@ -7049,6 +7084,8 @@ def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = Fa
| b| 8| 8|
+---+---+---------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("nth_value", _to_java_column(col), offset, ignoreNulls)
@@ -7561,6 +7598,8 @@ def date_format(date: "ColumnOrName", format: str) -> Column:
>>> df.select(date_format('dt', 'MM/dd/yyyy').alias('date')).collect()
[Row(date='04/08/2015')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("date_format", _to_java_column(date), format)
@@ -8418,6 +8457,8 @@ def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool
>>> df.select(months_between(df.date1, df.date2, False).alias('months')).collect()
[Row(months=3.9495967741935485)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function(
"months_between", _to_java_column(date1), _to_java_column(date2), roundOff
)
@@ -8459,6 +8500,8 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column:
>>> df.select(to_date(df.t, 'yyyy-MM-dd HH:mm:ss').alias('date')).collect()
[Row(date=datetime.date(1997, 2, 28))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if format is None:
return _invoke_function_over_columns("to_date", col)
else:
@@ -8595,6 +8638,8 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column:
|1997-02-28 10:30:00|
+-------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if format is None:
return _invoke_function_over_columns("to_timestamp", col)
else:
@@ -8852,6 +8897,8 @@ def trunc(date: "ColumnOrName", format: str) -> Column:
>>> df.select(trunc(df.d, 'mon').alias('month')).collect()
[Row(month=datetime.date(1997, 2, 1))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("trunc", _to_java_column(date), format)
@@ -8889,6 +8936,8 @@ def date_trunc(format: str, timestamp: "ColumnOrName") -> Column:
>>> df.select(date_trunc('mon', df.t).alias('month')).collect()
[Row(month=datetime.datetime(1997, 2, 1, 0, 0))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("date_trunc", format, _to_java_column(timestamp))
@@ -8922,6 +8971,8 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column:
>>> df.select(next_day(df.d, 'Sun').alias('date')).collect()
[Row(date=datetime.date(2015, 8, 2))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("next_day", _to_java_column(date), dayOfWeek)
@@ -8951,6 +9002,8 @@ def last_day(date: "ColumnOrName") -> Column:
>>> df.select(last_day(df.d).alias('date')).collect()
[Row(date=datetime.date(1997, 2, 28))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("last_day", _to_java_column(date))
@@ -8986,6 +9039,8 @@ def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss"
[Row(ts='2015-04-08 00:00:00')]
>>> spark.conf.unset("spark.sql.session.timeZone")
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("from_unixtime", _to_java_column(timestamp), format)
@@ -9066,6 +9121,8 @@ def unix_timestamp(
>>> spark.conf.unset("spark.sql.session.timeZone")
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if timestamp is None:
return _invoke_function("unix_timestamp")
return _invoke_function("unix_timestamp", _to_java_column(timestamp), format)
@@ -9120,6 +9177,8 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
>>> df.select(from_utc_timestamp(df.ts, df.tz).alias('local_time')).collect()
[Row(local_time=datetime.datetime(1997, 2, 28, 19, 30))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if isinstance(tz, Column):
tz = _to_java_column(tz)
return _invoke_function("from_utc_timestamp", _to_java_column(timestamp), tz)
@@ -9174,6 +9233,8 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
>>> df.select(to_utc_timestamp(df.ts, df.tz).alias('utc_time')).collect()
[Row(utc_time=datetime.datetime(1997, 2, 28, 1, 30))]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if isinstance(tz, Column):
tz = _to_java_column(tz)
return _invoke_function("to_utc_timestamp", _to_java_column(timestamp), tz)
@@ -9367,6 +9428,7 @@ def window(
|2016-03-11 09:00:05|2016-03-11 09:00:10| 1|
+-------------------+-------------------+---+
"""
+ from pyspark.sql.classic.column import _to_java_column
def check_string_field(field, fieldName): # type: ignore[no-untyped-def]
if not field or type(field) is not str:
@@ -9438,6 +9500,8 @@ def window_time(
... ).collect()
[Row(end='2016-03-11 09:00:10', window_time='2016-03-11 09:00:09.999999', sum=1)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
window_col = _to_java_column(windowColumn)
return _invoke_function("window_time", window_col)
@@ -9493,6 +9557,7 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str])
... w.session_window.end.cast("string").alias("end"), "sum").collect()
[Row(start='2016-03-11 09:00:07', end='2016-03-11 09:00:12', sum=1)]
"""
+ from pyspark.sql.classic.column import _to_java_column
def check_field(field: Union[Column, str], fieldName: str) -> None:
if field is None or not isinstance(field, (str, Column)):
@@ -9863,6 +9928,8 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column:
|Bob |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961|
+-----+----------------------------------------------------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if numBits not in [0, 224, 256, 384, 512]:
raise PySparkValueError(
error_class="VALUE_NOT_ALLOWED",
@@ -10000,6 +10067,8 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None
java.lang.RuntimeException: My error msg
...
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if errMsg is None:
return _invoke_function_over_columns("assert_true", col)
if not isinstance(errMsg, (str, Column)):
@@ -10042,6 +10111,8 @@ def raise_error(errMsg: Union[Column, str]) -> Column:
java.lang.RuntimeException: My error message
...
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if not isinstance(errMsg, (str, Column)):
raise PySparkTypeError(
error_class="NOT_COLUMN_OR_STR",
@@ -10368,6 +10439,8 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column:
>>> df.select(concat_ws('-', df.s, df.d).alias('s')).collect()
[Row(s='abcd-123')]
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function("concat_ws", sep, _to_seq(sc, cols, _to_java_column))
@@ -10405,6 +10478,8 @@ def decode(col: "ColumnOrName", charset: str) -> Column:
| abcd|
+----------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("decode", _to_java_column(col), charset)
@@ -10441,6 +10516,8 @@ def encode(col: "ColumnOrName", charset: str) -> Column:
| [61 62 63 64]|
+----------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("encode", _to_java_column(col), charset)
@@ -10470,6 +10547,8 @@ def format_number(col: "ColumnOrName", d: int) -> Column:
>>> spark.createDataFrame([(5,)], ['a']).select(format_number('a', 4).alias('v')).collect()
[Row(v='5.0000')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("format_number", _to_java_column(col), d)
@@ -10501,6 +10580,8 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column:
>>> df.select(format_string('%d %s', df.a, df.b).alias('v')).collect()
[Row(v='5 hello')]
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function("format_string", format, _to_seq(sc, cols, _to_java_column))
@@ -10539,6 +10620,8 @@ def instr(str: "ColumnOrName", substr: str) -> Column:
>>> df.select(instr(df.s, 'b').alias('s')).collect()
[Row(s=2)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("instr", _to_java_column(str), substr)
@@ -10585,6 +10668,8 @@ def overlay(
>>> df.select(overlay("x", "y", 7, 2).alias("overlayed")).collect()
[Row(overlayed='SPARK_COREL')]
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if not isinstance(pos, (int, str, Column)):
raise PySparkTypeError(
error_class="NOT_COLUMN_OR_INT_OR_STR",
@@ -10692,6 +10777,8 @@ def substring(str: "ColumnOrName", pos: int, len: int) -> Column:
>>> df.select(substring(df.s, 1, 2).alias('s')).collect()
[Row(s='ab')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("substring", _to_java_column(str), pos, len)
@@ -10730,6 +10817,8 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column:
>>> df.select(substring_index(df.s, '.', -3).alias('s')).collect()
[Row(s='b.c.d')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("substring_index", _to_java_column(str), delim, count)
@@ -10770,6 +10859,8 @@ def levenshtein(
>>> df0.select(levenshtein('l', 'r', 2).alias('d')).collect()
[Row(d=-1)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if threshold is None:
return _invoke_function_over_columns("levenshtein", left, right)
else:
@@ -10813,6 +10904,8 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column:
>>> df.select(locate('b', df.s, 1).alias('s')).collect()
[Row(s=2)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("locate", substr, _to_java_column(str), pos)
@@ -10846,6 +10939,8 @@ def lpad(col: "ColumnOrName", len: int, pad: str) -> Column:
>>> df.select(lpad(df.s, 6, '#').alias('s')).collect()
[Row(s='##abcd')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("lpad", _to_java_column(col), len, pad)
@@ -10879,6 +10974,8 @@ def rpad(col: "ColumnOrName", len: int, pad: str) -> Column:
>>> df.select(rpad(df.s, 6, '#').alias('s')).collect()
[Row(s='abcd##')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("rpad", _to_java_column(col), len, pad)
@@ -11242,6 +11339,8 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column:
>>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect()
[Row(d='')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("regexp_extract", _to_java_column(str), pattern, idx)
@@ -11320,6 +11419,8 @@ def regexp_replace(
>>> df.select(regexp_replace("str", col("pattern"), col("replacement")).alias('d')).collect()
[Row(d='-----')]
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if isinstance(pattern, str):
pattern_col = _create_column_from_literal(pattern)
else:
@@ -11668,6 +11769,8 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column:
... .alias('r')).collect()
[Row(r='1a2s3ae')]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("translate", _to_java_column(srcCol), matching, replace)
@@ -12082,6 +12185,8 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column:
| aa123cc|
+---------------+
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function("printf", _to_java_column(format), _to_seq(sc, cols, _to_java_column))
@@ -12634,6 +12739,8 @@ def elt(*inputs: "ColumnOrName") -> Column:
>>> df.select(elt(df.a, df.b, df.c).alias('r')).collect()
[Row(r='scala')]
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function("elt", _to_seq(sc, inputs, _to_java_column))
@@ -12935,6 +13042,8 @@ def collate(col: "ColumnOrName", collation: str) -> Column:
:class:`~pyspark.sql.Column`
A new column of string type, where each value has the specified collation.
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("collate", _to_java_column(col), collation)
@@ -13322,6 +13431,8 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column:
| true|
+----------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
value = value._jc if isinstance(value, Column) else value
return _invoke_function("array_contains", _to_java_column(col), value)
@@ -13569,6 +13680,8 @@ def array_join(
| NULL,NULL|
+-------------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
_get_active_spark_context()
if null_replacement is None:
return _invoke_function("array_join", _to_java_column(col), delimiter)
@@ -13749,6 +13862,8 @@ def array_position(col: "ColumnOrName", value: Any) -> Column:
| 3|
+-----------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("array_position", _to_java_column(col), value)
@@ -14183,6 +14298,8 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column:
| []|
+---------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("array_remove", _to_java_column(col), element)
@@ -15326,6 +15443,8 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column:
... get_json_object(df.jstring, '$.f2').alias("c1") ).collect()
[Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("get_json_object", _to_java_column(col), path)
@@ -15357,6 +15476,8 @@ def json_tuple(col: "ColumnOrName", *fields: str) -> Column:
>>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect()
[Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
if len(fields) == 0:
raise PySparkValueError(
error_class="CANNOT_BE_EMPTY",
@@ -15464,6 +15585,7 @@ def from_json(
|[1, 2, 3]|
+---------+
"""
+ from pyspark.sql.classic.column import _to_java_column
if isinstance(schema, DataType):
schema = schema.json()
@@ -15497,6 +15619,7 @@ def parse_json(
>>> df.select(to_json(parse_json(df.json))).collect()
[Row(to_json(parse_json(json))='{"a":1}')]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("parse_json", _to_java_column(col))
@@ -15525,6 +15648,7 @@ def is_variant_null(v: "ColumnOrName") -> Column:
>>> df.select(is_variant_null(parse_json(df.json)).alias("r")).collect()
[Row(r=False)]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("is_variant_null", _to_java_column(v))
@@ -15560,6 +15684,7 @@ def variant_get(v: "ColumnOrName", path: str, targetType: str) -> Column:
>>> df.select(variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect()
[Row(r=None)]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("variant_get", _to_java_column(v), path, targetType)
@@ -15597,6 +15722,7 @@ def try_variant_get(v: "ColumnOrName", path: str, targetType: str) -> Column:
>>> df.select(try_variant_get(parse_json(df.json), "$.a", "binary").alias("r")).collect()
[Row(r=None)]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("try_variant_get", _to_java_column(v), path, targetType)
@@ -15624,6 +15750,7 @@ def schema_of_variant(v: "ColumnOrName") -> Column:
>>> df.select(schema_of_variant(parse_json(df.json)).alias("r")).collect()
[Row(r='STRUCT<a: BIGINT>')]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("schema_of_variant", _to_java_column(v))
@@ -15651,6 +15778,7 @@ def schema_of_variant_agg(v: "ColumnOrName") -> Column:
>>> df.select(schema_of_variant_agg(parse_json(df.json)).alias("r")).collect()
[Row(r='STRUCT<a: BIGINT>')]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("schema_of_variant_agg", _to_java_column(v))
@@ -15745,6 +15873,7 @@ def to_json(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Co
|["Alice","Bob"]|
+---------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("to_json", _to_java_column(col), _options_to_str(options))
@@ -15787,6 +15916,8 @@ def schema_of_json(json: Union[Column, str], options: Optional[Dict[str, str]] =
>>> df.select(schema.alias("json")).collect()
[Row(json='STRUCT<a: BIGINT>')]
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if isinstance(json, str):
col = _create_column_from_literal(json)
elif isinstance(json, Column):
@@ -15924,6 +16055,7 @@ def from_xml(
>>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect()
[Row(xml=Row(a=[1, 2]))]
"""
+ from pyspark.sql.classic.column import _to_java_column
if isinstance(schema, StructType):
schema = schema.json()
@@ -16002,6 +16134,8 @@ def schema_of_xml(xml: Union[Column, str], options: Optional[Dict[str, str]] = N
... ).collect()
[Row(xml='STRUCT<values: STRUCT<value: ARRAY<BIGINT>>>')]
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if isinstance(xml, str):
col = _create_column_from_literal(xml)
elif isinstance(xml, Column):
@@ -16047,6 +16181,7 @@ def to_xml(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Col
>>> df.select(to_xml(df.value, {'rowTag':'person'}).alias("xml")).collect()
[Row(xml='<person>\\n <age>2</age>\\n <name>Alice</name>\\n</person>')]
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("to_xml", _to_java_column(col), _options_to_str(options))
@@ -16123,6 +16258,8 @@ def schema_of_csv(csv: Union[Column, str], options: Optional[Dict[str, str]] = N
|STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>|
+-------------------------------------------+
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
if isinstance(csv, str):
col = _create_column_from_literal(csv)
elif isinstance(csv, Column):
@@ -16223,6 +16360,7 @@ def to_csv(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Col
| 2,Alice,true|
+-------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
return _invoke_function("to_csv", _to_java_column(col), _options_to_str(options))
@@ -16630,6 +16768,8 @@ def sort_array(col: "ColumnOrName", asc: bool = True) -> Column:
| [NULL, NULL, NULL]|
+----------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("sort_array", _to_java_column(col), asc)
@@ -16923,6 +17063,8 @@ def map_contains_key(col: "ColumnOrName", value: Any) -> Column:
| false|
+--------------------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("map_contains_key", _to_java_column(col), value)
@@ -17653,6 +17795,7 @@ def from_csv(
| {1, 2, 3}|
+---------------+
"""
+ from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
_get_active_spark_context()
if isinstance(schema, str):
@@ -17681,6 +17824,7 @@ def _unresolved_named_lambda_variable(*name_parts: Any) -> Column:
name_parts : str
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_seq
sc = _get_active_spark_context()
name_parts_seq = _to_seq(sc, name_parts)
@@ -17731,6 +17875,7 @@ def _create_lambda(f: Callable) -> Callable:
- (Column, Column, Column) -> Column: ...
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_seq
parameters = _get_lambda_parameters(f)
@@ -17776,6 +17921,7 @@ def _invoke_higher_order_function(
:return: a Column
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_java_column
sc = _get_active_spark_context()
expressions = cast(JVMView, sc._jvm).org.apache.spark.sql.catalyst.expressions
@@ -19414,6 +19560,8 @@ def call_udf(udfName: str, *cols: "ColumnOrName") -> Column:
| cc|
+-----------+
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function("call_udf", udfName, _to_seq(sc, cols, _to_java_column))
@@ -19483,6 +19631,8 @@ def call_function(funcName: str, *cols: "ColumnOrName") -> Column:
| 102.0|
+------------------------------------+
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
sc = _get_active_spark_context()
return _invoke_function("call_function", funcName, _to_seq(sc, cols, _to_java_column))
@@ -19498,6 +19648,8 @@ def unwrap_udt(col: "ColumnOrName") -> Column:
-----
Supports Spark Connect.
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("unwrap_udt", _to_java_column(col))
@@ -19649,6 +19801,8 @@ def hll_sketch_estimate(col: "ColumnOrName") -> Column:
| 3|
+------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
return _invoke_function("hll_sketch_estimate", _to_java_column(col))
@@ -19687,6 +19841,8 @@ def hll_union(
| 6|
+------------+
"""
+ from pyspark.sql.classic.column import _to_java_column
+
if allowDifferentLgConfigK is not None:
return _invoke_function(
"hll_union", _to_java_column(col1), _to_java_column(col2), allowDifferentLgConfigK
diff --git a/python/pyspark/sql/functions/partitioning.py b/python/pyspark/sql/functions/partitioning.py
index 59c293577b08..e89901cdbe54 100644
--- a/python/pyspark/sql/functions/partitioning.py
+++ b/python/pyspark/sql/functions/partitioning.py
@@ -26,7 +26,7 @@ from typing import (
)
from pyspark.errors import PySparkTypeError
-from pyspark.sql.column import Column, _to_java_column, _create_column_from_literal
+from pyspark.sql.column import Column
from pyspark.sql.functions.builtin import _invoke_function_over_columns, _invoke_function
from pyspark.sql.utils import (
try_partitioning_remote_functions as _try_partitioning_remote_functions,
@@ -204,6 +204,8 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column:
method of the `DataFrameWriterV2`.
"""
+ from pyspark.sql.classic.column import _to_java_column, _create_column_from_literal
+
if not isinstance(numBuckets, (int, Column)):
raise PySparkTypeError(
error_class="NOT_COLUMN_OR_INT",
diff --git a/python/pyspark/sql/group.py b/python/pyspark/sql/group.py
index 15934c24b9d4..d26e23bc7160 100644
--- a/python/pyspark/sql/group.py
+++ b/python/pyspark/sql/group.py
@@ -19,7 +19,7 @@ import sys
from typing import Callable, List, Optional, TYPE_CHECKING, overload, Dict, Union, cast, Tuple
-from pyspark.sql.column import Column, _to_seq
+from pyspark.sql.column import Column
from pyspark.sql.session import SparkSession
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin
@@ -43,6 +43,8 @@ def dfapi(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]:
def df_varargs_api(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]:
+ from pyspark.sql.classic.column import _to_seq
+
def _api(self: "GroupedData", *cols: str) -> DataFrame:
name = f.__name__
jdf = getattr(self._jgd, name)(_to_seq(self.session._sc, cols))
@@ -175,6 +177,8 @@ class GroupedData(PandasGroupedOpsMixin):
| Bob| 5|
+-----+------------+
"""
+ from pyspark.sql.classic.column import _to_seq
+
assert exprs, "exprs should not be empty"
if len(exprs) == 1 and isinstance(exprs[0], dict):
jdf = self._jgd.agg(exprs[0])
diff --git a/python/pyspark/sql/observation.py b/python/pyspark/sql/observation.py
index 1dae5086e3dd..4ef4c78ba3c3 100644
--- a/python/pyspark/sql/observation.py
+++ b/python/pyspark/sql/observation.py
@@ -18,7 +18,6 @@ import os
from typing import Any, Dict, Optional, TYPE_CHECKING
from pyspark.errors import PySparkTypeError, PySparkValueError, PySparkAssertionError
-from pyspark.sql import column
from pyspark.sql.column import Column
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.utils import is_remote
@@ -116,6 +115,8 @@ class Observation:
:class:`DataFrame`
the observed :class:`DataFrame`.
"""
+ from pyspark.sql.classic.column import _to_seq
+
if self._jo is not None:
raise PySparkAssertionError(error_class="REUSE_OBSERVATION", message_parameters={})
@@ -124,7 +125,9 @@ class Observation:
cls = self._jvm.org.apache.spark.sql.Observation
self._jo = cls(self._name) if self._name is not None else cls()
observed_df = self._jo.on(
- df._jdf, exprs[0]._jc, column._to_seq(df._sc, [c._jc for c in exprs[1:]])
+ df._jdf,
+ exprs[0]._jc,
+ _to_seq(df._sc, [c._jc for c in exprs[1:]]),
)
return DataFrame(observed_df, df.sparkSession)
diff --git a/python/pyspark/sql/protobuf/functions.py b/python/pyspark/sql/protobuf/functions.py
index 63871e437571..1e75874e75f9 100644
--- a/python/pyspark/sql/protobuf/functions.py
+++ b/python/pyspark/sql/protobuf/functions.py
@@ -22,7 +22,7 @@ A collections of builtin protobuf functions
from typing import Dict, Optional, TYPE_CHECKING, cast
-from pyspark.sql.column import Column, _to_java_column
+from pyspark.sql.column import Column
from pyspark.sql.utils import get_active_spark_context, try_remote_protobuf_functions
from pyspark.util import _print_missing_jar
@@ -139,6 +139,7 @@ def from_protobuf(
+------------------+
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_java_column
sc = get_active_spark_context()
try:
@@ -260,6 +261,7 @@ def to_protobuf(
+----------------------------+
"""
from py4j.java_gateway import JVMView
+ from pyspark.sql.classic.column import _to_java_column
sc = get_active_spark_context()
try:
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 26fe8c5e6fa2..0ee235c8bf02 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -18,7 +18,7 @@ import sys
from typing import cast, overload, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING, Union
from pyspark.util import is_remote_only
-from pyspark.sql.column import _to_seq, _to_java_column, Column
+from pyspark.sql.column import Column
from pyspark.sql.types import StructType
from pyspark.sql import utils
from pyspark.sql.utils import to_str
@@ -619,6 +619,8 @@ class DataFrameReader(OptionUtils):
| Tom| 20| NULL|
+-----+----+------+
"""
+ from pyspark.sql.classic.column import _to_seq
+
mergeSchema = options.get("mergeSchema", None)
pathGlobFilter = options.get("pathGlobFilter", None)
modifiedBefore = options.get("modifiedBefore", None)
@@ -1042,6 +1044,8 @@ class DataFrameReader(OptionUtils):
|100|Hyukjin Kwon|
+---+------------+
"""
+ from pyspark.sql.classic.column import _to_seq
+
self._set_opts(
mergeSchema=mergeSchema,
pathGlobFilter=pathGlobFilter,
@@ -1440,6 +1444,8 @@ class DataFrameWriter(OptionUtils):
|100|
+---+
"""
+ from pyspark.sql.classic.column import _to_seq
+
if len(cols) == 1 and isinstance(cols[0], (list, tuple)):
cols = cols[0] # type: ignore[assignment]
self._jwrite = self._jwrite.partitionBy(
@@ -1503,6 +1509,8 @@ class DataFrameWriter(OptionUtils):
+---+------------+
>>> _ = spark.sql("DROP TABLE bucketed_table")
"""
+ from pyspark.sql.classic.column import _to_seq
+
if not isinstance(numBuckets, int):
raise PySparkTypeError(
error_class="NOT_INT",
@@ -1594,6 +1602,8 @@ class DataFrameWriter(OptionUtils):
+---+------------+
>>> _ = spark.sql("DROP TABLE sorted_bucketed_table")
"""
+ from pyspark.sql.classic.column import _to_seq
+
if isinstance(col, (list, tuple)):
if cols:
raise PySparkValueError(
@@ -2380,6 +2390,8 @@ class DataFrameWriterV2:
.. versionadded: 3.1.0
"""
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
col = _to_java_column(col)
cols = _to_seq(self._spark._sc, [_to_java_column(c) for c in cols])
self._jwriter.partitionedBy(col, cols)
@@ -2435,6 +2447,8 @@ class DataFrameWriterV2:
.. versionadded: 3.1.0
"""
+ from pyspark.sql.classic.column import _to_java_column
+
condition = _to_java_column(condition)
self._jwriter.overwrite(condition)
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index e0c2c815c827..1a01982e756d 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -40,7 +40,6 @@ from typing import (
from pyspark.conf import SparkConf
from pyspark.util import is_remote_only
-from pyspark.sql.column import _to_java_column
from pyspark.sql.conf import RuntimeConfig
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.functions import lit
@@ -1724,6 +1723,7 @@ class SparkSession(SparkConversionMixin):
| 3| 6| 1|
+---+---+---+
"""
+ from pyspark.sql.classic.column import _to_java_column
formatter = SQLStringFormatter(self)
if len(kwargs) > 0:
diff --git a/python/pyspark/sql/streaming/readwriter.py b/python/pyspark/sql/streaming/readwriter.py
index 58901f34cfc9..c2b75dd8f167 100644
--- a/python/pyspark/sql/streaming/readwriter.py
+++ b/python/pyspark/sql/streaming/readwriter.py
@@ -19,7 +19,6 @@ import sys
from collections.abc import Iterator
from typing import cast, overload, Any, Callable, List, Optional, TYPE_CHECKING, Union
-from pyspark.sql.column import _to_seq
from pyspark.sql.readwriter import OptionUtils, to_str
from pyspark.sql.streaming.query import StreamingQuery
from pyspark.sql.types import Row, StructType
@@ -1117,6 +1116,8 @@ class DataStreamWriter:
+...---------+-----+
...
"""
+ from pyspark.sql.classic.column import _to_seq
+
if len(cols) == 1 and isinstance(cols[0], (list, tuple)):
cols = cols[0]
self._jwrite = self._jwrite.partitionBy(_to_seq(self._spark._sc, cols))
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
index ec413d048d8e..9edd585da6a0 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
@@ -1334,7 +1334,7 @@ class ScalarPandasUDFTestsMixin:
return x + 1
def f2(x):
- assert type(x) == col_type
+ assert isinstance(x, col_type)
return x + 10
@pandas_udf("int")
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index e51ae69814bd..92921685d58e 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -42,7 +42,7 @@ class ColumnTestsMixin:
def test_validate_column_types(self):
from pyspark.sql.functions import udf, to_json
- from pyspark.sql.column import _to_java_column
+ from pyspark.sql.classic.column import _to_java_column
self.assertTrue("Column" in _to_java_column("a").getClass().toString())
self.assertTrue("Column" in _to_java_column("a").getClass().toString())
@@ -248,6 +248,13 @@ class ColumnTestsMixin:
message_parameters={"arg_name": "window", "arg_type": "int"},
)
+ def test_union_classmethod_usage(self):
+ df = self.spark.range(1)
+ self.assertEqual(df.select(Column.eqNullSafe(df.id, df.id)).first()[0], True)
+
+ def test_isinstance_dataframe(self):
+ self.assertIsInstance(self.spark.range(1).id, Column)
+
class ColumnTests(ColumnTestsMixin, ReusedSQLTestCase):
pass
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 0d0fc9042e62..7be59a28e479 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -27,7 +27,7 @@ from typing import Callable, Any, TYPE_CHECKING, Optional, cast, Union
from pyspark.util import PythonEvalType
-from pyspark.sql.column import Column, _to_java_expr, _to_seq
+from pyspark.sql.column import Column
from pyspark.sql.types import (
DataType,
StringType,
@@ -395,6 +395,8 @@ class UserDefinedFunction:
return judf
def __call__(self, *args: "ColumnOrName", **kwargs: "ColumnOrName") -> Column:
+ from pyspark.sql.classic.column import _to_java_expr, _to_seq
+
sc = get_active_spark_context()
assert sc._jvm is not None
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index 801ecc605e50..f56088020223 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -26,7 +26,6 @@ from typing import Any, Type, TYPE_CHECKING, Optional, Sequence, Union
from pyspark.errors import PySparkAttributeError, PySparkPicklingError, PySparkTypeError
from pyspark.util import PythonEvalType
-from pyspark.sql.column import _to_java_column, _to_java_expr, _to_seq
from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
from pyspark.sql.types import DataType, StructType, _parse_datatype_string
from pyspark.sql.udf import _wrap_function
@@ -374,6 +373,8 @@ class UserDefinedTableFunction:
return judtf
def __call__(self, *args: "ColumnOrName", **kwargs: "ColumnOrName") -> "DataFrame":
+ from pyspark.sql.classic.column import _to_java_column, _to_java_expr, _to_seq
+
from pyspark.sql import DataFrame, SparkSession
spark = SparkSession._getActiveSessionOrCreate()
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index e830806eee64..de7d58361c04 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -329,6 +329,33 @@ def dispatch_df_method(f: FuncT) -> FuncT:
return cast(FuncT, wrapped)
+def dispatch_col_method(f: FuncT) -> FuncT:
+ """
+ For the usecases of direct Column.method(col, ...), it checks if self
+ is a Connect DataFrame or Classic DataFrame, and dispatches.
+ """
+
+ @functools.wraps(f)
+ def wrapped(*args: Any, **kwargs: Any) -> Any:
+ if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
+ from pyspark.sql.connect.column import Column as ConnectColumn
+
+ if isinstance(args[0], ConnectColumn):
+ return getattr(ConnectColumn, f.__name__)(*args, **kwargs)
+ else:
+ from pyspark.sql.classic.column import Column as ClassicColumn
+
+ if isinstance(args[0], ClassicColumn):
+ return getattr(ClassicColumn, f.__name__)(*args, **kwargs)
+
+ raise PySparkNotImplementedError(
+ error_class="NOT_IMPLEMENTED",
+ message_parameters={"feature": f"DataFrame.{f.__name__}"},
+ )
+
+ return cast(FuncT, wrapped)
+
+
def pyspark_column_op(
func_name: str, left: "IndexOpsLike", right: Any, fillna: Any = None
) -> Union["SeriesOrIndex", None]:
@@ -359,7 +386,7 @@ def get_column_class() -> Type["Column"]:
if is_remote():
from pyspark.sql.connect.column import Column as ConnectColumn
- return ConnectColumn # type: ignore[return-value]
+ return ConnectColumn
else:
return PySparkColumn
diff --git a/python/pyspark/sql/window.py b/python/pyspark/sql/window.py
index 42d50dc1b3bd..93ed5c172ffe 100644
--- a/python/pyspark/sql/window.py
+++ b/python/pyspark/sql/window.py
@@ -17,7 +17,6 @@
import sys
from typing import cast, Iterable, List, Tuple, TYPE_CHECKING, Union
-from pyspark.sql.column import _to_seq, _to_java_column
from pyspark.sql.utils import (
try_remote_window,
try_remote_windowspec,
@@ -32,6 +31,8 @@ __all__ = ["Window", "WindowSpec"]
def _to_java_cols(cols: Tuple[Union["ColumnOrName", List["ColumnOrName_"]], ...]) -> "JavaObject":
+ from pyspark.sql.classic.column import _to_seq, _to_java_column
+
if len(cols) == 1 and isinstance(cols[0], list):
cols = cols[0] # type: ignore[assignment]
sc = get_active_spark_context()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org