You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/04/23 05:13:23 UTC

(spark) branch master updated: [SPARK-47933][PYTHON] Parent Column class for Spark Connect and Spark Classic

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new e01ac581f46a [SPARK-47933][PYTHON] Parent Column class for Spark Connect and Spark Classic
e01ac581f46a is described below

commit e01ac581f46aa595e66daf33fe92b56d1328bc78
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Tue Apr 23 14:13:09 2024 +0900

    [SPARK-47933][PYTHON] Parent Column class for Spark Connect and Spark Classic
    
    ### What changes were proposed in this pull request?
    
    Same as https://github.com/apache/spark/pull/46129 but for `Column` class.
    
    ### Why are the changes needed?
    
    Same as https://github.com/apache/spark/pull/46129
    
    ### Does this PR introduce _any_ user-facing change?
    
    Same as https://github.com/apache/spark/pull/46129
    
    ### How was this patch tested?
    
    Manually tested, and CI should verify them.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #46155 from HyukjinKwon/SPARK-47933.
    
    Authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 dev/sparktestsupport/modules.py                    |    2 +-
 python/pyspark/ml/connect/functions.py             |    2 +-
 python/pyspark/ml/functions.py                     |    2 +-
 python/pyspark/ml/stat.py                          |    3 +-
 python/pyspark/pandas/internal.py                  |    2 +-
 python/pyspark/pandas/spark/functions.py           |   44 +-
 python/pyspark/sql/avro/functions.py               |    4 +-
 python/pyspark/sql/classic/column.py               |  637 ++++++++++
 python/pyspark/sql/classic/dataframe.py            |    3 +-
 python/pyspark/sql/column.py                       | 1211 +++++++++-----------
 python/pyspark/sql/connect/_typing.py              |    2 +-
 python/pyspark/sql/connect/avro/functions.py       |    3 +-
 python/pyspark/sql/connect/column.py               |  404 ++++---
 python/pyspark/sql/connect/dataframe.py            |   24 +-
 python/pyspark/sql/connect/functions/builtin.py    |   55 +-
 .../pyspark/sql/connect/functions/partitioning.py  |    2 +-
 python/pyspark/sql/connect/group.py                |    8 +-
 python/pyspark/sql/connect/observation.py          |    2 +-
 python/pyspark/sql/connect/plan.py                 |    5 +-
 python/pyspark/sql/connect/protobuf/functions.py   |    2 +-
 python/pyspark/sql/connect/udf.py                  |    5 +-
 python/pyspark/sql/connect/udtf.py                 |    5 +-
 python/pyspark/sql/connect/window.py               |    6 +-
 python/pyspark/sql/functions/builtin.py            |  158 ++-
 python/pyspark/sql/functions/partitioning.py       |    4 +-
 python/pyspark/sql/group.py                        |    6 +-
 python/pyspark/sql/observation.py                  |    7 +-
 python/pyspark/sql/protobuf/functions.py           |    4 +-
 python/pyspark/sql/readwriter.py                   |   16 +-
 python/pyspark/sql/session.py                      |    2 +-
 python/pyspark/sql/streaming/readwriter.py         |    3 +-
 .../sql/tests/pandas/test_pandas_udf_scalar.py     |    2 +-
 python/pyspark/sql/tests/test_column.py            |    9 +-
 python/pyspark/sql/udf.py                          |    4 +-
 python/pyspark/sql/udtf.py                         |    3 +-
 python/pyspark/sql/utils.py                        |   29 +-
 python/pyspark/sql/window.py                       |    3 +-
 37 files changed, 1766 insertions(+), 917 deletions(-)

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 593cae5902c8..8a94187bfced 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -476,7 +476,7 @@ pyspark_sql = Module(
         "pyspark.sql.session",
         "pyspark.sql.conf",
         "pyspark.sql.catalog",
-        "pyspark.sql.column",
+        "pyspark.sql.classic.column",
         "pyspark.sql.classic.dataframe",
         "pyspark.sql.datasource",
         "pyspark.sql.group",
diff --git a/python/pyspark/ml/connect/functions.py b/python/pyspark/ml/connect/functions.py
index b305c04519ae..6597e6c4118a 100644
--- a/python/pyspark/ml/connect/functions.py
+++ b/python/pyspark/ml/connect/functions.py
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 from pyspark.ml import functions as PyMLFunctions
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col, lit
 
 
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index 466d94ccc888..25bbc290a3e9 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -28,7 +28,7 @@ except ImportError:
     pass  # Let it throw a better error message later when the API is invoked.
 
 from pyspark.sql.functions import pandas_udf
-from pyspark.sql.column import Column, _to_java_column
+from pyspark.sql.classic.column import Column, _to_java_column
 from pyspark.sql.types import (
     ArrayType,
     ByteType,
diff --git a/python/pyspark/ml/stat.py b/python/pyspark/ml/stat.py
index ec5da94079ea..d6020607aff2 100644
--- a/python/pyspark/ml/stat.py
+++ b/python/pyspark/ml/stat.py
@@ -22,7 +22,8 @@ from pyspark import since
 from pyspark.ml.common import _java2py, _py2java
 from pyspark.ml.linalg import Matrix, Vector
 from pyspark.ml.wrapper import JavaWrapper, _jvm
-from pyspark.sql.column import Column, _to_seq
+from pyspark.sql.column import Column
+from pyspark.sql.classic.column import _to_seq
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import lit
 
diff --git a/python/pyspark/pandas/internal.py b/python/pyspark/pandas/internal.py
index 2966db073d0c..767ec9a57f9b 100644
--- a/python/pyspark/pandas/internal.py
+++ b/python/pyspark/pandas/internal.py
@@ -959,7 +959,7 @@ class InternalFrame:
 
                 return sdf.select(
                     ConnectColumn(DistributedSequenceID()).alias(column_name),
-                    "*",  # type: ignore[call-overload]
+                    "*",
                 )
             else:
                 return PySparkDataFrame(
diff --git a/python/pyspark/pandas/spark/functions.py b/python/pyspark/pandas/spark/functions.py
index a598ff37c47f..91602ae2b2b8 100644
--- a/python/pyspark/pandas/spark/functions.py
+++ b/python/pyspark/pandas/spark/functions.py
@@ -25,9 +25,9 @@ def product(col: Column, dropna: bool) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_product",
-            col,  # type: ignore[arg-type]
+            col,
             lit(dropna),
         )
 
@@ -42,9 +42,9 @@ def stddev(col: Column, ddof: int) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_stddev",
-            col,  # type: ignore[arg-type]
+            col,
             lit(ddof),
         )
 
@@ -59,9 +59,9 @@ def var(col: Column, ddof: int) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_var",
-            col,  # type: ignore[arg-type]
+            col,
             lit(ddof),
         )
 
@@ -76,9 +76,9 @@ def skew(col: Column) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_skew",
-            col,  # type: ignore[arg-type]
+            col,
         )
 
     else:
@@ -92,9 +92,9 @@ def kurt(col: Column) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_kurt",
-            col,  # type: ignore[arg-type]
+            col,
         )
 
     else:
@@ -108,9 +108,9 @@ def mode(col: Column, dropna: bool) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_mode",
-            col,  # type: ignore[arg-type]
+            col,
             lit(dropna),
         )
 
@@ -125,10 +125,10 @@ def covar(col1: Column, col2: Column, ddof: int) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "pandas_covar",
-            col1,  # type: ignore[arg-type]
-            col2,  # type: ignore[arg-type]
+            col1,
+            col2,
             lit(ddof),
         )
 
@@ -143,9 +143,9 @@ def ewm(col: Column, alpha: float, ignore_na: bool) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "ewm",
-            col,  # type: ignore[arg-type]
+            col,
             lit(alpha),
             lit(ignore_na),
         )
@@ -161,9 +161,9 @@ def null_index(col: Column) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "null_index",
-            col,  # type: ignore[arg-type]
+            col,
         )
 
     else:
@@ -177,11 +177,11 @@ def timestampdiff(unit: str, start: Column, end: Column) -> Column:
     if is_remote():
         from pyspark.sql.connect.functions.builtin import _invoke_function_over_columns, lit
 
-        return _invoke_function_over_columns(  # type: ignore[return-value]
+        return _invoke_function_over_columns(
             "timestampdiff",
             lit(unit),
-            start,  # type: ignore[arg-type]
-            end,  # type: ignore[arg-type]
+            start,
+            end,
         )
 
     else:
diff --git a/python/pyspark/sql/avro/functions.py b/python/pyspark/sql/avro/functions.py
index 5cebfa384045..40a4ce337db0 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -22,7 +22,7 @@ A collections of builtin avro functions
 
 from typing import Dict, Optional, TYPE_CHECKING, cast
 
-from pyspark.sql.column import Column, _to_java_column
+from pyspark.sql.column import Column
 from pyspark.sql.utils import get_active_spark_context, try_remote_avro_functions
 from pyspark.util import _print_missing_jar
 
@@ -78,6 +78,7 @@ def from_avro(
     [Row(value=Row(avro=Row(age=2, name='Alice')))]
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_java_column
 
     sc = get_active_spark_context()
     try:
@@ -128,6 +129,7 @@ def to_avro(data: "ColumnOrName", jsonFormatSchema: str = "") -> Column:
     [Row(suite=bytearray(b'\\x02\\x00'))]
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_java_column
 
     sc = get_active_spark_context()
     try:
diff --git a/python/pyspark/sql/classic/column.py b/python/pyspark/sql/classic/column.py
new file mode 100644
index 000000000000..dc8a9a1c93b4
--- /dev/null
+++ b/python/pyspark/sql/classic/column.py
@@ -0,0 +1,637 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import json
+import warnings
+from typing import (
+    cast,
+    overload,
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    TYPE_CHECKING,
+    Union,
+)
+
+from pyspark.sql.column import Column as ParentColumn
+from pyspark.errors import PySparkAttributeError, PySparkTypeError, PySparkValueError
+from pyspark.errors.utils import with_origin_to_class
+from pyspark.sql.types import DataType
+from pyspark.sql.utils import get_active_spark_context
+
+if TYPE_CHECKING:
+    from py4j.java_gateway import JavaObject
+    from pyspark.core.context import SparkContext
+    from pyspark.sql._typing import ColumnOrName, LiteralType, DecimalLiteral, DateTimeLiteral
+    from pyspark.sql.window import WindowSpec
+
+__all__ = ["Column"]
+
+
+def _create_column_from_literal(
+    literal: Union["LiteralType", "DecimalLiteral", "DateTimeLiteral", "ParentColumn"]
+) -> "JavaObject":
+    from py4j.java_gateway import JVMView
+
+    sc = get_active_spark_context()
+    return cast(JVMView, sc._jvm).functions.lit(literal)
+
+
+def _create_column_from_name(name: str) -> "JavaObject":
+    from py4j.java_gateway import JVMView
+
+    sc = get_active_spark_context()
+    return cast(JVMView, sc._jvm).functions.col(name)
+
+
+def _to_java_column(col: "ColumnOrName") -> "JavaObject":
+    if isinstance(col, Column):
+        jcol = col._jc
+    elif isinstance(col, str):
+        jcol = _create_column_from_name(col)
+    else:
+        raise PySparkTypeError(
+            error_class="NOT_COLUMN_OR_STR",
+            message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
+        )
+    return jcol
+
+
+def _to_java_expr(col: "ColumnOrName") -> "JavaObject":
+    return _to_java_column(col).expr()
+
+
+@overload
+def _to_seq(sc: "SparkContext", cols: Iterable["JavaObject"]) -> "JavaObject":
+    ...
+
+
+@overload
+def _to_seq(
+    sc: "SparkContext",
+    cols: Iterable["ColumnOrName"],
+    converter: Optional[Callable[["ColumnOrName"], "JavaObject"]],
+) -> "JavaObject":
+    ...
+
+
+def _to_seq(
+    sc: "SparkContext",
+    cols: Union[Iterable["ColumnOrName"], Iterable["JavaObject"]],
+    converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
+) -> "JavaObject":
+    """
+    Convert a list of Columns (or names) into a JVM Seq of Column.
+
+    An optional `converter` could be used to convert items in `cols`
+    into JVM Column objects.
+    """
+    if converter:
+        cols = [converter(c) for c in cols]
+    assert sc._jvm is not None
+    return sc._jvm.PythonUtils.toSeq(cols)
+
+
+def _to_list(
+    sc: "SparkContext",
+    cols: List["ColumnOrName"],
+    converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
+) -> "JavaObject":
+    """
+    Convert a list of Columns (or names) into a JVM (Scala) List of Columns.
+
+    An optional `converter` could be used to convert items in `cols`
+    into JVM Column objects.
+    """
+    if converter:
+        cols = [converter(c) for c in cols]
+    assert sc._jvm is not None
+    return sc._jvm.PythonUtils.toList(cols)
+
+
+def _unary_op(name: str, self: ParentColumn) -> ParentColumn:
+    """Create a method for given unary operator"""
+
+    jc = getattr(self._jc, name)()
+    return Column(jc)
+
+
+def _func_op(name: str, self: ParentColumn) -> ParentColumn:
+    from py4j.java_gateway import JVMView
+
+    sc = get_active_spark_context()
+    jc = getattr(cast(JVMView, sc._jvm).functions, name)(self._jc)
+    return Column(jc)
+
+
+def _bin_func_op(
+    name: str,
+    self: ParentColumn,
+    other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+    reverse: bool = False,
+) -> ParentColumn:
+    from py4j.java_gateway import JVMView
+
+    sc = get_active_spark_context()
+    fn = getattr(cast(JVMView, sc._jvm).functions, name)
+    jc = other._jc if isinstance(other, ParentColumn) else _create_column_from_literal(other)
+    njc = fn(self._jc, jc) if not reverse else fn(jc, self._jc)
+    return Column(njc)
+
+
+def _bin_op(
+    name: str,
+    self: ParentColumn,
+    other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+) -> ParentColumn:
+    """Create a method for given binary operator"""
+    jc = other._jc if isinstance(other, ParentColumn) else other
+    njc = getattr(self._jc, name)(jc)
+    return Column(njc)
+
+
+def _reverse_op(
+    name: str,
+    self: ParentColumn,
+    other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+) -> ParentColumn:
+    """Create a method for binary operator (this object is on right side)"""
+    jother = _create_column_from_literal(other)
+    jc = getattr(jother, name)(self._jc)
+    return Column(jc)
+
+
+@with_origin_to_class
+class Column(ParentColumn):
+    def __new__(
+        cls,
+        jc: "JavaObject",
+    ) -> "Column":
+        self = object.__new__(cls)
+        self.__init__(jc)  # type: ignore[misc]
+        return self
+
+    def __init__(self, jc: "JavaObject") -> None:
+        self._jc = jc
+
+    # arithmetic operators
+    def __neg__(self) -> ParentColumn:
+        return _func_op("negate", self)
+
+    def __add__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("plus", self, other)
+
+    def __sub__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("minus", self, other)
+
+    def __mul__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("multiply", self, other)
+
+    def __div__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("divide", self, other)
+
+    def __truediv__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("divide", self, other)
+
+    def __mod__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("mod", self, other)
+
+    def __radd__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("plus", self, other)
+
+    def __rsub__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _reverse_op("minus", self, other)
+
+    def __rmul__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("multiply", self, other)
+
+    def __rdiv__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _reverse_op("divide", self, other)
+
+    def __rtruediv__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _reverse_op("divide", self, other)
+
+    def __rmod__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _reverse_op("mod", self, other)
+
+    def __pow__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_func_op("pow", self, other)
+
+    def __rpow__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_func_op("pow", self, other, reverse=True)
+
+    # logistic operators
+    def __eq__(  # type: ignore[override]
+        self,
+        other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+    ) -> ParentColumn:
+        return _bin_op("equalTo", self, other)
+
+    def __ne__(  # type: ignore[override]
+        self,
+        other: Any,
+    ) -> ParentColumn:
+        return _bin_op("notEqual", self, other)
+
+    def __lt__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("lt", self, other)
+
+    def __le__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("leq", self, other)
+
+    def __ge__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("geq", self, other)
+
+    def __gt__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("gt", self, other)
+
+    def eqNullSafe(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("eqNullSafe", self, other)
+
+    # `and`, `or`, `not` cannot be overloaded in Python,
+    # so use bitwise operators as boolean operators
+    def __and__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("and", self, other)
+
+    def __or__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("or", self, other)
+
+    def __invert__(self) -> ParentColumn:
+        return _func_op("not", self)
+
+    def __rand__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("and", self, other)
+
+    def __ror__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("or", self, other)
+
+    # container operators
+    def __contains__(self, item: Any) -> None:
+        raise PySparkValueError(
+            error_class="CANNOT_APPLY_IN_FOR_COLUMN",
+            message_parameters={},
+        )
+
+    # bitwise operators
+    def bitwiseOR(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("bitwiseOR", self, other)
+
+    def bitwiseAND(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("bitwiseAND", self, other)
+
+    def bitwiseXOR(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("bitwiseXOR", self, other)
+
+    def getItem(self, key: Any) -> ParentColumn:
+        if isinstance(key, Column):
+            warnings.warn(
+                "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
+                "be supported in the future release. Use `column[key]` or `column.key` syntax "
+                "instead.",
+                FutureWarning,
+            )
+        return self[key]
+
+    def getField(self, name: Any) -> ParentColumn:
+        if isinstance(name, Column):
+            warnings.warn(
+                "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
+                "be supported in the future release. Use `column[name]` or `column.name` syntax "
+                "instead.",
+                FutureWarning,
+            )
+        return self[name]
+
+    def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:
+        if not isinstance(fieldName, str):
+            raise PySparkTypeError(
+                error_class="NOT_STR",
+                message_parameters={"arg_name": "fieldName", "arg_type": type(fieldName).__name__},
+            )
+
+        if not isinstance(col, Column):
+            raise PySparkTypeError(
+                error_class="NOT_COLUMN",
+                message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
+            )
+
+        return Column(self._jc.withField(fieldName, col._jc))
+
+    def dropFields(self, *fieldNames: str) -> ParentColumn:
+        sc = get_active_spark_context()
+        jc = self._jc.dropFields(_to_seq(sc, fieldNames))
+        return Column(jc)
+
+    def __getattr__(self, item: Any) -> ParentColumn:
+        if item.startswith("__"):
+            raise PySparkAttributeError(
+                error_class="CANNOT_ACCESS_TO_DUNDER",
+                message_parameters={},
+            )
+        return self[item]
+
+    def __getitem__(self, k: Any) -> ParentColumn:
+        if isinstance(k, slice):
+            if k.step is not None:
+                raise PySparkValueError(
+                    error_class="SLICE_WITH_STEP",
+                    message_parameters={},
+                )
+            return self.substr(k.start, k.stop)
+        else:
+            return _bin_op("apply", self, k)
+
+    def __iter__(self) -> None:
+        raise PySparkTypeError(
+            error_class="NOT_ITERABLE", message_parameters={"objectName": "Column"}
+        )
+
+    # string methods
+    def contains(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("contains", self, other)
+
+    def startswith(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("startsWith", self, other)
+
+    def endswith(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("endsWith", self, other)
+
+    def like(self: ParentColumn, other: str) -> ParentColumn:
+        njc = getattr(self._jc, "like")(other)
+        return Column(njc)
+
+    def rlike(self: ParentColumn, other: str) -> ParentColumn:
+        njc = getattr(self._jc, "rlike")(other)
+        return Column(njc)
+
+    def ilike(self: ParentColumn, other: str) -> ParentColumn:
+        njc = getattr(self._jc, "ilike")(other)
+        return Column(njc)
+
+    def substr(
+        self, startPos: Union[int, ParentColumn], length: Union[int, ParentColumn]
+    ) -> ParentColumn:
+        if type(startPos) != type(length):
+            raise PySparkTypeError(
+                error_class="NOT_SAME_TYPE",
+                message_parameters={
+                    "arg_name1": "startPos",
+                    "arg_name2": "length",
+                    "arg_type1": type(startPos).__name__,
+                    "arg_type2": type(length).__name__,
+                },
+            )
+        if isinstance(startPos, int):
+            jc = self._jc.substr(startPos, length)
+        elif isinstance(startPos, Column):
+            jc = self._jc.substr(startPos._jc, cast(ParentColumn, length)._jc)
+        else:
+            raise PySparkTypeError(
+                error_class="NOT_COLUMN_OR_INT",
+                message_parameters={"arg_name": "startPos", "arg_type": type(startPos).__name__},
+            )
+        return Column(jc)
+
+    def isin(self, *cols: Any) -> ParentColumn:
+        if len(cols) == 1 and isinstance(cols[0], (list, set)):
+            cols = cast(Tuple, cols[0])
+        cols = cast(
+            Tuple,
+            [c._jc if isinstance(c, Column) else _create_column_from_literal(c) for c in cols],
+        )
+        sc = get_active_spark_context()
+        jc = getattr(self._jc, "isin")(_to_seq(sc, cols))
+        return Column(jc)
+
+    # order
+    def asc(self) -> ParentColumn:
+        return _unary_op("asc", self)
+
+    def asc_nulls_first(self) -> ParentColumn:
+        return _unary_op("asc_nulls_first", self)
+
+    def asc_nulls_last(self) -> ParentColumn:
+        return _unary_op("asc_nulls_last", self)
+
+    def desc(self) -> ParentColumn:
+        return _unary_op("desc", self)
+
+    def desc_nulls_first(self) -> ParentColumn:
+        return _unary_op("desc_nulls_first", self)
+
+    def desc_nulls_last(self) -> ParentColumn:
+        return _unary_op("desc_nulls_last", self)
+
+    def isNull(self) -> ParentColumn:
+        return _unary_op("isNull", self)
+
+    def isNotNull(self) -> ParentColumn:
+        return _unary_op("isNotNull", self)
+
+    def isNaN(self) -> ParentColumn:
+        return _unary_op("isNaN", self)
+
+    def alias(self, *alias: str, **kwargs: Any) -> ParentColumn:
+        metadata = kwargs.pop("metadata", None)
+        assert not kwargs, "Unexpected kwargs where passed: %s" % kwargs
+
+        sc = get_active_spark_context()
+        if len(alias) == 1:
+            if metadata:
+                assert sc._jvm is not None
+                jmeta = sc._jvm.org.apache.spark.sql.types.Metadata.fromJson(json.dumps(metadata))
+                return Column(getattr(self._jc, "as")(alias[0], jmeta))
+            else:
+                return Column(getattr(self._jc, "as")(alias[0]))
+        else:
+            if metadata is not None:
+                raise PySparkValueError(
+                    error_class="ONLY_ALLOWED_FOR_SINGLE_COLUMN",
+                    message_parameters={"arg_name": "metadata"},
+                )
+            return Column(getattr(self._jc, "as")(_to_seq(sc, list(alias))))
+
+    def name(self, *alias: str, **kwargs: Any) -> ParentColumn:
+        return self.alias(*alias, **kwargs)
+
+    def cast(self, dataType: Union[DataType, str]) -> ParentColumn:
+        if isinstance(dataType, str):
+            jc = self._jc.cast(dataType)
+        elif isinstance(dataType, DataType):
+            from pyspark.sql import SparkSession
+
+            spark = SparkSession._getActiveSessionOrCreate()
+            jdt = spark._jsparkSession.parseDataType(dataType.json())
+            jc = self._jc.cast(jdt)
+        else:
+            raise PySparkTypeError(
+                error_class="NOT_DATATYPE_OR_STR",
+                message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+            )
+        return Column(jc)
+
+    def try_cast(self, dataType: Union[DataType, str]) -> ParentColumn:
+        if isinstance(dataType, str):
+            jc = self._jc.try_cast(dataType)
+        elif isinstance(dataType, DataType):
+            from pyspark.sql import SparkSession
+
+            spark = SparkSession._getActiveSessionOrCreate()
+            jdt = spark._jsparkSession.parseDataType(dataType.json())
+            jc = self._jc.try_cast(jdt)
+        else:
+            raise PySparkTypeError(
+                error_class="NOT_DATATYPE_OR_STR",
+                message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+            )
+        return Column(jc)
+
+    def astype(self, dataType: Union[DataType, str]) -> ParentColumn:
+        return self.cast(dataType)
+
+    def between(
+        self,
+        lowerBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+        upperBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+    ) -> ParentColumn:
+        return (self >= lowerBound) & (self <= upperBound)
+
+    def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
+        if not isinstance(condition, Column):
+            raise PySparkTypeError(
+                error_class="NOT_COLUMN",
+                message_parameters={"arg_name": "condition", "arg_type": type(condition).__name__},
+            )
+        v = value._jc if isinstance(value, Column) else value
+        jc = self._jc.when(condition._jc, v)
+        return Column(jc)
+
+    def otherwise(self, value: Any) -> ParentColumn:
+        v = value._jc if isinstance(value, Column) else value
+        jc = self._jc.otherwise(v)
+        return Column(jc)
+
+    def over(self, window: "WindowSpec") -> ParentColumn:
+        from pyspark.sql.window import WindowSpec
+
+        if not isinstance(window, WindowSpec):
+            raise PySparkTypeError(
+                error_class="NOT_WINDOWSPEC",
+                message_parameters={"arg_name": "window", "arg_type": type(window).__name__},
+            )
+        jc = self._jc.over(window._jspec)
+        return Column(jc)
+
+    def __nonzero__(self) -> None:
+        raise PySparkValueError(
+            error_class="CANNOT_CONVERT_COLUMN_INTO_BOOL",
+            message_parameters={},
+        )
+
+    __bool__ = __nonzero__
+
+    def __repr__(self) -> str:
+        return "Column<'%s'>" % self._jc.toString()
+
+
+def _test() -> None:
+    import doctest
+    from pyspark.sql import SparkSession
+    import pyspark.sql.column
+
+    # It inherits docstrings but doctests cannot detect them so we run
+    # the parent classe's doctests here directly.
+    globs = pyspark.sql.column.__dict__.copy()
+    spark = (
+        SparkSession.builder.master("local[4]").appName("sql.classic.column tests").getOrCreate()
+    )
+    globs["spark"] = spark
+
+    (failure_count, test_count) = doctest.testmod(
+        pyspark.sql.column,
+        globs=globs,
+        optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
+    )
+    spark.stop()
+    if failure_count:
+        sys.exit(-1)
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py
index b8b07356bfe9..db9f22517dda 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -54,7 +54,8 @@ from pyspark.util import (
 from pyspark.serializers import BatchedSerializer, CPickleSerializer, UTF8Deserializer
 from pyspark.storagelevel import StorageLevel
 from pyspark.traceback_utils import SCCallSiteSync
-from pyspark.sql.column import Column, _to_seq, _to_list, _to_java_column
+from pyspark.sql.column import Column
+from pyspark.sql.classic.column import _to_seq, _to_list, _to_java_column
 from pyspark.sql.readwriter import DataFrameWriter, DataFrameWriterV2
 from pyspark.sql.streaming import DataStreamWriter
 from pyspark.sql.types import (
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 2e79e30285ba..4ea621b626bb 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -15,197 +15,28 @@
 # limitations under the License.
 #
 
+# mypy: disable-error-code="empty-body"
+
 import sys
-import json
-import warnings
 from typing import (
-    cast,
     overload,
     Any,
-    Callable,
-    Iterable,
-    List,
-    Optional,
-    Tuple,
     TYPE_CHECKING,
     Union,
 )
 
-from pyspark.errors import PySparkAttributeError, PySparkTypeError, PySparkValueError
-from pyspark.errors.utils import with_origin_to_class
+from pyspark.sql.utils import dispatch_col_method
 from pyspark.sql.types import DataType
-from pyspark.sql.utils import get_active_spark_context
+from pyspark.errors import PySparkValueError
 
 if TYPE_CHECKING:
     from py4j.java_gateway import JavaObject
-    from pyspark.core.context import SparkContext
-    from pyspark.sql._typing import ColumnOrName, LiteralType, DecimalLiteral, DateTimeLiteral
+    from pyspark.sql._typing import LiteralType, DecimalLiteral, DateTimeLiteral
     from pyspark.sql.window import WindowSpec
 
 __all__ = ["Column"]
 
 
-def _create_column_from_literal(literal: Union["LiteralType", "DecimalLiteral"]) -> "Column":
-    from py4j.java_gateway import JVMView
-
-    sc = get_active_spark_context()
-    return cast(JVMView, sc._jvm).functions.lit(literal)
-
-
-def _create_column_from_name(name: str) -> "Column":
-    from py4j.java_gateway import JVMView
-
-    sc = get_active_spark_context()
-    return cast(JVMView, sc._jvm).functions.col(name)
-
-
-def _to_java_column(col: "ColumnOrName") -> "JavaObject":
-    if isinstance(col, Column):
-        jcol = col._jc
-    elif isinstance(col, str):
-        jcol = _create_column_from_name(col)
-    else:
-        raise PySparkTypeError(
-            error_class="NOT_COLUMN_OR_STR",
-            message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
-        )
-    return jcol
-
-
-def _to_java_expr(col: "ColumnOrName") -> "JavaObject":
-    return _to_java_column(col).expr()
-
-
-@overload
-def _to_seq(sc: "SparkContext", cols: Iterable["JavaObject"]) -> "JavaObject":
-    ...
-
-
-@overload
-def _to_seq(
-    sc: "SparkContext",
-    cols: Iterable["ColumnOrName"],
-    converter: Optional[Callable[["ColumnOrName"], "JavaObject"]],
-) -> "JavaObject":
-    ...
-
-
-def _to_seq(
-    sc: "SparkContext",
-    cols: Union[Iterable["ColumnOrName"], Iterable["JavaObject"]],
-    converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
-) -> "JavaObject":
-    """
-    Convert a list of Columns (or names) into a JVM Seq of Column.
-
-    An optional `converter` could be used to convert items in `cols`
-    into JVM Column objects.
-    """
-    if converter:
-        cols = [converter(c) for c in cols]
-    assert sc._jvm is not None
-    return sc._jvm.PythonUtils.toSeq(cols)
-
-
-def _to_list(
-    sc: "SparkContext",
-    cols: List["ColumnOrName"],
-    converter: Optional[Callable[["ColumnOrName"], "JavaObject"]] = None,
-) -> "JavaObject":
-    """
-    Convert a list of Columns (or names) into a JVM (Scala) List of Columns.
-
-    An optional `converter` could be used to convert items in `cols`
-    into JVM Column objects.
-    """
-    if converter:
-        cols = [converter(c) for c in cols]
-    assert sc._jvm is not None
-    return sc._jvm.PythonUtils.toList(cols)
-
-
-def _unary_op(
-    name: str,
-    doc: str = "unary operator",
-) -> Callable[["Column"], "Column"]:
-    """Create a method for given unary operator"""
-
-    def _(self: "Column") -> "Column":
-        jc = getattr(self._jc, name)()
-        return Column(jc)
-
-    _.__doc__ = doc
-    return _
-
-
-def _func_op(name: str, doc: str = "") -> Callable[["Column"], "Column"]:
-    def _(self: "Column") -> "Column":
-        from py4j.java_gateway import JVMView
-
-        sc = get_active_spark_context()
-        jc = getattr(cast(JVMView, sc._jvm).functions, name)(self._jc)
-        return Column(jc)
-
-    _.__doc__ = doc
-    return _
-
-
-def _bin_func_op(
-    name: str,
-    reverse: bool = False,
-    doc: str = "binary function",
-) -> Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"]:
-    def _(self: "Column", other: Union["Column", "LiteralType", "DecimalLiteral"]) -> "Column":
-        from py4j.java_gateway import JVMView
-
-        sc = get_active_spark_context()
-        fn = getattr(cast(JVMView, sc._jvm).functions, name)
-        jc = other._jc if isinstance(other, Column) else _create_column_from_literal(other)
-        njc = fn(self._jc, jc) if not reverse else fn(jc, self._jc)
-        return Column(njc)
-
-    _.__doc__ = doc
-    return _
-
-
-def _bin_op(
-    name: str,
-    doc: str = "binary operator",
-) -> Callable[
-    ["Column", Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]], "Column"
-]:
-    """Create a method for given binary operator"""
-
-    def _(
-        self: "Column",
-        other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
-    ) -> "Column":
-        jc = other._jc if isinstance(other, Column) else other
-        njc = getattr(self._jc, name)(jc)
-        return Column(njc)
-
-    _.__doc__ = doc
-    _.__name__ = name
-    return _
-
-
-def _reverse_op(
-    name: str,
-    doc: str = "binary operator",
-) -> Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"]:
-    """Create a method for binary operator (this object is on right side)"""
-
-    def _(self: "Column", other: Union["LiteralType", "DecimalLiteral"]) -> "Column":
-        jother = _create_column_from_literal(other)
-        jc = getattr(jother, name)(self._jc)
-        return Column(jc)
-
-    _.__doc__ = doc
-    _.__name__ = name
-    return _
-
-
-@with_origin_to_class
 class Column:
 
     """
@@ -237,146 +68,251 @@ class Column:
     Column<...>
     """
 
+    # HACK ALERT!! this is to reduce the backward compatibility concern, and returns
+    # Spark Classic Column by default. This is NOT an API, and NOT supposed to
+    # be directly invoked. DO NOT use this constructor.
+    def __new__(
+        cls,
+        jc: "JavaObject",
+    ) -> "Column":
+        from pyspark.sql.classic.column import Column
+
+        return Column.__new__(Column, jc)
+
     def __init__(self, jc: "JavaObject") -> None:
         self._jc = jc
 
     # arithmetic operators
-    __neg__ = _func_op("negate")
-    __add__ = cast(
-        Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_op("plus"),
-    )
-    __sub__ = cast(
-        Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_op("minus"),
-    )
-    __mul__ = cast(
-        Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_op("multiply"),
-    )
-    __div__ = cast(
-        Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_op("divide"),
-    )
-    __truediv__ = cast(
-        Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_op("divide"),
-    )
-    __mod__ = cast(
-        Callable[["Column", Union["Column", "LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_op("mod"),
-    )
-    __radd__ = cast(
-        Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"], _bin_op("plus")
-    )
-    __rsub__ = _reverse_op("minus")
-    __rmul__ = cast(
-        Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"], _bin_op("multiply")
-    )
-    __rdiv__ = _reverse_op("divide")
-    __rtruediv__ = _reverse_op("divide")
-    __rmod__ = _reverse_op("mod")
-
-    __pow__ = _bin_func_op("pow")
-    __rpow__ = cast(
-        Callable[["Column", Union["LiteralType", "DecimalLiteral"]], "Column"],
-        _bin_func_op("pow", reverse=True),
-    )
+    @dispatch_col_method
+    def __neg__(self) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __add__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __sub__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __mul__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __div__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __truediv__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __mod__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __radd__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rsub__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rmul__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rdiv__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rtruediv__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rmod__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __pow__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rpow__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
 
     # logistic operators
+    @dispatch_col_method
     def __eq__(  # type: ignore[override]
         self,
         other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
     ) -> "Column":
         """binary function"""
-        return _bin_op("equalTo")(self, other)
+        ...
 
+    @dispatch_col_method
     def __ne__(  # type: ignore[override]
         self,
         other: Any,
     ) -> "Column":
         """binary function"""
-        return _bin_op("notEqual")(self, other)
+        ...
 
-    __lt__ = _bin_op("lt")
-    __le__ = _bin_op("leq")
-    __ge__ = _bin_op("geq")
-    __gt__ = _bin_op("gt")
+    @dispatch_col_method
+    def __lt__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
 
-    _eqNullSafe_doc = """
-    Equality test that is safe for null values.
+    @dispatch_col_method
+    def __le__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
 
-    .. versionadded:: 2.3.0
+    @dispatch_col_method
+    def __ge__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+    @dispatch_col_method
+    def __gt__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def eqNullSafe(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """
+        Equality test that is safe for null values.
 
-    Parameters
-    ----------
-    other
-        a value or :class:`Column`
+        .. versionadded:: 2.3.0
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df1 = spark.createDataFrame([
-    ...     Row(id=1, value='foo'),
-    ...     Row(id=2, value=None)
-    ... ])
-    >>> df1.select(
-    ...     df1['value'] == 'foo',
-    ...     df1['value'].eqNullSafe('foo'),
-    ...     df1['value'].eqNullSafe(None)
-    ... ).show()
-    +-------------+---------------+----------------+
-    |(value = foo)|(value <=> foo)|(value <=> NULL)|
-    +-------------+---------------+----------------+
-    |         true|           true|           false|
-    |         NULL|          false|            true|
-    +-------------+---------------+----------------+
-    >>> df2 = spark.createDataFrame([
-    ...     Row(value = 'bar'),
-    ...     Row(value = None)
-    ... ])
-    >>> df1.join(df2, df1["value"] == df2["value"]).count()
-    0
-    >>> df1.join(df2, df1["value"].eqNullSafe(df2["value"])).count()
-    1
-    >>> df2 = spark.createDataFrame([
-    ...     Row(id=1, value=float('NaN')),
-    ...     Row(id=2, value=42.0),
-    ...     Row(id=3, value=None)
-    ... ])
-    >>> df2.select(
-    ...     df2['value'].eqNullSafe(None),
-    ...     df2['value'].eqNullSafe(float('NaN')),
-    ...     df2['value'].eqNullSafe(42.0)
-    ... ).show()
-    +----------------+---------------+----------------+
-    |(value <=> NULL)|(value <=> NaN)|(value <=> 42.0)|
-    +----------------+---------------+----------------+
-    |           false|           true|           false|
-    |           false|          false|            true|
-    |            true|          false|           false|
-    +----------------+---------------+----------------+
-
-    Notes
-    -----
-    Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the
-    `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_
-    for details.
-    """
-    eqNullSafe = _bin_op("eqNullSafe", _eqNullSafe_doc)
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        other
+            a value or :class:`Column`
+
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df1 = spark.createDataFrame([
+        ...     Row(id=1, value='foo'),
+        ...     Row(id=2, value=None)
+        ... ])
+        >>> df1.select(
+        ...     df1['value'] == 'foo',
+        ...     df1['value'].eqNullSafe('foo'),
+        ...     df1['value'].eqNullSafe(None)
+        ... ).show()
+        +-------------+---------------+----------------+
+        |(value = foo)|(value <=> foo)|(value <=> NULL)|
+        +-------------+---------------+----------------+
+        |         true|           true|           false|
+        |         NULL|          false|            true|
+        +-------------+---------------+----------------+
+        >>> df2 = spark.createDataFrame([
+        ...     Row(value = 'bar'),
+        ...     Row(value = None)
+        ... ])
+        >>> df1.join(df2, df1["value"] == df2["value"]).count()
+        0
+        >>> df1.join(df2, df1["value"].eqNullSafe(df2["value"])).count()
+        1
+        >>> df2 = spark.createDataFrame([
+        ...     Row(id=1, value=float('NaN')),
+        ...     Row(id=2, value=42.0),
+        ...     Row(id=3, value=None)
+        ... ])
+        >>> df2.select(
+        ...     df2['value'].eqNullSafe(None),
+        ...     df2['value'].eqNullSafe(float('NaN')),
+        ...     df2['value'].eqNullSafe(42.0)
+        ... ).show()
+        +----------------+---------------+----------------+
+        |(value <=> NULL)|(value <=> NaN)|(value <=> 42.0)|
+        +----------------+---------------+----------------+
+        |           false|           true|           false|
+        |           false|          false|            true|
+        |            true|          false|           false|
+        +----------------+---------------+----------------+
+
+        Notes
+        -----
+        Unlike Pandas, PySpark doesn't consider NaN values to be NULL. See the
+        `NaN Semantics <https://spark.apache.org/docs/latest/sql-ref-datatypes.html#nan-semantics>`_
+        for details.
+        """
+        ...
 
     # `and`, `or`, `not` cannot be overloaded in Python,
     # so use bitwise operators as boolean operators
-    __and__ = _bin_op("and")
-    __or__ = _bin_op("or")
-    __invert__ = _func_op("not")
-    __rand__ = _bin_op("and")
-    __ror__ = _bin_op("or")
+    @dispatch_col_method
+    def __and__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __or__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __invert__(self) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __rand__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
+
+    @dispatch_col_method
+    def __ror__(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        ...
 
     # container operators
+    @dispatch_col_method
     def __contains__(self, item: Any) -> None:
         raise PySparkValueError(
             error_class="CANNOT_APPLY_IN_FOR_COLUMN",
@@ -384,68 +320,82 @@ class Column:
         )
 
     # bitwise operators
-    _bitwiseOR_doc = """
-    Compute bitwise OR of this expression with another expression.
+    @dispatch_col_method
+    def bitwiseOR(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """ "
+        Compute bitwise OR of this expression with another expression.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    Parameters
-    ----------
-    other
-        a value or :class:`Column` to calculate bitwise or(|) with
-        this :class:`Column`.
+        Parameters
+        ----------
+        other
+            a value or :class:`Column` to calculate bitwise or(|) with
+            this :class:`Column`.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(a=170, b=75)])
-    >>> df.select(df.a.bitwiseOR(df.b)).collect()
-    [Row((a | b)=235)]
-    """
-    _bitwiseAND_doc = """
-    Compute bitwise AND of this expression with another expression.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([Row(a=170, b=75)])
+        >>> df.select(df.a.bitwiseOR(df.b)).collect()
+        [Row((a | b)=235)]
+        """
+        ...
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+    @dispatch_col_method
+    def bitwiseAND(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """
+        Compute bitwise AND of this expression with another expression.
 
-    Parameters
-    ----------
-    other
-        a value or :class:`Column` to calculate bitwise and(&) with
-        this :class:`Column`.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(a=170, b=75)])
-    >>> df.select(df.a.bitwiseAND(df.b)).collect()
-    [Row((a & b)=10)]
-    """
-    _bitwiseXOR_doc = """
-    Compute bitwise XOR of this expression with another expression.
+        Parameters
+        ----------
+        other
+            a value or :class:`Column` to calculate bitwise and(&) with
+            this :class:`Column`.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([Row(a=170, b=75)])
+        >>> df.select(df.a.bitwiseAND(df.b)).collect()
+        [Row((a & b)=10)]
+        """
+        ...
 
-    Parameters
-    ----------
-    other
-        a value or :class:`Column` to calculate bitwise xor(^) with
-        this :class:`Column`.
+    @dispatch_col_method
+    def bitwiseXOR(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """
+        Compute bitwise XOR of this expression with another expression.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(a=170, b=75)])
-    >>> df.select(df.a.bitwiseXOR(df.b)).collect()
-    [Row((a ^ b)=225)]
-    """
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    bitwiseOR = _bin_op("bitwiseOR", _bitwiseOR_doc)
-    bitwiseAND = _bin_op("bitwiseAND", _bitwiseAND_doc)
-    bitwiseXOR = _bin_op("bitwiseXOR", _bitwiseXOR_doc)
+        Parameters
+        ----------
+        other
+            a value or :class:`Column` to calculate bitwise xor(^) with
+            this :class:`Column`.
+
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([Row(a=170, b=75)])
+        >>> df.select(df.a.bitwiseXOR(df.b)).collect()
+        [Row((a ^ b)=225)]
+        """
+        ...
 
+    @dispatch_col_method
     def getItem(self, key: Any) -> "Column":
         """
         An expression that gets an item at position ``ordinal`` out of a list,
@@ -480,15 +430,9 @@ class Column:
         |   1| value|
         +----+------+
         """
-        if isinstance(key, Column):
-            warnings.warn(
-                "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
-                "be supported in the future release. Use `column[key]` or `column.key` syntax "
-                "instead.",
-                FutureWarning,
-            )
-        return self[key]
+        ...
 
+    @dispatch_col_method
     def getField(self, name: Any) -> "Column":
         """
         An expression that gets a field by name in a :class:`StructType`.
@@ -528,15 +472,9 @@ class Column:
         |  1|
         +---+
         """
-        if isinstance(name, Column):
-            warnings.warn(
-                "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
-                "be supported in the future release. Use `column[name]` or `column.name` syntax "
-                "instead.",
-                FutureWarning,
-            )
-        return self[name]
+        ...
 
+    @dispatch_col_method
     def withField(self, fieldName: str, col: "Column") -> "Column":
         """
         An expression that adds/replaces a field in :class:`StructType` by name.
@@ -578,20 +516,9 @@ class Column:
         |  4|
         +---+
         """
-        if not isinstance(fieldName, str):
-            raise PySparkTypeError(
-                error_class="NOT_STR",
-                message_parameters={"arg_name": "fieldName", "arg_type": type(fieldName).__name__},
-            )
-
-        if not isinstance(col, Column):
-            raise PySparkTypeError(
-                error_class="NOT_COLUMN",
-                message_parameters={"arg_name": "col", "arg_type": type(col).__name__},
-            )
-
-        return Column(self._jc.withField(fieldName, col._jc))
+        ...
 
+    @dispatch_col_method
     def dropFields(self, *fieldNames: str) -> "Column":
         """
         An expression that drops fields in :class:`StructType` by name.
@@ -656,10 +583,9 @@ class Column:
         +--------------+
 
         """
-        sc = get_active_spark_context()
-        jc = self._jc.dropFields(_to_seq(sc, fieldNames))
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def __getattr__(self, item: Any) -> "Column":
         """
         An expression that gets an item at position ``ordinal`` out of a list,
@@ -690,13 +616,9 @@ class Column:
         | value|
         +------+
         """
-        if item.startswith("__"):
-            raise PySparkAttributeError(
-                error_class="CANNOT_ACCESS_TO_DUNDER",
-                message_parameters={},
-            )
-        return self[item]
+        ...
 
+    @dispatch_col_method
     def __getitem__(self, k: Any) -> "Column":
         """
         An expression that gets an item at position ``ordinal`` out of a list,
@@ -728,85 +650,90 @@ class Column:
         |            abc| value|
         +---------------+------+
         """
-        if isinstance(k, slice):
-            if k.step is not None:
-                raise PySparkValueError(
-                    error_class="SLICE_WITH_STEP",
-                    message_parameters={},
-                )
-            return self.substr(k.start, k.stop)
-        else:
-            return _bin_op("apply")(self, k)
+        ...
 
+    @dispatch_col_method
     def __iter__(self) -> None:
-        raise PySparkTypeError(
-            error_class="NOT_ITERABLE", message_parameters={"objectName": "Column"}
-        )
+        ...
 
     # string methods
-    _contains_doc = """
-    Contains the other element. Returns a boolean :class:`Column` based on a string match.
+    @dispatch_col_method
+    def contains(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """
+        Contains the other element. Returns a boolean :class:`Column` based on a string match.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    Parameters
-    ----------
-    other
-        string in line. A value as a literal or a :class:`Column`.
+        Parameters
+        ----------
+        other
+            string in line. A value as a literal or a :class:`Column`.
 
-    Examples
-    --------
-    >>> df = spark.createDataFrame(
-    ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-    >>> df.filter(df.name.contains('o')).collect()
-    [Row(age=5, name='Bob')]
-    """
-    _startswith_doc = """
-    String starts with. Returns a boolean :class:`Column` based on a string match.
+        Examples
+        --------
+        >>> df = spark.createDataFrame(
+        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
+        >>> df.filter(df.name.contains('o')).collect()
+        [Row(age=5, name='Bob')]
+        """
+        ...
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+    @dispatch_col_method
+    def startswith(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """
+        String starts with. Returns a boolean :class:`Column` based on a string match.
 
-    Parameters
-    ----------
-    other : :class:`Column` or str
-        string at start of line (do not use a regex `^`)
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    Examples
-    --------
-    >>> df = spark.createDataFrame(
-    ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-    >>> df.filter(df.name.startswith('Al')).collect()
-    [Row(age=2, name='Alice')]
-    >>> df.filter(df.name.startswith('^Al')).collect()
-    []
-    """
-    _endswith_doc = """
-    String ends with. Returns a boolean :class:`Column` based on a string match.
+        Parameters
+        ----------
+        other : :class:`Column` or str
+            string at start of line (do not use a regex `^`)
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        Examples
+        --------
+        >>> df = spark.createDataFrame(
+        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
+        >>> df.filter(df.name.startswith('Al')).collect()
+        [Row(age=2, name='Alice')]
+        >>> df.filter(df.name.startswith('^Al')).collect()
+        []
+        """
+        ...
 
-    Parameters
-    ----------
-    other : :class:`Column` or str
-        string at end of line (do not use a regex `$`)
+    @dispatch_col_method
+    def endswith(
+        self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> "Column":
+        """
+        String ends with. Returns a boolean :class:`Column` based on a string match.
 
-    Examples
-    --------
-    >>> df = spark.createDataFrame(
-    ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
-    >>> df.filter(df.name.endswith('ice')).collect()
-    [Row(age=2, name='Alice')]
-    >>> df.filter(df.name.endswith('ice$')).collect()
-    []
-    """
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    contains = _bin_op("contains", _contains_doc)
-    startswith = _bin_op("startsWith", _startswith_doc)
-    endswith = _bin_op("endsWith", _endswith_doc)
+        Parameters
+        ----------
+        other : :class:`Column` or str
+            string at end of line (do not use a regex `$`)
 
+        Examples
+        --------
+        >>> df = spark.createDataFrame(
+        ...      [(2, "Alice"), (5, "Bob")], ["age", "name"])
+        >>> df.filter(df.name.endswith('ice')).collect()
+        [Row(age=2, name='Alice')]
+        >>> df.filter(df.name.endswith('ice$')).collect()
+        []
+        """
+        ...
+
+    @dispatch_col_method
     def like(self: "Column", other: str) -> "Column":
         """
         SQL like expression. Returns a boolean :class:`Column` based on a SQL LIKE match.
@@ -836,9 +763,9 @@ class Column:
         >>> df.filter(df.name.like('Al%')).collect()
         [Row(age=2, name='Alice')]
         """
-        njc = getattr(self._jc, "like")(other)
-        return Column(njc)
+        ...
 
+    @dispatch_col_method
     def rlike(self: "Column", other: str) -> "Column":
         """
         SQL RLIKE expression (LIKE with Regex). Returns a boolean :class:`Column` based on a regex
@@ -865,9 +792,9 @@ class Column:
         >>> df.filter(df.name.rlike('ice$')).collect()
         [Row(age=2, name='Alice')]
         """
-        njc = getattr(self._jc, "rlike")(other)
-        return Column(njc)
+        ...
 
+    @dispatch_col_method
     def ilike(self: "Column", other: str) -> "Column":
         """
         SQL ILIKE expression (case insensitive LIKE). Returns a boolean :class:`Column`
@@ -900,8 +827,7 @@ class Column:
         >>> df.filter(df.name.ilike('%Ice')).collect()
         [Row(age=2, name='Alice')]
         """
-        njc = getattr(self._jc, "ilike")(other)
-        return Column(njc)
+        ...
 
     @overload
     def substr(self, startPos: int, length: int) -> "Column":
@@ -911,6 +837,7 @@ class Column:
     def substr(self, startPos: "Column", length: "Column") -> "Column":
         ...
 
+    @dispatch_col_method
     def substr(self, startPos: Union[int, "Column"], length: Union[int, "Column"]) -> "Column":
         """
         Return a :class:`Column` which is a substring of the column.
@@ -949,27 +876,9 @@ class Column:
         >>> df.select(df.name.substr(df.sidx, df.eidx).alias("col")).collect()
         [Row(col='ice'), Row(col='ob')]
         """
-        if type(startPos) != type(length):
-            raise PySparkTypeError(
-                error_class="NOT_SAME_TYPE",
-                message_parameters={
-                    "arg_name1": "startPos",
-                    "arg_name2": "length",
-                    "arg_type1": type(startPos).__name__,
-                    "arg_type2": type(length).__name__,
-                },
-            )
-        if isinstance(startPos, int):
-            jc = self._jc.substr(startPos, length)
-        elif isinstance(startPos, Column):
-            jc = self._jc.substr(startPos._jc, cast("Column", length)._jc)
-        else:
-            raise PySparkTypeError(
-                error_class="NOT_COLUMN_OR_INT",
-                message_parameters={"arg_name": "startPos", "arg_type": type(startPos).__name__},
-            )
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def isin(self, *cols: Any) -> "Column":
         """
         A boolean expression that is evaluated to true if the value of this
@@ -1023,164 +932,184 @@ class Column:
         |  8|Mike|
         +---+----+
         """
-        if len(cols) == 1 and isinstance(cols[0], (list, set)):
-            cols = cast(Tuple, cols[0])
-        cols = cast(
-            Tuple,
-            [c._jc if isinstance(c, Column) else _create_column_from_literal(c) for c in cols],
-        )
-        sc = get_active_spark_context()
-        jc = getattr(self._jc, "isin")(_to_seq(sc, cols))
-        return Column(jc)
+        ...
 
     # order
-    _asc_doc = """
-    Returns a sort expression based on the ascending order of the column.
+    @dispatch_col_method
+    def asc(self) -> "Column":
+        """
+        Returns a sort expression based on the ascending order of the column.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
-    >>> df.select(df.name).orderBy(df.name.asc()).collect()
-    [Row(name='Alice'), Row(name='Tom')]
-    """
-    _asc_nulls_first_doc = """
-    Returns a sort expression based on ascending order of the column, and null values
-    return before non-null values.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
+        >>> df.select(df.name).orderBy(df.name.asc()).collect()
+        [Row(name='Alice'), Row(name='Tom')]
+        """
+        ...
 
-    .. versionadded:: 2.4.0
+    @dispatch_col_method
+    def asc_nulls_first(self) -> "Column":
+        """
+        Returns a sort expression based on ascending order of the column, and null values
+        return before non-null values.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        .. versionadded:: 2.4.0
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
-    >>> df.select(df.name).orderBy(df.name.asc_nulls_first()).collect()
-    [Row(name=None), Row(name='Alice'), Row(name='Tom')]
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    """
-    _asc_nulls_last_doc = """
-    Returns a sort expression based on ascending order of the column, and null values
-    appear after non-null values.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame(
+        ...     [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+        >>> df.select(df.name).orderBy(df.name.asc_nulls_first()).collect()
+        [Row(name=None), Row(name='Alice'), Row(name='Tom')]
 
-    .. versionadded:: 2.4.0
+        """
+        ...
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+    @dispatch_col_method
+    def asc_nulls_last(self) -> "Column":
+        """
+        Returns a sort expression based on ascending order of the column, and null values
+        appear after non-null values.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
-    >>> df.select(df.name).orderBy(df.name.asc_nulls_last()).collect()
-    [Row(name='Alice'), Row(name='Tom'), Row(name=None)]
+        .. versionadded:: 2.4.0
 
-    """
-    _desc_doc = """
-    Returns a sort expression based on the descending order of the column.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame(
+        ...     [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+        >>> df.select(df.name).orderBy(df.name.asc_nulls_last()).collect()
+        [Row(name='Alice'), Row(name='Tom'), Row(name=None)]
 
-    .. versionadded:: 2.4.0
+        """
+        ...
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+    @dispatch_col_method
+    def desc(self) -> "Column":
+        """
+        Returns a sort expression based on the descending order of the column.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
-    >>> df.select(df.name).orderBy(df.name.desc()).collect()
-    [Row(name='Tom'), Row(name='Alice')]
-    """
-    _desc_nulls_first_doc = """
-    Returns a sort expression based on the descending order of the column, and null values
-    appear before non-null values.
+        .. versionadded:: 2.4.0
 
-    .. versionadded:: 2.4.0
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([('Tom', 80), ('Alice', None)], ["name", "height"])
+        >>> df.select(df.name).orderBy(df.name.desc()).collect()
+        [Row(name='Tom'), Row(name='Alice')]
+        """
+        ...
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
-    >>> df.select(df.name).orderBy(df.name.desc_nulls_first()).collect()
-    [Row(name=None), Row(name='Tom'), Row(name='Alice')]
+    @dispatch_col_method
+    def desc_nulls_first(self) -> "Column":
+        """
+        Returns a sort expression based on the descending order of the column, and null values
+        appear before non-null values.
 
-    """
-    _desc_nulls_last_doc = """
-    Returns a sort expression based on the descending order of the column, and null values
-    appear after non-null values.
+        .. versionadded:: 2.4.0
 
-    .. versionadded:: 2.4.0
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame(
+        ...     [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+        >>> df.select(df.name).orderBy(df.name.desc_nulls_first()).collect()
+        [Row(name=None), Row(name='Tom'), Row(name='Alice')]
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
-    >>> df.select(df.name).orderBy(df.name.desc_nulls_last()).collect()
-    [Row(name='Tom'), Row(name='Alice'), Row(name=None)]
-    """
+        """
+        ...
 
-    asc = _unary_op("asc", _asc_doc)
-    asc_nulls_first = _unary_op("asc_nulls_first", _asc_nulls_first_doc)
-    asc_nulls_last = _unary_op("asc_nulls_last", _asc_nulls_last_doc)
-    desc = _unary_op("desc", _desc_doc)
-    desc_nulls_first = _unary_op("desc_nulls_first", _desc_nulls_first_doc)
-    desc_nulls_last = _unary_op("desc_nulls_last", _desc_nulls_last_doc)
+    @dispatch_col_method
+    def desc_nulls_last(self) -> "Column":
+        """
+        Returns a sort expression based on the descending order of the column, and null values
+        appear after non-null values.
 
-    _isNull_doc = """
-    True if the current expression is null.
+        .. versionadded:: 2.4.0
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
-    >>> df.filter(df.height.isNull()).collect()
-    [Row(name='Alice', height=None)]
-    """
-    _isNotNull_doc = """
-    True if the current expression is NOT null.
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame(
+        ...     [('Tom', 80), (None, 60), ('Alice', None)], ["name", "height"])
+        >>> df.select(df.name).orderBy(df.name.desc_nulls_last()).collect()
+        [Row(name='Tom'), Row(name='Alice'), Row(name=None)]
+        """
+        ...
 
-    .. versionchanged:: 3.4.0
-        Supports Spark Connect.
+    @dispatch_col_method
+    def isNull(self) -> "Column":
+        """
+        True if the current expression is null.
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
-    >>> df.filter(df.height.isNotNull()).collect()
-    [Row(name='Tom', height=80)]
-    """
-    _isNaN_doc = """
-    True if the current expression is NaN.
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
-    .. versionadded:: 4.0.0
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
+        >>> df.filter(df.height.isNull()).collect()
+        [Row(name='Alice', height=None)]
+        """
+        ...
 
-    Examples
-    --------
-    >>> from pyspark.sql import Row
-    >>> df = spark.createDataFrame(
-    ...     [Row(name='Tom', height=80.0), Row(name='Alice', height=float('nan'))])
-    >>> df.filter(df.height.isNaN()).collect()
-    [Row(name='Alice', height=nan)]
-    """
+    @dispatch_col_method
+    def isNotNull(self) -> "Column":
+        """
+        True if the current expression is NOT null.
 
-    isNull = _unary_op("isNull", _isNull_doc)
-    isNotNull = _unary_op("isNotNull", _isNotNull_doc)
-    isNaN = _unary_op("isNaN", _isNaN_doc)
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
 
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame([Row(name='Tom', height=80), Row(name='Alice', height=None)])
+        >>> df.filter(df.height.isNotNull()).collect()
+        [Row(name='Tom', height=80)]
+        """
+        ...
+
+    @dispatch_col_method
+    def isNaN(self) -> "Column":
+        """
+        True if the current expression is NaN.
+
+        .. versionadded:: 4.0.0
+
+        Examples
+        --------
+        >>> from pyspark.sql import Row
+        >>> df = spark.createDataFrame(
+        ...     [Row(name='Tom', height=80.0), Row(name='Alice', height=float('nan'))])
+        >>> df.filter(df.height.isNaN()).collect()
+        [Row(name='Alice', height=nan)]
+        """
+        ...
+
+    @dispatch_col_method
     def alias(self, *alias: str, **kwargs: Any) -> "Column":
         """
         Returns this column aliased with a new name or names (in the case of expressions that
@@ -1220,34 +1149,18 @@ class Column:
         >>> df.select(df.age.alias("age3", metadata={'max': 99})).schema['age3'].metadata['max']
         99
         """
+        ...
 
-        metadata = kwargs.pop("metadata", None)
-        assert not kwargs, "Unexpected kwargs where passed: %s" % kwargs
-
-        sc = get_active_spark_context()
-        if len(alias) == 1:
-            if metadata:
-                assert sc._jvm is not None
-                jmeta = sc._jvm.org.apache.spark.sql.types.Metadata.fromJson(json.dumps(metadata))
-                return Column(getattr(self._jc, "as")(alias[0], jmeta))
-            else:
-                return Column(getattr(self._jc, "as")(alias[0]))
-        else:
-            if metadata is not None:
-                raise PySparkValueError(
-                    error_class="ONLY_ALLOWED_FOR_SINGLE_COLUMN",
-                    message_parameters={"arg_name": "metadata"},
-                )
-            return Column(getattr(self._jc, "as")(_to_seq(sc, list(alias))))
-
+    @dispatch_col_method
     def name(self, *alias: str, **kwargs: Any) -> "Column":
         """
         :func:`name` is an alias for :func:`alias`.
 
         .. versionadded:: 2.0.0
         """
-        return self.alias(*alias, **kwargs)
+        ...
 
+    @dispatch_col_method
     def cast(self, dataType: Union[DataType, str]) -> "Column":
         """
         Casts the column into type ``dataType``.
@@ -1278,21 +1191,9 @@ class Column:
         >>> df.select(df.age.cast(StringType()).alias('ages')).collect()
         [Row(ages='2'), Row(ages='5')]
         """
-        if isinstance(dataType, str):
-            jc = self._jc.cast(dataType)
-        elif isinstance(dataType, DataType):
-            from pyspark.sql import SparkSession
-
-            spark = SparkSession._getActiveSessionOrCreate()
-            jdt = spark._jsparkSession.parseDataType(dataType.json())
-            jc = self._jc.cast(jdt)
-        else:
-            raise PySparkTypeError(
-                error_class="NOT_DATATYPE_OR_STR",
-                message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
-            )
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def try_cast(self, dataType: Union[DataType, str]) -> "Column":
         """
         This is a special version of `cast` that performs the same operation, but returns a NULL
@@ -1340,29 +1241,18 @@ class Column:
         | NULL|
         +-----+
         """
-        if isinstance(dataType, str):
-            jc = self._jc.try_cast(dataType)
-        elif isinstance(dataType, DataType):
-            from pyspark.sql import SparkSession
-
-            spark = SparkSession._getActiveSessionOrCreate()
-            jdt = spark._jsparkSession.parseDataType(dataType.json())
-            jc = self._jc.try_cast(jdt)
-        else:
-            raise PySparkTypeError(
-                error_class="NOT_DATATYPE_OR_STR",
-                message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
-            )
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def astype(self, dataType: Union[DataType, str]) -> "Column":
         """
         :func:`astype` is an alias for :func:`cast`.
 
         .. versionadded:: 1.4.0
         """
-        return self.cast(dataType)
+        ...
 
+    @dispatch_col_method
     def between(
         self,
         lowerBound: Union["Column", "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
@@ -1470,8 +1360,9 @@ class Column:
         |  Bob|                                                              true|
         +-----+------------------------------------------------------------------+
         """
-        return (self >= lowerBound) & (self <= upperBound)
+        ...
 
+    @dispatch_col_method
     def when(self, condition: "Column", value: Any) -> "Column":
         """
         Evaluates a list of conditions and returns one of multiple possible result expressions.
@@ -1545,15 +1436,9 @@ class Column:
         --------
         pyspark.sql.functions.when
         """
-        if not isinstance(condition, Column):
-            raise PySparkTypeError(
-                error_class="NOT_COLUMN",
-                message_parameters={"arg_name": "condition", "arg_type": type(condition).__name__},
-            )
-        v = value._jc if isinstance(value, Column) else value
-        jc = self._jc.when(condition._jc, v)
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def otherwise(self, value: Any) -> "Column":
         """
         Evaluates a list of conditions and returns one of multiple possible result expressions.
@@ -1591,10 +1476,9 @@ class Column:
         --------
         pyspark.sql.functions.when
         """
-        v = value._jc if isinstance(value, Column) else value
-        jc = self._jc.otherwise(v)
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def over(self, window: "WindowSpec") -> "Column":
         """
         Define a windowing column.
@@ -1635,26 +1519,19 @@ class Column:
         |  2|Alice|   1|  2|
         +---+-----+----+---+
         """
-        from pyspark.sql.window import WindowSpec
-
-        if not isinstance(window, WindowSpec):
-            raise PySparkTypeError(
-                error_class="NOT_WINDOWSPEC",
-                message_parameters={"arg_name": "window", "arg_type": type(window).__name__},
-            )
-        jc = self._jc.over(window._jspec)
-        return Column(jc)
+        ...
 
+    @dispatch_col_method
     def __nonzero__(self) -> None:
-        raise PySparkValueError(
-            error_class="CANNOT_CONVERT_COLUMN_INTO_BOOL",
-            message_parameters={},
-        )
+        ...
 
-    __bool__ = __nonzero__
+    @dispatch_col_method
+    def __bool__(self) -> None:
+        ...
 
+    @dispatch_col_method
     def __repr__(self) -> str:
-        return "Column<'%s'>" % self._jc.toString()
+        ...
 
 
 def _test() -> None:
diff --git a/python/pyspark/sql/connect/_typing.py b/python/pyspark/sql/connect/_typing.py
index 1b8516427dbd..efb3e0e8eb50 100644
--- a/python/pyspark/sql/connect/_typing.py
+++ b/python/pyspark/sql/connect/_typing.py
@@ -22,7 +22,7 @@ import decimal
 import pyarrow
 from pandas.core.frame import DataFrame as PandasDataFrame
 
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.types import DataType
 from pyspark.sql.streaming.state import GroupState
 
diff --git a/python/pyspark/sql/connect/avro/functions.py b/python/pyspark/sql/connect/avro/functions.py
index f153b17acf58..7e46bc8d33be 100644
--- a/python/pyspark/sql/connect/avro/functions.py
+++ b/python/pyspark/sql/connect/avro/functions.py
@@ -26,8 +26,7 @@ check_dependencies(__name__)
 from typing import Dict, Optional, TYPE_CHECKING
 
 from pyspark.sql.avro import functions as PyAvroFunctions
-
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col, _options_to_col, lit
 
 if TYPE_CHECKING:
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index 4436b36907a9..146c517d1b1a 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -24,16 +24,14 @@ import warnings
 
 from typing import (
     TYPE_CHECKING,
-    Callable,
     Any,
     Union,
-    overload,
     Optional,
 )
 
+from pyspark.sql.column import Column as ParentColumn
 from pyspark.errors import PySparkTypeError, PySparkAttributeError, PySparkValueError
 from pyspark.sql.types import DataType
-from pyspark.sql.column import Column as PySparkColumn
 
 import pyspark.sql.connect.proto as proto
 from pyspark.sql.connect.expressions import (
@@ -60,53 +58,52 @@ if TYPE_CHECKING:
     from pyspark.sql.connect.window import WindowSpec
 
 
-def _func_op(name: str, doc: Optional[str] = "") -> Callable[["Column"], "Column"]:
-    def wrapped(self: "Column") -> "Column":
-        return Column(UnresolvedFunction(name, [self._expr]))
-
-    wrapped.__doc__ = doc
-    return wrapped
+def _func_op(name: str, self: ParentColumn) -> ParentColumn:
+    return Column(UnresolvedFunction(name, [self._expr]))  # type: ignore[list-item]
 
 
 def _bin_op(
-    name: str, doc: Optional[str] = "binary function", reverse: bool = False
-) -> Callable[["Column", Any], "Column"]:
-    def wrapped(self: "Column", other: Any) -> "Column":
-        if other is None or isinstance(
-            other,
-            (
-                bool,
-                float,
-                int,
-                str,
-                datetime.datetime,
-                datetime.date,
-                decimal.Decimal,
-                datetime.timedelta,
-            ),
-        ):
-            other_expr = LiteralExpression._from_value(other)
-        else:
-            other_expr = other._expr
-
-        if not reverse:
-            return Column(UnresolvedFunction(name, [self._expr, other_expr]))
-        else:
-            return Column(UnresolvedFunction(name, [other_expr, self._expr]))
-
-    wrapped.__doc__ = doc
-    return wrapped
-
-
-def _unary_op(name: str, doc: Optional[str] = "unary function") -> Callable[["Column"], "Column"]:
-    def wrapped(self: "Column") -> "Column":
-        return Column(UnresolvedFunction(name, [self._expr]))
-
-    wrapped.__doc__ = doc
-    return wrapped
-
+    name: str,
+    self: ParentColumn,
+    other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"],
+    reverse: bool = False,
+) -> ParentColumn:
+    if other is None or isinstance(
+        other,
+        (
+            bool,
+            float,
+            int,
+            str,
+            datetime.datetime,
+            datetime.date,
+            decimal.Decimal,
+            datetime.timedelta,
+        ),
+    ):
+        other_expr = LiteralExpression._from_value(other)
+    else:
+        other_expr = other._expr  # type: ignore[assignment]
+
+    if not reverse:
+        return Column(UnresolvedFunction(name, [self._expr, other_expr]))  # type: ignore[list-item]
+    else:
+        return Column(UnresolvedFunction(name, [other_expr, self._expr]))  # type: ignore[list-item]
+
+
+def _unary_op(name: str, self: ParentColumn) -> ParentColumn:
+    return Column(UnresolvedFunction(name, [self._expr]))  # type: ignore[list-item]
+
+
+class Column(ParentColumn):
+    def __new__(
+        cls,
+        expr: "Expression",
+    ) -> "Column":
+        self = object.__new__(cls)
+        self.__init__(expr)  # type: ignore[misc]
+        return self
 
-class Column:
     def __init__(self, expr: "Expression") -> None:
         if not isinstance(expr, Expression):
             raise PySparkTypeError(
@@ -115,36 +112,128 @@ class Column:
             )
         self._expr = expr
 
-    __gt__ = _bin_op(">")
-    __lt__ = _bin_op("<")
-    __add__ = _bin_op("+")
-    __sub__ = _bin_op("-")
-    __mul__ = _bin_op("*")
-    __div__ = _bin_op("/")
-    __truediv__ = _bin_op("/")
-    __mod__ = _bin_op("%")
-    __radd__ = _bin_op("+", reverse=True)
-    __rsub__ = _bin_op("-", reverse=True)
-    __rmul__ = _bin_op("*", reverse=True)
-    __rdiv__ = _bin_op("/", reverse=True)
-    __rtruediv__ = _bin_op("/", reverse=True)
-    __rmod__ = _bin_op("%", reverse=True)
-    __pow__ = _bin_op("power")
-    __rpow__ = _bin_op("power", reverse=True)
-    __ge__ = _bin_op(">=")
-    __le__ = _bin_op("<=")
-
-    eqNullSafe = _bin_op("<=>", PySparkColumn.eqNullSafe.__doc__)
-
-    __neg__ = _func_op("negative")
+    def __gt__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op(">", self, other)
+
+    def __lt__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("<", self, other)
+
+    def __add__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("+", self, other)
+
+    def __sub__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("-", self, other)
+
+    def __mul__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("*", self, other)
+
+    def __div__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("/", self, other)
+
+    def __truediv__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("/", self, other)
+
+    def __mod__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("%", self, other)
+
+    def __radd__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("+", self, other, reverse=True)
+
+    def __rsub__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("-", self, other, reverse=True)
+
+    def __rmul__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("*", self, other, reverse=True)
+
+    def __rdiv__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("/", self, other, reverse=True)
+
+    def __rtruediv__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("/", self, other, reverse=True)
+
+    def __rmod__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("%", self, other, reverse=True)
+
+    def __pow__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("power", self, other)
+
+    def __rpow__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("power", self, other, reverse=True)
+
+    def __ge__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op(">=", self, other)
+
+    def __le__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("<=", self, other)
+
+    def eqNullSafe(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("<=>", self, other)
+
+    def __neg__(self) -> ParentColumn:
+        return _func_op("negative", self)
 
     # `and`, `or`, `not` cannot be overloaded in Python,
     # so use bitwise operators as boolean operators
-    __and__ = _bin_op("and")
-    __or__ = _bin_op("or")
-    __invert__ = _func_op("not")
-    __rand__ = _bin_op("and")
-    __ror__ = _bin_op("or")
+    def __and__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("and", self, other)
+
+    def __or__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("or", self, other)
+
+    def __invert__(self) -> ParentColumn:
+        return _func_op("not", self)
+
+    def __rand__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("and", self, other)
+
+    def __ror__(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("or", self, other)
 
     # container operators
     def __contains__(self, item: Any) -> None:
@@ -154,27 +243,53 @@ class Column:
         )
 
     # bitwise operators
-    bitwiseOR = _bin_op("|", PySparkColumn.bitwiseOR.__doc__)
-    bitwiseAND = _bin_op("&", PySparkColumn.bitwiseAND.__doc__)
-    bitwiseXOR = _bin_op("^", PySparkColumn.bitwiseXOR.__doc__)
+    def bitwiseOR(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("|", self, other)
+
+    def bitwiseAND(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("&", self, other)
 
-    isNull = _unary_op("isnull", PySparkColumn.isNull.__doc__)
-    isNotNull = _unary_op("isnotnull", PySparkColumn.isNotNull.__doc__)
-    isNaN = _unary_op("isNaN", PySparkColumn.isNaN.__doc__)
+    def bitwiseXOR(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("^", self, other)
+
+    def isNull(self) -> ParentColumn:
+        return _unary_op("isNull", self)
+
+    def isNotNull(self) -> ParentColumn:
+        return _unary_op("isNotNull", self)
+
+    def isNaN(self) -> ParentColumn:
+        return _unary_op("isNaN", self)
 
     def __ne__(  # type: ignore[override]
         self,
         other: Any,
-    ) -> "Column":
-        """binary function"""
-        return _func_op("not")(_bin_op("==")(self, other))
+    ) -> ParentColumn:
+        return _func_op("not", _bin_op("==", self, other))
 
     # string methods
-    contains = _bin_op("contains", PySparkColumn.contains.__doc__)
-    startswith = _bin_op("startswith", PySparkColumn.startswith.__doc__)
-    endswith = _bin_op("endswith", PySparkColumn.endswith.__doc__)
-
-    def when(self, condition: "Column", value: Any) -> "Column":
+    def contains(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("contains", self, other)
+
+    def startswith(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("startsWith", self, other)
+
+    def endswith(
+        self, other: Union[ParentColumn, "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
+    ) -> ParentColumn:
+        return _bin_op("endsWith", self, other)
+
+    def when(self, condition: ParentColumn, value: Any) -> ParentColumn:
         if not isinstance(condition, Column):
             raise PySparkTypeError(
                 error_class="NOT_COLUMN",
@@ -202,9 +317,7 @@ class Column:
 
         return Column(CaseWhen(branches=_branches, else_value=None))
 
-    when.__doc__ = PySparkColumn.when.__doc__
-
-    def otherwise(self, value: Any) -> "Column":
+    def otherwise(self, value: Any) -> ParentColumn:
         if not isinstance(self._expr, CaseWhen):
             raise PySparkTypeError(
                 "otherwise() can only be applied on a Column previously generated by when()"
@@ -222,21 +335,18 @@ class Column:
 
         return Column(CaseWhen(branches=self._expr._branches, else_value=_value))
 
-    otherwise.__doc__ = PySparkColumn.otherwise.__doc__
+    def like(self: ParentColumn, other: str) -> ParentColumn:
+        return _bin_op("like", self, other)
 
-    like = _bin_op("like", PySparkColumn.like.__doc__)
-    rlike = _bin_op("rlike", PySparkColumn.rlike.__doc__)
-    ilike = _bin_op("ilike", PySparkColumn.ilike.__doc__)
+    def rlike(self: ParentColumn, other: str) -> ParentColumn:
+        return _bin_op("rlike", self, other)
 
-    @overload
-    def substr(self, startPos: int, length: int) -> "Column":
-        ...
+    def ilike(self: ParentColumn, other: str) -> ParentColumn:
+        return _bin_op("ilike", self, other)
 
-    @overload
-    def substr(self, startPos: "Column", length: "Column") -> "Column":
-        ...
-
-    def substr(self, startPos: Union[int, "Column"], length: Union[int, "Column"]) -> "Column":
+    def substr(
+        self, startPos: Union[int, ParentColumn], length: Union[int, ParentColumn]
+    ) -> ParentColumn:
         if type(startPos) != type(length):
             raise PySparkTypeError(
                 error_class="NOT_SAME_TYPE",
@@ -259,14 +369,13 @@ class Column:
                 error_class="NOT_COLUMN_OR_INT",
                 message_parameters={"arg_name": "startPos", "arg_type": type(length).__name__},
             )
-        return Column(UnresolvedFunction("substr", [self._expr, start_expr, length_expr]))
-
-    substr.__doc__ = PySparkColumn.substr.__doc__
+        return Column(
+            UnresolvedFunction(
+                "substr", [self._expr, start_expr, length_expr]  # type: ignore[list-item]
+            )
+        )
 
-    def __eq__(self, other: Any) -> "Column":  # type: ignore[override]
-        """Returns a binary expression with the current column as the left
-        side and the other expression as the right side.
-        """
+    def __eq__(self, other: Any) -> ParentColumn:  # type: ignore[override]
         if other is None or isinstance(
             other, (bool, float, int, str, datetime.datetime, datetime.date, decimal.Decimal)
         ):
@@ -279,46 +388,30 @@ class Column:
     def to_plan(self, session: "SparkConnectClient") -> proto.Expression:
         return self._expr.to_plan(session)
 
-    def alias(self, *alias: str, **kwargs: Any) -> "Column":
+    def alias(self, *alias: str, **kwargs: Any) -> ParentColumn:
         return Column(self._expr.alias(*alias, **kwargs))
 
-    alias.__doc__ = PySparkColumn.alias.__doc__
-
     name = alias
 
-    name.__doc__ = PySparkColumn.name.__doc__
-
-    def asc(self) -> "Column":
+    def asc(self) -> ParentColumn:
         return self.asc_nulls_first()
 
-    asc.__doc__ = PySparkColumn.asc.__doc__
-
-    def asc_nulls_first(self) -> "Column":
+    def asc_nulls_first(self) -> ParentColumn:
         return Column(SortOrder(self._expr, ascending=True, nullsFirst=True))
 
-    asc_nulls_first.__doc__ = PySparkColumn.asc_nulls_first.__doc__
-
-    def asc_nulls_last(self) -> "Column":
+    def asc_nulls_last(self) -> ParentColumn:
         return Column(SortOrder(self._expr, ascending=True, nullsFirst=False))
 
-    asc_nulls_last.__doc__ = PySparkColumn.asc_nulls_last.__doc__
-
-    def desc(self) -> "Column":
+    def desc(self) -> ParentColumn:
         return self.desc_nulls_last()
 
-    desc.__doc__ = PySparkColumn.desc.__doc__
-
-    def desc_nulls_first(self) -> "Column":
+    def desc_nulls_first(self) -> ParentColumn:
         return Column(SortOrder(self._expr, ascending=False, nullsFirst=True))
 
-    desc_nulls_first.__doc__ = PySparkColumn.desc_nulls_first.__doc__
-
-    def desc_nulls_last(self) -> "Column":
+    def desc_nulls_last(self) -> ParentColumn:
         return Column(SortOrder(self._expr, ascending=False, nullsFirst=False))
 
-    desc_nulls_last.__doc__ = PySparkColumn.desc_nulls_last.__doc__
-
-    def cast(self, dataType: Union[DataType, str]) -> "Column":
+    def cast(self, dataType: Union[DataType, str]) -> ParentColumn:
         if isinstance(dataType, (DataType, str)):
             return Column(CastExpression(expr=self._expr, data_type=dataType))
         else:
@@ -327,11 +420,9 @@ class Column:
                 message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
             )
 
-    cast.__doc__ = PySparkColumn.cast.__doc__
-
     astype = cast
 
-    def try_cast(self, dataType: Union[DataType, str]) -> "Column":
+    def try_cast(self, dataType: Union[DataType, str]) -> ParentColumn:
         if isinstance(dataType, (DataType, str)):
             return Column(
                 CastExpression(
@@ -346,12 +437,10 @@ class Column:
                 message_parameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
             )
 
-    try_cast.__doc__ = PySparkColumn.try_cast.__doc__
-
     def __repr__(self) -> str:
         return "Column<'%s'>" % self._expr.__repr__()
 
-    def over(self, window: "WindowSpec") -> "Column":
+    def over(self, window: "WindowSpec") -> ParentColumn:  # type: ignore[override]
         from pyspark.sql.connect.window import WindowSpec
 
         if not isinstance(window, WindowSpec):
@@ -362,9 +451,7 @@ class Column:
 
         return Column(WindowExpression(windowFunction=self._expr, windowSpec=window))
 
-    over.__doc__ = PySparkColumn.over.__doc__
-
-    def isin(self, *cols: Any) -> "Column":
+    def isin(self, *cols: Any) -> ParentColumn:
         if len(cols) == 1 and isinstance(cols[0], (list, set)):
             _cols = list(cols[0])
         else:
@@ -379,18 +466,14 @@ class Column:
 
         return Column(UnresolvedFunction("in", _exprs))
 
-    isin.__doc__ = PySparkColumn.isin.__doc__
-
     def between(
         self,
-        lowerBound: Union["Column", "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
-        upperBound: Union["Column", "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
-    ) -> "Column":
+        lowerBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+        upperBound: Union[ParentColumn, "LiteralType", "DateTimeLiteral", "DecimalLiteral"],
+    ) -> ParentColumn:
         return (self >= lowerBound) & (self <= upperBound)
 
-    between.__doc__ = PySparkColumn.between.__doc__
-
-    def getItem(self, key: Any) -> "Column":
+    def getItem(self, key: Any) -> ParentColumn:
         if isinstance(key, Column):
             warnings.warn(
                 "A column as 'key' in getItem is deprecated as of Spark 3.0, and will not "
@@ -400,9 +483,7 @@ class Column:
             )
         return self[key]
 
-    getItem.__doc__ = PySparkColumn.getItem.__doc__
-
-    def getField(self, name: Any) -> "Column":
+    def getField(self, name: Any) -> ParentColumn:
         if isinstance(name, Column):
             warnings.warn(
                 "A column as 'name' in getField is deprecated as of Spark 3.0, and will not "
@@ -412,9 +493,7 @@ class Column:
             )
         return self[name]
 
-    getField.__doc__ = PySparkColumn.getField.__doc__
-
-    def withField(self, fieldName: str, col: "Column") -> "Column":
+    def withField(self, fieldName: str, col: ParentColumn) -> ParentColumn:
         if not isinstance(fieldName, str):
             raise PySparkTypeError(
                 error_class="NOT_STR",
@@ -429,9 +508,7 @@ class Column:
 
         return Column(WithField(self._expr, fieldName, col._expr))
 
-    withField.__doc__ = PySparkColumn.withField.__doc__
-
-    def dropFields(self, *fieldNames: str) -> "Column":
+    def dropFields(self, *fieldNames: str) -> ParentColumn:
         dropField: Optional[DropField] = None
         for fieldName in fieldNames:
             if not isinstance(fieldName, str):
@@ -458,9 +535,7 @@ class Column:
 
         return Column(dropField)
 
-    dropFields.__doc__ = PySparkColumn.dropFields.__doc__
-
-    def __getattr__(self, item: Any) -> "Column":
+    def __getattr__(self, item: Any) -> ParentColumn:
         if item == "_jc":
             raise PySparkAttributeError(
                 error_class="JVM_ATTRIBUTE_NOT_SUPPORTED", message_parameters={"attr_name": "_jc"}
@@ -471,7 +546,7 @@ class Column:
             )
         return self[item]
 
-    def __getitem__(self, k: Any) -> "Column":
+    def __getitem__(self, k: Any) -> ParentColumn:
         if isinstance(k, slice):
             if k.step is not None:
                 raise PySparkValueError(
@@ -499,9 +574,6 @@ class Column:
     __bool__ = __nonzero__
 
 
-Column.__doc__ = PySparkColumn.__doc__
-
-
 def _test() -> None:
     import os
     import sys
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 98a9cd4875c6..c1d6b4f81c3c 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -73,7 +73,7 @@ from pyspark.sql.connect.conversion import ArrowTableToRowsConversion
 from pyspark.sql.connect.group import GroupedData
 from pyspark.sql.connect.readwriter import DataFrameWriter, DataFrameWriterV2
 from pyspark.sql.connect.streaming.readwriter import DataStreamWriter
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.expressions import (
     ColumnReference,
     UnresolvedRegex,
@@ -255,12 +255,14 @@ class DataFrame(ParentDataFrame):
         return DataFrame(plan.SubqueryAlias(self._plan, alias), session=self._session)
 
     def colRegex(self, colName: str) -> Column:
+        from pyspark.sql.connect.column import Column as ConnectColumn
+
         if not isinstance(colName, str):
             raise PySparkTypeError(
                 error_class="NOT_STR",
                 message_parameters={"arg_name": "colName", "arg_type": type(colName).__name__},
             )
-        return Column(UnresolvedRegex(colName, self._plan._plan_id))
+        return ConnectColumn(UnresolvedRegex(colName, self._plan._plan_id))
 
     @property
     def dtypes(self) -> List[Tuple[str, str]]:
@@ -750,7 +752,7 @@ class DataFrame(ParentDataFrame):
             session=self._session,
         )
 
-    orderBy = sort  # type: ignore[assignment]
+    orderBy = sort
 
     def sortWithinPartitions(
         self,
@@ -959,7 +961,7 @@ class DataFrame(ParentDataFrame):
             self._session,
         )
 
-    melt = unpivot  # type: ignore[assignment]
+    melt = unpivot
 
     def withWatermark(self, eventTime: str, delayThreshold: str) -> ParentDataFrame:
         # TODO: reuse error handling code in sql.DataFrame.withWatermark()
@@ -1703,9 +1705,11 @@ class DataFrame(ParentDataFrame):
     def __getitem__(
         self, item: Union[int, str, Column, List, Tuple]
     ) -> Union[Column, ParentDataFrame]:
+        from pyspark.sql.connect.column import Column as ConnectColumn
+
         if isinstance(item, str):
             if item == "*":
-                return Column(
+                return ConnectColumn(
                     UnresolvedStar(
                         unparsed_target=None,
                         plan_id=self._plan._plan_id,
@@ -1716,7 +1720,7 @@ class DataFrame(ParentDataFrame):
                 # if (sparkSession.sessionState.conf.supportQuotedRegexColumnName) {
                 #   colRegex(colName)
                 # } else {
-                #   Column(addDataFrameIdToCol(resolve(colName)))
+                #   ConnectColumn(addDataFrameIdToCol(resolve(colName)))
                 # }
 
                 # validate the column name
@@ -1742,7 +1746,9 @@ class DataFrame(ParentDataFrame):
             )
 
     def _col(self, name: str) -> Column:
-        return Column(
+        from pyspark.sql.connect.column import Column as ConnectColumn
+
+        return ConnectColumn(
             ColumnReference(
                 unparsed_identifier=name,
                 plan_id=self._plan._plan_id,
@@ -2037,9 +2043,7 @@ class DataFrame(ParentDataFrame):
         def foreach_func(row: Any) -> None:
             f(row)
 
-        self.select(  # type: ignore[call-overload]
-            F.struct(*self.schema.fieldNames()).alias("row")
-        ).select(
+        self.select(F.struct(*self.schema.fieldNames()).alias("row")).select(
             F.udf(foreach_func, StructType())("row")  # type: ignore[arg-type]
         ).collect()
 
diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py
index 4a3f634d21bb..519e53c3a13f 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -43,7 +43,7 @@ import numpy as np
 
 from pyspark.errors import PySparkTypeError, PySparkValueError
 from pyspark.sql.dataframe import DataFrame as ParentDataFrame
-from pyspark.sql.connect.column import Column
+from pyspark.sql import Column
 from pyspark.sql.connect.expressions import (
     CaseWhen,
     SortOrder,
@@ -112,14 +112,16 @@ def _invoke_function(name: str, *args: Union[Column, Expression]) -> Column:
     -------
     :class:`Column`
     """
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     expressions: List[Expression] = []
     for arg in args:
         assert isinstance(arg, (Column, Expression))
         if isinstance(arg, Column):
-            expressions.append(arg._expr)
+            expressions.append(arg._expr)  # type: ignore[arg-type]
         else:
             expressions.append(arg)
-    return Column(UnresolvedFunction(name, expressions))
+    return ConnectColumn(UnresolvedFunction(name, expressions))
 
 
 def _invoke_function_over_columns(name: str, *cols: "ColumnOrName") -> Column:
@@ -180,6 +182,8 @@ def _create_lambda(f: Callable) -> LambdaFunction:
             - (Column, Column) -> Column: ...
             - (Column, Column, Column) -> Column: ...
     """
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     parameters = _get_lambda_parameters(f)
 
     arg_names = ["x", "y", "z"][: len(parameters)]
@@ -187,7 +191,7 @@ def _create_lambda(f: Callable) -> LambdaFunction:
         UnresolvedNamedLambdaVariable([UnresolvedNamedLambdaVariable.fresh_var_name(arg_name)])
         for arg_name in arg_names
     ]
-    arg_cols = [Column(arg_expr) for arg_expr in arg_exprs]
+    arg_cols = [ConnectColumn(arg_expr) for arg_expr in arg_exprs]
 
     result = f(*arg_cols)
 
@@ -197,7 +201,7 @@ def _create_lambda(f: Callable) -> LambdaFunction:
             message_parameters={"func_name": f.__name__, "return_type": type(result).__name__},
         )
 
-    return LambdaFunction(result._expr, arg_exprs)
+    return LambdaFunction(result._expr, arg_exprs)  # type: ignore[arg-type]
 
 
 def _invoke_higher_order_function(
@@ -234,12 +238,14 @@ def _options_to_col(options: Dict[str, Any]) -> Column:
 
 
 def col(col: str) -> Column:
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     if col == "*":
-        return Column(UnresolvedStar(unparsed_target=None))
+        return ConnectColumn(UnresolvedStar(unparsed_target=None))
     elif col.endswith(".*"):
-        return Column(UnresolvedStar(unparsed_target=col))
+        return ConnectColumn(UnresolvedStar(unparsed_target=col))
     else:
-        return Column(ColumnReference(unparsed_identifier=col))
+        return ConnectColumn(ColumnReference(unparsed_identifier=col))
 
 
 col.__doc__ = pysparkfuncs.col.__doc__
@@ -249,6 +255,8 @@ column = col
 
 
 def lit(col: Any) -> Column:
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     if isinstance(col, Column):
         return col
     elif isinstance(col, list):
@@ -272,7 +280,7 @@ def lit(col: Any) -> Column:
 
         return array(*[lit(c) for c in col])
     else:
-        return Column(LiteralExpression._from_value(col))
+        return ConnectColumn(LiteralExpression._from_value(col))
 
 
 lit.__doc__ = pysparkfuncs.lit.__doc__
@@ -336,7 +344,9 @@ coalesce.__doc__ = pysparkfuncs.coalesce.__doc__
 
 
 def expr(str: str) -> Column:
-    return Column(SQLExpression(str))
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
+    return ConnectColumn(SQLExpression(str))
 
 
 expr.__doc__ = pysparkfuncs.expr.__doc__
@@ -429,6 +439,8 @@ spark_partition_id.__doc__ = pysparkfuncs.spark_partition_id.__doc__
 
 
 def when(condition: Column, value: Any) -> Column:
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     # Explicitly not using ColumnOrName type here to make reading condition less opaque
     if not isinstance(condition, Column):
         raise PySparkTypeError(
@@ -438,7 +450,12 @@ def when(condition: Column, value: Any) -> Column:
 
     value_col = value if isinstance(value, Column) else lit(value)
 
-    return Column(CaseWhen(branches=[(condition._expr, value_col._expr)], else_value=None))
+    return ConnectColumn(
+        CaseWhen(
+            branches=[(condition._expr, value_col._expr)],  # type: ignore[list-item]
+            else_value=None,
+        )
+    )
 
 
 when.__doc__ = pysparkfuncs.when.__doc__
@@ -1045,8 +1062,12 @@ countDistinct.__doc__ = pysparkfuncs.countDistinct.__doc__
 
 
 def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     _exprs = [_to_col(c)._expr for c in [col] + list(cols)]
-    return Column(UnresolvedFunction("count", _exprs, is_distinct=True))
+    return ConnectColumn(
+        UnresolvedFunction("count", _exprs, is_distinct=True)  # type: ignore[arg-type]
+    )
 
 
 count_distinct.__doc__ = pysparkfuncs.count_distinct.__doc__
@@ -1293,7 +1314,11 @@ sumDistinct.__doc__ = pysparkfuncs.sumDistinct.__doc__
 
 
 def sum_distinct(col: "ColumnOrName") -> Column:
-    return Column(UnresolvedFunction("sum", [_to_col(col)._expr], is_distinct=True))
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
+    return ConnectColumn(
+        UnresolvedFunction("sum", [_to_col(col)._expr], is_distinct=True)  # type: ignore[list-item]
+    )
 
 
 sum_distinct.__doc__ = pysparkfuncs.sum_distinct.__doc__
@@ -4101,8 +4126,10 @@ udtf.__doc__ = pysparkfuncs.udtf.__doc__
 
 
 def call_function(funcName: str, *cols: "ColumnOrName") -> Column:
+    from pyspark.sql.connect.column import Column as ConnectColumn
+
     expressions = [_to_col(c)._expr for c in cols]
-    return Column(CallFunction(funcName, expressions))
+    return ConnectColumn(CallFunction(funcName, expressions))  # type: ignore[arg-type]
 
 
 call_function.__doc__ = pysparkfuncs.call_function.__doc__
diff --git a/python/pyspark/sql/connect/functions/partitioning.py b/python/pyspark/sql/connect/functions/partitioning.py
index bfeddad7d568..5d2dd58313bb 100644
--- a/python/pyspark/sql/connect/functions/partitioning.py
+++ b/python/pyspark/sql/connect/functions/partitioning.py
@@ -22,7 +22,7 @@ from typing import Union, TYPE_CHECKING
 
 from pyspark.errors import PySparkTypeError
 from pyspark.sql import functions as pysparkfuncs
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.functions.builtin import _to_col, _invoke_function_over_columns
 from pyspark.sql.connect.functions.builtin import lit, _invoke_function
 
diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py
index b866f61efe4a..699dce76c4a1 100644
--- a/python/pyspark/sql/connect/group.py
+++ b/python/pyspark/sql/connect/group.py
@@ -38,7 +38,7 @@ from pyspark.sql.types import NumericType
 from pyspark.sql.types import StructType
 
 import pyspark.sql.connect.plan as plan
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.functions import builtin as F
 from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
 
@@ -61,10 +61,10 @@ class GroupedData:
         self,
         df: "DataFrame",
         group_type: str,
-        grouping_cols: Sequence["Column"],
-        pivot_col: Optional["Column"] = None,
+        grouping_cols: Sequence[Column],
+        pivot_col: Optional[Column] = None,
         pivot_values: Optional[Sequence["LiteralType"]] = None,
-        grouping_sets: Optional[Sequence[Sequence["Column"]]] = None,
+        grouping_sets: Optional[Sequence[Sequence[Column]]] = None,
     ) -> None:
         from pyspark.sql.connect.dataframe import DataFrame
 
diff --git a/python/pyspark/sql/connect/observation.py b/python/pyspark/sql/connect/observation.py
index 4fefb8aac41f..2471cf04cfbe 100644
--- a/python/pyspark/sql/connect/observation.py
+++ b/python/pyspark/sql/connect/observation.py
@@ -23,7 +23,7 @@ from pyspark.errors import (
     IllegalArgumentException,
     PySparkAssertionError,
 )
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.dataframe import DataFrame
 from pyspark.sql.observation import Observation as PySparkObservation
 import pyspark.sql.connect.plan as plan
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
index 239ee23c2061..4ac4946745f5 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -14,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+# mypy: disable-error-code="operator"
+
 from pyspark.resource import ResourceProfile
 from pyspark.sql.connect.utils import check_dependencies
 
@@ -45,8 +48,8 @@ from pyspark.storagelevel import StorageLevel
 from pyspark.sql.types import DataType
 
 import pyspark.sql.connect.proto as proto
+from pyspark.sql.column import Column
 from pyspark.sql.connect.conversion import storage_level_to_proto
-from pyspark.sql.connect.column import Column
 from pyspark.sql.connect.expressions import Expression
 from pyspark.sql.connect.types import pyspark_types_to_proto_types, UnparsedDataType
 from pyspark.errors import (
diff --git a/python/pyspark/sql/connect/protobuf/functions.py b/python/pyspark/sql/connect/protobuf/functions.py
index 058925942fa0..07e9b4b8c686 100644
--- a/python/pyspark/sql/connect/protobuf/functions.py
+++ b/python/pyspark/sql/connect/protobuf/functions.py
@@ -27,7 +27,7 @@ from typing import Dict, Optional, TYPE_CHECKING
 
 from pyspark.sql.protobuf import functions as PyProtobufFunctions
 
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.functions.builtin import _invoke_function, _to_col, _options_to_col, lit
 
 if TYPE_CHECKING:
diff --git a/python/pyspark/sql/connect/udf.py b/python/pyspark/sql/connect/udf.py
index f3aa719b2bb6..a5faf9446fff 100644
--- a/python/pyspark/sql/connect/udf.py
+++ b/python/pyspark/sql/connect/udf.py
@@ -161,7 +161,10 @@ class UserDefinedFunction:
         self, *args: "ColumnOrName", **kwargs: "ColumnOrName"
     ) -> CommonInlineUserDefinedFunction:
         def to_expr(col: "ColumnOrName") -> Expression:
-            return col._expr if isinstance(col, Column) else ColumnReference(col)
+            if isinstance(col, Column):
+                return col._expr
+            else:
+                return ColumnReference(col)  # type: ignore[arg-type]
 
         arg_exprs: List[Expression] = [to_expr(arg) for arg in args] + [
             NamedArgumentExpression(key, to_expr(value)) for key, value in kwargs.items()
diff --git a/python/pyspark/sql/connect/udtf.py b/python/pyspark/sql/connect/udtf.py
index 4ee39dc89b8e..739289d72a3b 100644
--- a/python/pyspark/sql/connect/udtf.py
+++ b/python/pyspark/sql/connect/udtf.py
@@ -141,7 +141,10 @@ class UserDefinedTableFunction:
         self, *args: "ColumnOrName", **kwargs: "ColumnOrName"
     ) -> CommonInlineUserDefinedTableFunction:
         def to_expr(col: "ColumnOrName") -> Expression:
-            return col._expr if isinstance(col, Column) else ColumnReference(col)
+            if isinstance(col, Column):
+                return col._expr
+            else:
+                return ColumnReference(col)  # type: ignore[arg-type]
 
         arg_exprs: List[Expression] = [to_expr(arg) for arg in args] + [
             NamedArgumentExpression(key, to_expr(value)) for key, value in kwargs.items()
diff --git a/python/pyspark/sql/connect/window.py b/python/pyspark/sql/connect/window.py
index e30a5b7d7a9e..04cf4c91d320 100644
--- a/python/pyspark/sql/connect/window.py
+++ b/python/pyspark/sql/connect/window.py
@@ -21,7 +21,7 @@ check_dependencies(__name__)
 import sys
 from typing import TYPE_CHECKING, Union, Sequence, List, Optional
 
-from pyspark.sql.connect.column import Column
+from pyspark.sql.column import Column
 from pyspark.sql.connect.expressions import (
     ColumnReference,
     Expression,
@@ -107,7 +107,7 @@ class WindowSpec:
         newPartitionSpec: List[Expression] = []
         for c in _cols:
             if isinstance(c, Column):
-                newPartitionSpec.append(c._expr)
+                newPartitionSpec.append(c._expr)  # type: ignore[arg-type]
             else:
                 newPartitionSpec.append(ColumnReference(c))
 
@@ -143,7 +143,7 @@ class WindowSpec:
                 if isinstance(c._expr, SortOrder):
                     newOrderSpec.append(c._expr)
                 else:
-                    newOrderSpec.append(SortOrder(c._expr))
+                    newOrderSpec.append(SortOrder(c._expr))  # type: ignore[arg-type]
             else:
                 newOrderSpec.append(SortOrder(ColumnReference(c)))
 
diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py
index cef6eeed4fbc..96be5de0180b 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -40,7 +40,7 @@ from typing import (
 )
 
 from pyspark.errors import PySparkTypeError, PySparkValueError
-from pyspark.sql.column import Column, _to_java_column, _to_seq, _create_column_from_literal
+from pyspark.sql.column import Column
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.types import ArrayType, DataType, StringType, StructType, _from_numpy_type
 
@@ -106,6 +106,8 @@ def _invoke_function_over_columns(name: str, *cols: "ColumnOrName") -> Column:
     Invokes n-ary JVM function identified by name
     and wraps the result with :class:`~pyspark.sql.Column`.
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function(name, *(_to_java_column(col) for col in cols))
 
 
@@ -114,6 +116,8 @@ def _invoke_function_over_seq_of_columns(name: str, cols: "Iterable[ColumnOrName
     Invokes unary JVM function identified by name with
     and wraps the result with :class:`~pyspark.sql.Column`.
     """
+    from pyspark.sql.classic.column import _to_java_column, _to_seq
+
     sc = _get_active_spark_context()
     return _invoke_function(name, _to_seq(sc, cols, _to_java_column))
 
@@ -123,6 +127,7 @@ def _invoke_binary_math_function(name: str, col1: Any, col2: Any) -> Column:
     Invokes binary JVM math function identified by name
     and wraps the result with :class:`~pyspark.sql.Column`.
     """
+    from pyspark.sql.classic.column import _to_java_column, _create_column_from_literal
 
     # For legacy reasons, the arguments here can be implicitly converted into column
     cols = [
@@ -1005,6 +1010,8 @@ def mode(col: "ColumnOrName", deterministic: bool = False) -> Column:
     |                                    -10|
     +---------------------------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("mode", _to_java_column(col), deterministic)
 
 
@@ -5123,6 +5130,8 @@ def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> C
     |           95546|      102065|
     +----------------+------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if rsd is None:
         return _invoke_function_over_columns("approx_count_distinct", col)
     else:
@@ -5386,6 +5395,8 @@ def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column:
     |                             2|
     +------------------------------+
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function(
         "count_distinct", _to_java_column(col), _to_seq(sc, cols, _to_java_column)
@@ -5443,6 +5454,8 @@ def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
     |  Bob|         5|
     +-----+----------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("first", _to_java_column(col), ignorenulls)
 
 
@@ -5714,6 +5727,8 @@ def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column:
     |  Bob|        5|
     +-----+---------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("last", _to_java_column(col), ignorenulls)
 
 
@@ -5847,6 +5862,8 @@ def percentile(
     |  2|  19.967859769284075|
     +---+--------------------+
     """
+    from pyspark.sql.classic.column import _to_seq, _create_column_from_literal, _to_java_column
+
     sc = _get_active_spark_context()
 
     if isinstance(percentage, (list, tuple)):
@@ -5924,6 +5941,8 @@ def percentile_approx(
      |-- key: long (nullable = true)
      |-- median: double (nullable = true)
     """
+    from pyspark.sql.classic.column import _to_seq, _create_column_from_literal, _to_java_column
+
     sc = _get_active_spark_context()
 
     if isinstance(percentage, (list, tuple)):
@@ -5998,6 +6017,8 @@ def approx_percentile(
      |-- key: long (nullable = true)
      |-- approx_percentile(value, 0.5, 1000000): double (nullable = true)
     """
+    from pyspark.sql.classic.column import _to_seq, _create_column_from_literal, _to_java_column
+
     sc = _get_active_spark_context()
 
     if isinstance(percentage, (list, tuple)):
@@ -6279,6 +6300,8 @@ def shiftleft(col: "ColumnOrName", numBits: int) -> Column:
     >>> spark.createDataFrame([(21,)], ['a']).select(shiftleft('a', 1).alias('r')).collect()
     [Row(r=42)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("shiftleft", _to_java_column(col), numBits)
 
 
@@ -6324,6 +6347,8 @@ def shiftright(col: "ColumnOrName", numBits: int) -> Column:
     >>> spark.createDataFrame([(42,)], ['a']).select(shiftright('a', 1).alias('r')).collect()
     [Row(r=21)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("shiftright", _to_java_column(col), numBits)
 
 
@@ -6370,6 +6395,8 @@ def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column:
     >>> df.select(shiftrightunsigned('a', 1).alias('r')).collect()
     [Row(r=9223372036854775787)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("shiftrightunsigned", _to_java_column(col), numBits)
 
 
@@ -6684,6 +6711,8 @@ def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = Non
     |1.3862943611198906|
     +------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if arg2 is None:
         return _invoke_function_over_columns("log", cast("ColumnOrName", arg1))
     else:
@@ -6781,6 +6810,8 @@ def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column:
     >>> df.select(conv(df.n, 2, 16).alias('hex')).collect()
     [Row(hex='15')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("conv", _to_java_column(col), fromBase, toBase)
 
 
@@ -6894,6 +6925,8 @@ def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) ->
     |  b|  8|           -1|
     +---+---+-------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("lag", _to_java_column(col), offset, default)
 
 
@@ -6975,6 +7008,8 @@ def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) ->
     |  b|  8|        -1|
     +---+---+----------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("lead", _to_java_column(col), offset, default)
 
 
@@ -7049,6 +7084,8 @@ def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = Fa
     |  b|  8|        8|
     +---+---+---------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("nth_value", _to_java_column(col), offset, ignoreNulls)
 
 
@@ -7561,6 +7598,8 @@ def date_format(date: "ColumnOrName", format: str) -> Column:
     >>> df.select(date_format('dt', 'MM/dd/yyyy').alias('date')).collect()
     [Row(date='04/08/2015')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("date_format", _to_java_column(date), format)
 
 
@@ -8418,6 +8457,8 @@ def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool
     >>> df.select(months_between(df.date1, df.date2, False).alias('months')).collect()
     [Row(months=3.9495967741935485)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function(
         "months_between", _to_java_column(date1), _to_java_column(date2), roundOff
     )
@@ -8459,6 +8500,8 @@ def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column:
     >>> df.select(to_date(df.t, 'yyyy-MM-dd HH:mm:ss').alias('date')).collect()
     [Row(date=datetime.date(1997, 2, 28))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if format is None:
         return _invoke_function_over_columns("to_date", col)
     else:
@@ -8595,6 +8638,8 @@ def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column:
     |1997-02-28 10:30:00|
     +-------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if format is None:
         return _invoke_function_over_columns("to_timestamp", col)
     else:
@@ -8852,6 +8897,8 @@ def trunc(date: "ColumnOrName", format: str) -> Column:
     >>> df.select(trunc(df.d, 'mon').alias('month')).collect()
     [Row(month=datetime.date(1997, 2, 1))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("trunc", _to_java_column(date), format)
 
 
@@ -8889,6 +8936,8 @@ def date_trunc(format: str, timestamp: "ColumnOrName") -> Column:
     >>> df.select(date_trunc('mon', df.t).alias('month')).collect()
     [Row(month=datetime.datetime(1997, 2, 1, 0, 0))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("date_trunc", format, _to_java_column(timestamp))
 
 
@@ -8922,6 +8971,8 @@ def next_day(date: "ColumnOrName", dayOfWeek: str) -> Column:
     >>> df.select(next_day(df.d, 'Sun').alias('date')).collect()
     [Row(date=datetime.date(2015, 8, 2))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("next_day", _to_java_column(date), dayOfWeek)
 
 
@@ -8951,6 +9002,8 @@ def last_day(date: "ColumnOrName") -> Column:
     >>> df.select(last_day(df.d).alias('date')).collect()
     [Row(date=datetime.date(1997, 2, 28))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("last_day", _to_java_column(date))
 
 
@@ -8986,6 +9039,8 @@ def from_unixtime(timestamp: "ColumnOrName", format: str = "yyyy-MM-dd HH:mm:ss"
     [Row(ts='2015-04-08 00:00:00')]
     >>> spark.conf.unset("spark.sql.session.timeZone")
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("from_unixtime", _to_java_column(timestamp), format)
 
 
@@ -9066,6 +9121,8 @@ def unix_timestamp(
 
     >>> spark.conf.unset("spark.sql.session.timeZone")
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if timestamp is None:
         return _invoke_function("unix_timestamp")
     return _invoke_function("unix_timestamp", _to_java_column(timestamp), format)
@@ -9120,6 +9177,8 @@ def from_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
     >>> df.select(from_utc_timestamp(df.ts, df.tz).alias('local_time')).collect()
     [Row(local_time=datetime.datetime(1997, 2, 28, 19, 30))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if isinstance(tz, Column):
         tz = _to_java_column(tz)
     return _invoke_function("from_utc_timestamp", _to_java_column(timestamp), tz)
@@ -9174,6 +9233,8 @@ def to_utc_timestamp(timestamp: "ColumnOrName", tz: "ColumnOrName") -> Column:
     >>> df.select(to_utc_timestamp(df.ts, df.tz).alias('utc_time')).collect()
     [Row(utc_time=datetime.datetime(1997, 2, 28, 1, 30))]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if isinstance(tz, Column):
         tz = _to_java_column(tz)
     return _invoke_function("to_utc_timestamp", _to_java_column(timestamp), tz)
@@ -9367,6 +9428,7 @@ def window(
     |2016-03-11 09:00:05|2016-03-11 09:00:10|  1|
     +-------------------+-------------------+---+
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     def check_string_field(field, fieldName):  # type: ignore[no-untyped-def]
         if not field or type(field) is not str:
@@ -9438,6 +9500,8 @@ def window_time(
     ... ).collect()
     [Row(end='2016-03-11 09:00:10', window_time='2016-03-11 09:00:09.999999', sum=1)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     window_col = _to_java_column(windowColumn)
     return _invoke_function("window_time", window_col)
 
@@ -9493,6 +9557,7 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str])
     ...          w.session_window.end.cast("string").alias("end"), "sum").collect()
     [Row(start='2016-03-11 09:00:07', end='2016-03-11 09:00:12', sum=1)]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     def check_field(field: Union[Column, str], fieldName: str) -> None:
         if field is None or not isinstance(field, (str, Column)):
@@ -9863,6 +9928,8 @@ def sha2(col: "ColumnOrName", numBits: int) -> Column:
     |Bob  |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961|
     +-----+----------------------------------------------------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if numBits not in [0, 224, 256, 384, 512]:
         raise PySparkValueError(
             error_class="VALUE_NOT_ALLOWED",
@@ -10000,6 +10067,8 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None
     java.lang.RuntimeException: My error msg
     ...
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if errMsg is None:
         return _invoke_function_over_columns("assert_true", col)
     if not isinstance(errMsg, (str, Column)):
@@ -10042,6 +10111,8 @@ def raise_error(errMsg: Union[Column, str]) -> Column:
     java.lang.RuntimeException: My error message
     ...
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if not isinstance(errMsg, (str, Column)):
         raise PySparkTypeError(
             error_class="NOT_COLUMN_OR_STR",
@@ -10368,6 +10439,8 @@ def concat_ws(sep: str, *cols: "ColumnOrName") -> Column:
     >>> df.select(concat_ws('-', df.s, df.d).alias('s')).collect()
     [Row(s='abcd-123')]
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function("concat_ws", sep, _to_seq(sc, cols, _to_java_column))
 
@@ -10405,6 +10478,8 @@ def decode(col: "ColumnOrName", charset: str) -> Column:
     |            abcd|
     +----------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("decode", _to_java_column(col), charset)
 
 
@@ -10441,6 +10516,8 @@ def encode(col: "ColumnOrName", charset: str) -> Column:
     |   [61 62 63 64]|
     +----------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("encode", _to_java_column(col), charset)
 
 
@@ -10470,6 +10547,8 @@ def format_number(col: "ColumnOrName", d: int) -> Column:
     >>> spark.createDataFrame([(5,)], ['a']).select(format_number('a', 4).alias('v')).collect()
     [Row(v='5.0000')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("format_number", _to_java_column(col), d)
 
 
@@ -10501,6 +10580,8 @@ def format_string(format: str, *cols: "ColumnOrName") -> Column:
     >>> df.select(format_string('%d %s', df.a, df.b).alias('v')).collect()
     [Row(v='5 hello')]
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function("format_string", format, _to_seq(sc, cols, _to_java_column))
 
@@ -10539,6 +10620,8 @@ def instr(str: "ColumnOrName", substr: str) -> Column:
     >>> df.select(instr(df.s, 'b').alias('s')).collect()
     [Row(s=2)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("instr", _to_java_column(str), substr)
 
 
@@ -10585,6 +10668,8 @@ def overlay(
     >>> df.select(overlay("x", "y", 7, 2).alias("overlayed")).collect()
     [Row(overlayed='SPARK_COREL')]
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if not isinstance(pos, (int, str, Column)):
         raise PySparkTypeError(
             error_class="NOT_COLUMN_OR_INT_OR_STR",
@@ -10692,6 +10777,8 @@ def substring(str: "ColumnOrName", pos: int, len: int) -> Column:
     >>> df.select(substring(df.s, 1, 2).alias('s')).collect()
     [Row(s='ab')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("substring", _to_java_column(str), pos, len)
 
 
@@ -10730,6 +10817,8 @@ def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column:
     >>> df.select(substring_index(df.s, '.', -3).alias('s')).collect()
     [Row(s='b.c.d')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("substring_index", _to_java_column(str), delim, count)
 
 
@@ -10770,6 +10859,8 @@ def levenshtein(
     >>> df0.select(levenshtein('l', 'r', 2).alias('d')).collect()
     [Row(d=-1)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if threshold is None:
         return _invoke_function_over_columns("levenshtein", left, right)
     else:
@@ -10813,6 +10904,8 @@ def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column:
     >>> df.select(locate('b', df.s, 1).alias('s')).collect()
     [Row(s=2)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("locate", substr, _to_java_column(str), pos)
 
 
@@ -10846,6 +10939,8 @@ def lpad(col: "ColumnOrName", len: int, pad: str) -> Column:
     >>> df.select(lpad(df.s, 6, '#').alias('s')).collect()
     [Row(s='##abcd')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("lpad", _to_java_column(col), len, pad)
 
 
@@ -10879,6 +10974,8 @@ def rpad(col: "ColumnOrName", len: int, pad: str) -> Column:
     >>> df.select(rpad(df.s, 6, '#').alias('s')).collect()
     [Row(s='abcd##')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("rpad", _to_java_column(col), len, pad)
 
 
@@ -11242,6 +11339,8 @@ def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column:
     >>> df.select(regexp_extract('str', '(a+)(b)?(c)', 2).alias('d')).collect()
     [Row(d='')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("regexp_extract", _to_java_column(str), pattern, idx)
 
 
@@ -11320,6 +11419,8 @@ def regexp_replace(
     >>> df.select(regexp_replace("str", col("pattern"), col("replacement")).alias('d')).collect()
     [Row(d='-----')]
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if isinstance(pattern, str):
         pattern_col = _create_column_from_literal(pattern)
     else:
@@ -11668,6 +11769,8 @@ def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column:
     ...     .alias('r')).collect()
     [Row(r='1a2s3ae')]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("translate", _to_java_column(srcCol), matching, replace)
 
 
@@ -12082,6 +12185,8 @@ def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column:
     |        aa123cc|
     +---------------+
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function("printf", _to_java_column(format), _to_seq(sc, cols, _to_java_column))
 
@@ -12634,6 +12739,8 @@ def elt(*inputs: "ColumnOrName") -> Column:
     >>> df.select(elt(df.a, df.b, df.c).alias('r')).collect()
     [Row(r='scala')]
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function("elt", _to_seq(sc, inputs, _to_java_column))
 
@@ -12935,6 +13042,8 @@ def collate(col: "ColumnOrName", collation: str) -> Column:
     :class:`~pyspark.sql.Column`
         A new column of string type, where each value has the specified collation.
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("collate", _to_java_column(col), collation)
 
 
@@ -13322,6 +13431,8 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column:
     |      true|
     +----------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     value = value._jc if isinstance(value, Column) else value
     return _invoke_function("array_contains", _to_java_column(col), value)
 
@@ -13569,6 +13680,8 @@ def array_join(
     |                NULL,NULL|
     +-------------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     _get_active_spark_context()
     if null_replacement is None:
         return _invoke_function("array_join", _to_java_column(col), delimiter)
@@ -13749,6 +13862,8 @@ def array_position(col: "ColumnOrName", value: Any) -> Column:
     |                      3|
     +-----------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("array_position", _to_java_column(col), value)
 
 
@@ -14183,6 +14298,8 @@ def array_remove(col: "ColumnOrName", element: Any) -> Column:
     |                   []|
     +---------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("array_remove", _to_java_column(col), element)
 
 
@@ -15326,6 +15443,8 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column:
     ...                   get_json_object(df.jstring, '$.f2').alias("c1") ).collect()
     [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("get_json_object", _to_java_column(col), path)
 
 
@@ -15357,6 +15476,8 @@ def json_tuple(col: "ColumnOrName", *fields: str) -> Column:
     >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect()
     [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)]
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     if len(fields) == 0:
         raise PySparkValueError(
             error_class="CANNOT_BE_EMPTY",
@@ -15464,6 +15585,7 @@ def from_json(
     |[1, 2, 3]|
     +---------+
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     if isinstance(schema, DataType):
         schema = schema.json()
@@ -15497,6 +15619,7 @@ def parse_json(
     >>> df.select(to_json(parse_json(df.json))).collect()
     [Row(to_json(parse_json(json))='{"a":1}')]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("parse_json", _to_java_column(col))
 
@@ -15525,6 +15648,7 @@ def is_variant_null(v: "ColumnOrName") -> Column:
     >>> df.select(is_variant_null(parse_json(df.json)).alias("r")).collect()
     [Row(r=False)]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("is_variant_null", _to_java_column(v))
 
@@ -15560,6 +15684,7 @@ def variant_get(v: "ColumnOrName", path: str, targetType: str) -> Column:
     >>> df.select(variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect()
     [Row(r=None)]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("variant_get", _to_java_column(v), path, targetType)
 
@@ -15597,6 +15722,7 @@ def try_variant_get(v: "ColumnOrName", path: str, targetType: str) -> Column:
     >>> df.select(try_variant_get(parse_json(df.json), "$.a", "binary").alias("r")).collect()
     [Row(r=None)]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("try_variant_get", _to_java_column(v), path, targetType)
 
@@ -15624,6 +15750,7 @@ def schema_of_variant(v: "ColumnOrName") -> Column:
     >>> df.select(schema_of_variant(parse_json(df.json)).alias("r")).collect()
     [Row(r='STRUCT<a: BIGINT>')]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("schema_of_variant", _to_java_column(v))
 
@@ -15651,6 +15778,7 @@ def schema_of_variant_agg(v: "ColumnOrName") -> Column:
     >>> df.select(schema_of_variant_agg(parse_json(df.json)).alias("r")).collect()
     [Row(r='STRUCT<a: BIGINT>')]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("schema_of_variant_agg", _to_java_column(v))
 
@@ -15745,6 +15873,7 @@ def to_json(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Co
     |["Alice","Bob"]|
     +---------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("to_json", _to_java_column(col), _options_to_str(options))
 
@@ -15787,6 +15916,8 @@ def schema_of_json(json: Union[Column, str], options: Optional[Dict[str, str]] =
     >>> df.select(schema.alias("json")).collect()
     [Row(json='STRUCT<a: BIGINT>')]
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if isinstance(json, str):
         col = _create_column_from_literal(json)
     elif isinstance(json, Column):
@@ -15924,6 +16055,7 @@ def from_xml(
     >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect()
     [Row(xml=Row(a=[1, 2]))]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     if isinstance(schema, StructType):
         schema = schema.json()
@@ -16002,6 +16134,8 @@ def schema_of_xml(xml: Union[Column, str], options: Optional[Dict[str, str]] = N
     ... ).collect()
     [Row(xml='STRUCT<values: STRUCT<value: ARRAY<BIGINT>>>')]
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if isinstance(xml, str):
         col = _create_column_from_literal(xml)
     elif isinstance(xml, Column):
@@ -16047,6 +16181,7 @@ def to_xml(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Col
     >>> df.select(to_xml(df.value, {'rowTag':'person'}).alias("xml")).collect()
     [Row(xml='<person>\\n    <age>2</age>\\n    <name>Alice</name>\\n</person>')]
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("to_xml", _to_java_column(col), _options_to_str(options))
 
@@ -16123,6 +16258,8 @@ def schema_of_csv(csv: Union[Column, str], options: Optional[Dict[str, str]] = N
     |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>|
     +-------------------------------------------+
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
+
     if isinstance(csv, str):
         col = _create_column_from_literal(csv)
     elif isinstance(csv, Column):
@@ -16223,6 +16360,7 @@ def to_csv(col: "ColumnOrName", options: Optional[Dict[str, str]] = None) -> Col
     | 2,Alice,true|
     +-------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
 
     return _invoke_function("to_csv", _to_java_column(col), _options_to_str(options))
 
@@ -16630,6 +16768,8 @@ def sort_array(col: "ColumnOrName", asc: bool = True) -> Column:
     |    [NULL, NULL, NULL]|
     +----------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("sort_array", _to_java_column(col), asc)
 
 
@@ -16923,6 +17063,8 @@ def map_contains_key(col: "ColumnOrName", value: Any) -> Column:
     |                     false|
     +--------------------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("map_contains_key", _to_java_column(col), value)
 
 
@@ -17653,6 +17795,7 @@ def from_csv(
     |      {1, 2, 3}|
     +---------------+
     """
+    from pyspark.sql.classic.column import _create_column_from_literal, _to_java_column
 
     _get_active_spark_context()
     if isinstance(schema, str):
@@ -17681,6 +17824,7 @@ def _unresolved_named_lambda_variable(*name_parts: Any) -> Column:
     name_parts : str
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_seq
 
     sc = _get_active_spark_context()
     name_parts_seq = _to_seq(sc, name_parts)
@@ -17731,6 +17875,7 @@ def _create_lambda(f: Callable) -> Callable:
             - (Column, Column, Column) -> Column: ...
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_seq
 
     parameters = _get_lambda_parameters(f)
 
@@ -17776,6 +17921,7 @@ def _invoke_higher_order_function(
     :return: a Column
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_java_column
 
     sc = _get_active_spark_context()
     expressions = cast(JVMView, sc._jvm).org.apache.spark.sql.catalyst.expressions
@@ -19414,6 +19560,8 @@ def call_udf(udfName: str, *cols: "ColumnOrName") -> Column:
     |         cc|
     +-----------+
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function("call_udf", udfName, _to_seq(sc, cols, _to_java_column))
 
@@ -19483,6 +19631,8 @@ def call_function(funcName: str, *cols: "ColumnOrName") -> Column:
     |                               102.0|
     +------------------------------------+
     """
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     sc = _get_active_spark_context()
     return _invoke_function("call_function", funcName, _to_seq(sc, cols, _to_java_column))
 
@@ -19498,6 +19648,8 @@ def unwrap_udt(col: "ColumnOrName") -> Column:
     -----
     Supports Spark Connect.
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("unwrap_udt", _to_java_column(col))
 
 
@@ -19649,6 +19801,8 @@ def hll_sketch_estimate(col: "ColumnOrName") -> Column:
     |           3|
     +------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     return _invoke_function("hll_sketch_estimate", _to_java_column(col))
 
 
@@ -19687,6 +19841,8 @@ def hll_union(
     |           6|
     +------------+
     """
+    from pyspark.sql.classic.column import _to_java_column
+
     if allowDifferentLgConfigK is not None:
         return _invoke_function(
             "hll_union", _to_java_column(col1), _to_java_column(col2), allowDifferentLgConfigK
diff --git a/python/pyspark/sql/functions/partitioning.py b/python/pyspark/sql/functions/partitioning.py
index 59c293577b08..e89901cdbe54 100644
--- a/python/pyspark/sql/functions/partitioning.py
+++ b/python/pyspark/sql/functions/partitioning.py
@@ -26,7 +26,7 @@ from typing import (
 )
 
 from pyspark.errors import PySparkTypeError
-from pyspark.sql.column import Column, _to_java_column, _create_column_from_literal
+from pyspark.sql.column import Column
 from pyspark.sql.functions.builtin import _invoke_function_over_columns, _invoke_function
 from pyspark.sql.utils import (
     try_partitioning_remote_functions as _try_partitioning_remote_functions,
@@ -204,6 +204,8 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column:
     method of the `DataFrameWriterV2`.
 
     """
+    from pyspark.sql.classic.column import _to_java_column, _create_column_from_literal
+
     if not isinstance(numBuckets, (int, Column)):
         raise PySparkTypeError(
             error_class="NOT_COLUMN_OR_INT",
diff --git a/python/pyspark/sql/group.py b/python/pyspark/sql/group.py
index 15934c24b9d4..d26e23bc7160 100644
--- a/python/pyspark/sql/group.py
+++ b/python/pyspark/sql/group.py
@@ -19,7 +19,7 @@ import sys
 
 from typing import Callable, List, Optional, TYPE_CHECKING, overload, Dict, Union, cast, Tuple
 
-from pyspark.sql.column import Column, _to_seq
+from pyspark.sql.column import Column
 from pyspark.sql.session import SparkSession
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.pandas.group_ops import PandasGroupedOpsMixin
@@ -43,6 +43,8 @@ def dfapi(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]:
 
 
 def df_varargs_api(f: Callable[..., DataFrame]) -> Callable[..., DataFrame]:
+    from pyspark.sql.classic.column import _to_seq
+
     def _api(self: "GroupedData", *cols: str) -> DataFrame:
         name = f.__name__
         jdf = getattr(self._jgd, name)(_to_seq(self.session._sc, cols))
@@ -175,6 +177,8 @@ class GroupedData(PandasGroupedOpsMixin):
         |  Bob|           5|
         +-----+------------+
         """
+        from pyspark.sql.classic.column import _to_seq
+
         assert exprs, "exprs should not be empty"
         if len(exprs) == 1 and isinstance(exprs[0], dict):
             jdf = self._jgd.agg(exprs[0])
diff --git a/python/pyspark/sql/observation.py b/python/pyspark/sql/observation.py
index 1dae5086e3dd..4ef4c78ba3c3 100644
--- a/python/pyspark/sql/observation.py
+++ b/python/pyspark/sql/observation.py
@@ -18,7 +18,6 @@ import os
 from typing import Any, Dict, Optional, TYPE_CHECKING
 
 from pyspark.errors import PySparkTypeError, PySparkValueError, PySparkAssertionError
-from pyspark.sql import column
 from pyspark.sql.column import Column
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.utils import is_remote
@@ -116,6 +115,8 @@ class Observation:
         :class:`DataFrame`
             the observed :class:`DataFrame`.
         """
+        from pyspark.sql.classic.column import _to_seq
+
         if self._jo is not None:
             raise PySparkAssertionError(error_class="REUSE_OBSERVATION", message_parameters={})
 
@@ -124,7 +125,9 @@ class Observation:
         cls = self._jvm.org.apache.spark.sql.Observation
         self._jo = cls(self._name) if self._name is not None else cls()
         observed_df = self._jo.on(
-            df._jdf, exprs[0]._jc, column._to_seq(df._sc, [c._jc for c in exprs[1:]])
+            df._jdf,
+            exprs[0]._jc,
+            _to_seq(df._sc, [c._jc for c in exprs[1:]]),
         )
         return DataFrame(observed_df, df.sparkSession)
 
diff --git a/python/pyspark/sql/protobuf/functions.py b/python/pyspark/sql/protobuf/functions.py
index 63871e437571..1e75874e75f9 100644
--- a/python/pyspark/sql/protobuf/functions.py
+++ b/python/pyspark/sql/protobuf/functions.py
@@ -22,7 +22,7 @@ A collections of builtin protobuf functions
 
 from typing import Dict, Optional, TYPE_CHECKING, cast
 
-from pyspark.sql.column import Column, _to_java_column
+from pyspark.sql.column import Column
 from pyspark.sql.utils import get_active_spark_context, try_remote_protobuf_functions
 from pyspark.util import _print_missing_jar
 
@@ -139,6 +139,7 @@ def from_protobuf(
     +------------------+
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_java_column
 
     sc = get_active_spark_context()
     try:
@@ -260,6 +261,7 @@ def to_protobuf(
     +----------------------------+
     """
     from py4j.java_gateway import JVMView
+    from pyspark.sql.classic.column import _to_java_column
 
     sc = get_active_spark_context()
     try:
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 26fe8c5e6fa2..0ee235c8bf02 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -18,7 +18,7 @@ import sys
 from typing import cast, overload, Dict, Iterable, List, Optional, Tuple, TYPE_CHECKING, Union
 
 from pyspark.util import is_remote_only
-from pyspark.sql.column import _to_seq, _to_java_column, Column
+from pyspark.sql.column import Column
 from pyspark.sql.types import StructType
 from pyspark.sql import utils
 from pyspark.sql.utils import to_str
@@ -619,6 +619,8 @@ class DataFrameReader(OptionUtils):
         |  Tom|  20|  NULL|
         +-----+----+------+
         """
+        from pyspark.sql.classic.column import _to_seq
+
         mergeSchema = options.get("mergeSchema", None)
         pathGlobFilter = options.get("pathGlobFilter", None)
         modifiedBefore = options.get("modifiedBefore", None)
@@ -1042,6 +1044,8 @@ class DataFrameReader(OptionUtils):
         |100|Hyukjin Kwon|
         +---+------------+
         """
+        from pyspark.sql.classic.column import _to_seq
+
         self._set_opts(
             mergeSchema=mergeSchema,
             pathGlobFilter=pathGlobFilter,
@@ -1440,6 +1444,8 @@ class DataFrameWriter(OptionUtils):
         |100|
         +---+
         """
+        from pyspark.sql.classic.column import _to_seq
+
         if len(cols) == 1 and isinstance(cols[0], (list, tuple)):
             cols = cols[0]  # type: ignore[assignment]
         self._jwrite = self._jwrite.partitionBy(
@@ -1503,6 +1509,8 @@ class DataFrameWriter(OptionUtils):
         +---+------------+
         >>> _ = spark.sql("DROP TABLE bucketed_table")
         """
+        from pyspark.sql.classic.column import _to_seq
+
         if not isinstance(numBuckets, int):
             raise PySparkTypeError(
                 error_class="NOT_INT",
@@ -1594,6 +1602,8 @@ class DataFrameWriter(OptionUtils):
         +---+------------+
         >>> _ = spark.sql("DROP TABLE sorted_bucketed_table")
         """
+        from pyspark.sql.classic.column import _to_seq
+
         if isinstance(col, (list, tuple)):
             if cols:
                 raise PySparkValueError(
@@ -2380,6 +2390,8 @@ class DataFrameWriterV2:
 
         .. versionadded: 3.1.0
         """
+        from pyspark.sql.classic.column import _to_seq, _to_java_column
+
         col = _to_java_column(col)
         cols = _to_seq(self._spark._sc, [_to_java_column(c) for c in cols])
         self._jwriter.partitionedBy(col, cols)
@@ -2435,6 +2447,8 @@ class DataFrameWriterV2:
 
         .. versionadded: 3.1.0
         """
+        from pyspark.sql.classic.column import _to_java_column
+
         condition = _to_java_column(condition)
         self._jwriter.overwrite(condition)
 
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index e0c2c815c827..1a01982e756d 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -40,7 +40,6 @@ from typing import (
 
 from pyspark.conf import SparkConf
 from pyspark.util import is_remote_only
-from pyspark.sql.column import _to_java_column
 from pyspark.sql.conf import RuntimeConfig
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import lit
@@ -1724,6 +1723,7 @@ class SparkSession(SparkConversionMixin):
         |  3|  6|  1|
         +---+---+---+
         """
+        from pyspark.sql.classic.column import _to_java_column
 
         formatter = SQLStringFormatter(self)
         if len(kwargs) > 0:
diff --git a/python/pyspark/sql/streaming/readwriter.py b/python/pyspark/sql/streaming/readwriter.py
index 58901f34cfc9..c2b75dd8f167 100644
--- a/python/pyspark/sql/streaming/readwriter.py
+++ b/python/pyspark/sql/streaming/readwriter.py
@@ -19,7 +19,6 @@ import sys
 from collections.abc import Iterator
 from typing import cast, overload, Any, Callable, List, Optional, TYPE_CHECKING, Union
 
-from pyspark.sql.column import _to_seq
 from pyspark.sql.readwriter import OptionUtils, to_str
 from pyspark.sql.streaming.query import StreamingQuery
 from pyspark.sql.types import Row, StructType
@@ -1117,6 +1116,8 @@ class DataStreamWriter:
         +...---------+-----+
         ...
         """
+        from pyspark.sql.classic.column import _to_seq
+
         if len(cols) == 1 and isinstance(cols[0], (list, tuple)):
             cols = cols[0]
         self._jwrite = self._jwrite.partitionBy(_to_seq(self._spark._sc, cols))
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
index ec413d048d8e..9edd585da6a0 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
@@ -1334,7 +1334,7 @@ class ScalarPandasUDFTestsMixin:
             return x + 1
 
         def f2(x):
-            assert type(x) == col_type
+            assert isinstance(x, col_type)
             return x + 10
 
         @pandas_udf("int")
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index e51ae69814bd..92921685d58e 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -42,7 +42,7 @@ class ColumnTestsMixin:
 
     def test_validate_column_types(self):
         from pyspark.sql.functions import udf, to_json
-        from pyspark.sql.column import _to_java_column
+        from pyspark.sql.classic.column import _to_java_column
 
         self.assertTrue("Column" in _to_java_column("a").getClass().toString())
         self.assertTrue("Column" in _to_java_column("a").getClass().toString())
@@ -248,6 +248,13 @@ class ColumnTestsMixin:
             message_parameters={"arg_name": "window", "arg_type": "int"},
         )
 
+    def test_union_classmethod_usage(self):
+        df = self.spark.range(1)
+        self.assertEqual(df.select(Column.eqNullSafe(df.id, df.id)).first()[0], True)
+
+    def test_isinstance_dataframe(self):
+        self.assertIsInstance(self.spark.range(1).id, Column)
+
 
 class ColumnTests(ColumnTestsMixin, ReusedSQLTestCase):
     pass
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 0d0fc9042e62..7be59a28e479 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -27,7 +27,7 @@ from typing import Callable, Any, TYPE_CHECKING, Optional, cast, Union
 
 
 from pyspark.util import PythonEvalType
-from pyspark.sql.column import Column, _to_java_expr, _to_seq
+from pyspark.sql.column import Column
 from pyspark.sql.types import (
     DataType,
     StringType,
@@ -395,6 +395,8 @@ class UserDefinedFunction:
         return judf
 
     def __call__(self, *args: "ColumnOrName", **kwargs: "ColumnOrName") -> Column:
+        from pyspark.sql.classic.column import _to_java_expr, _to_seq
+
         sc = get_active_spark_context()
 
         assert sc._jvm is not None
diff --git a/python/pyspark/sql/udtf.py b/python/pyspark/sql/udtf.py
index 801ecc605e50..f56088020223 100644
--- a/python/pyspark/sql/udtf.py
+++ b/python/pyspark/sql/udtf.py
@@ -26,7 +26,6 @@ from typing import Any, Type, TYPE_CHECKING, Optional, Sequence, Union
 
 from pyspark.errors import PySparkAttributeError, PySparkPicklingError, PySparkTypeError
 from pyspark.util import PythonEvalType
-from pyspark.sql.column import _to_java_column, _to_java_expr, _to_seq
 from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
 from pyspark.sql.types import DataType, StructType, _parse_datatype_string
 from pyspark.sql.udf import _wrap_function
@@ -374,6 +373,8 @@ class UserDefinedTableFunction:
         return judtf
 
     def __call__(self, *args: "ColumnOrName", **kwargs: "ColumnOrName") -> "DataFrame":
+        from pyspark.sql.classic.column import _to_java_column, _to_java_expr, _to_seq
+
         from pyspark.sql import DataFrame, SparkSession
 
         spark = SparkSession._getActiveSessionOrCreate()
diff --git a/python/pyspark/sql/utils.py b/python/pyspark/sql/utils.py
index e830806eee64..de7d58361c04 100644
--- a/python/pyspark/sql/utils.py
+++ b/python/pyspark/sql/utils.py
@@ -329,6 +329,33 @@ def dispatch_df_method(f: FuncT) -> FuncT:
     return cast(FuncT, wrapped)
 
 
+def dispatch_col_method(f: FuncT) -> FuncT:
+    """
+    For the usecases of direct Column.method(col, ...), it checks if self
+    is a Connect DataFrame or Classic DataFrame, and dispatches.
+    """
+
+    @functools.wraps(f)
+    def wrapped(*args: Any, **kwargs: Any) -> Any:
+        if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
+            from pyspark.sql.connect.column import Column as ConnectColumn
+
+            if isinstance(args[0], ConnectColumn):
+                return getattr(ConnectColumn, f.__name__)(*args, **kwargs)
+        else:
+            from pyspark.sql.classic.column import Column as ClassicColumn
+
+            if isinstance(args[0], ClassicColumn):
+                return getattr(ClassicColumn, f.__name__)(*args, **kwargs)
+
+        raise PySparkNotImplementedError(
+            error_class="NOT_IMPLEMENTED",
+            message_parameters={"feature": f"DataFrame.{f.__name__}"},
+        )
+
+    return cast(FuncT, wrapped)
+
+
 def pyspark_column_op(
     func_name: str, left: "IndexOpsLike", right: Any, fillna: Any = None
 ) -> Union["SeriesOrIndex", None]:
@@ -359,7 +386,7 @@ def get_column_class() -> Type["Column"]:
     if is_remote():
         from pyspark.sql.connect.column import Column as ConnectColumn
 
-        return ConnectColumn  # type: ignore[return-value]
+        return ConnectColumn
     else:
         return PySparkColumn
 
diff --git a/python/pyspark/sql/window.py b/python/pyspark/sql/window.py
index 42d50dc1b3bd..93ed5c172ffe 100644
--- a/python/pyspark/sql/window.py
+++ b/python/pyspark/sql/window.py
@@ -17,7 +17,6 @@
 import sys
 from typing import cast, Iterable, List, Tuple, TYPE_CHECKING, Union
 
-from pyspark.sql.column import _to_seq, _to_java_column
 from pyspark.sql.utils import (
     try_remote_window,
     try_remote_windowspec,
@@ -32,6 +31,8 @@ __all__ = ["Window", "WindowSpec"]
 
 
 def _to_java_cols(cols: Tuple[Union["ColumnOrName", List["ColumnOrName_"]], ...]) -> "JavaObject":
+    from pyspark.sql.classic.column import _to_seq, _to_java_column
+
     if len(cols) == 1 and isinstance(cols[0], list):
         cols = cols[0]  # type: ignore[assignment]
     sc = get_active_spark_context()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org