You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/02/07 05:31:23 UTC

[spark] branch branch-3.4 updated: [SPARK-42363][CONNECT] Remove SparkSession.register_udf

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.4 by this push:
     new 47839097a7e [SPARK-42363][CONNECT] Remove SparkSession.register_udf
47839097a7e is described below

commit 47839097a7e49da961c10c6399b20e79fe2dd264
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon Feb 6 21:31:02 2023 -0800

    [SPARK-42363][CONNECT] Remove SparkSession.register_udf
    
    ### What changes were proposed in this pull request?
    
    This PR proposes to remove `SparkSession.register_udf` and `pyspark.sql.connect.function_builder`.
    
    ### Why are the changes needed?
    
    - For API parity w/ regular PySpark.
    - They have to be implemented via `SparkSession.udf` (SPARK-41843).
    - For non-registered Python and Pandas UDFs, they are implemented at SPARK-42124 and SPARK-42125.
    - We implemented all functions so we do not need `pyspark.sql.connect.function_builder` workaround anymore.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No to end users.
    
    Yee to the dev because it removes an API `SparkSession.register_udf` and `pyspark.sql.connect.function_builder`.
    
    ### How was this patch tested?
    
    CI in this PR should verify them
    
    Closes #39916 from HyukjinKwon/remove-register-udf.
    
    Authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit 9fbdb2dc095131ebd5be41f61e9481574a40be4d)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../src/main/protobuf/spark/connect/commands.proto |  40 -------
 .../sql/connect/planner/SparkConnectPlanner.scala  |  39 +-----
 python/pyspark/sql/connect/client.py               |  19 ---
 python/pyspark/sql/connect/function_builder.py     | 109 -----------------
 python/pyspark/sql/connect/proto/commands_pb2.py   |  59 ++++------
 python/pyspark/sql/connect/proto/commands_pb2.pyi  | 131 +--------------------
 python/pyspark/sql/connect/session.py              |   3 -
 .../sql/tests/connect/test_connect_basic.py        |  10 --
 .../pyspark/sql/tests/connect/test_connect_plan.py |  10 --
 python/pyspark/testing/connectutils.py             |   2 -
 10 files changed, 22 insertions(+), 400 deletions(-)

diff --git a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
index da81da844e7..05c91d2c992 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
@@ -31,7 +31,6 @@ option java_package = "org.apache.spark.connect.proto";
 // produce a relational result.
 message Command {
   oneof command_type {
-    CreateScalarFunction create_function = 1;
     WriteOperation write_operation = 2;
     CreateDataFrameViewCommand create_dataframe_view = 3;
     WriteOperationV2 write_operation_v2 = 4;
@@ -43,45 +42,6 @@ message Command {
   }
 }
 
-// Simple message that is used to create a scalar function based on the provided function body.
-//
-// This message is used to register for example a Python UDF in the session catalog by providing
-// the serialized method body.
-//
-// TODO(SPARK-40532) It is required to add the interpreter / language version to the command
-//   parameters.
-message CreateScalarFunction {
-  // (Required) Fully qualified name of the function including the catalog / schema names.
-  repeated string parts = 1;
-
-  // (Required) the function language.
-  FunctionLanguage language = 2;
-
-  // (Required) if this is a temporary function.
-  bool temporary = 3;
-
-  // (Optional) A list of argument types. Can be empty when the function does not take an argument.
-  repeated DataType argument_types = 4;
-
-  // (Required) the return type of the function.
-  DataType return_type = 5;
-
-  // How the function body is defined:
-  oneof function_definition {
-    // As a raw string serialized:
-    bytes serialized_function = 6;
-    // As a code literal
-    string literal_string = 7;
-  }
-
-  enum FunctionLanguage {
-    FUNCTION_LANGUAGE_UNSPECIFIED = 0;
-    FUNCTION_LANGUAGE_SQL = 1;
-    FUNCTION_LANGUAGE_PYTHON = 2;
-    FUNCTION_LANGUAGE_SCALA = 3;
-  }
-}
-
 // A command that can create DataFrame global temp view or local temp view.
 message CreateDataFrameViewCommand {
   // (Required) The relation that this view will be built on.
diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 08df2274840..28071acf8a4 100644
--- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -24,7 +24,7 @@ import com.google.common.collect.{Lists, Maps}
 import com.google.protobuf.{Any => ProtoAny}
 
 import org.apache.spark.TaskContext
-import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction}
+import org.apache.spark.api.python.SimplePythonFunction
 import org.apache.spark.connect.proto
 import org.apache.spark.sql.{Column, Dataset, Encoders, SparkSession}
 import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier}
@@ -44,7 +44,6 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.QueryExecution
 import org.apache.spark.sql.execution.arrow.ArrowConverters
 import org.apache.spark.sql.execution.command.CreateViewCommand
-import org.apache.spark.sql.execution.python.UserDefinedPythonFunction
 import org.apache.spark.sql.functions.{col, expr}
 import org.apache.spark.sql.internal.CatalogImpl
 import org.apache.spark.sql.types._
@@ -1403,8 +1402,6 @@ class SparkConnectPlanner(val session: SparkSession) {
 
   def process(command: proto.Command): Unit = {
     command.getCommandTypeCase match {
-      case proto.Command.CommandTypeCase.CREATE_FUNCTION =>
-        handleCreateScalarFunction(command.getCreateFunction)
       case proto.Command.CommandTypeCase.WRITE_OPERATION =>
         handleWriteOperation(command.getWriteOperation)
       case proto.Command.CommandTypeCase.CREATE_DATAFRAME_VIEW =>
@@ -1429,40 +1426,6 @@ class SparkConnectPlanner(val session: SparkSession) {
       .getOrElse(throw InvalidPlanInput("No handler found for extension"))
   }
 
-  /**
-   * This is a helper function that registers a new Python function in the SparkSession.
-   *
-   * Right now this function is very rudimentary and bare-bones just to showcase how it is
-   * possible to remotely serialize a Python function and execute it on the Spark cluster. If the
-   * Python version on the client and server diverge, the execution of the function that is
-   * serialized will most likely fail.
-   *
-   * @param cf
-   */
-  private def handleCreateScalarFunction(cf: proto.CreateScalarFunction): Unit = {
-    val function = SimplePythonFunction(
-      cf.getSerializedFunction.toByteArray,
-      // Empty environment variables
-      Maps.newHashMap(),
-      // No imported Python libraries
-      Lists.newArrayList(),
-      pythonExec,
-      "3.9", // TODO(SPARK-40532) This needs to be an actual Python version.
-      // Empty broadcast variables
-      Lists.newArrayList(),
-      // Null accumulator
-      null)
-
-    val udf = UserDefinedPythonFunction(
-      cf.getPartsList.asScala.head,
-      function,
-      StringType,
-      PythonEvalType.SQL_BATCHED_UDF,
-      udfDeterministic = false)
-
-    session.udf.registerPython(cf.getPartsList.asScala.head, udf)
-  }
-
   private def handleCreateViewCommand(createView: proto.CreateDataFrameViewCommand): Unit = {
     val viewType = if (createView.getIsGlobal) GlobalTempView else LocalTempView
 
diff --git a/python/pyspark/sql/connect/client.py b/python/pyspark/sql/connect/client.py
index 133f58ea21a..36f1328b1bd 100644
--- a/python/pyspark/sql/connect/client.py
+++ b/python/pyspark/sql/connect/client.py
@@ -58,8 +58,6 @@ from google.rpc import error_details_pb2
 import pyspark.sql.connect.proto as pb2
 import pyspark.sql.connect.proto.base_pb2_grpc as grpc_lib
 import pyspark.sql.connect.types as types
-import pyspark.sql.types
-from pyspark import cloudpickle
 from pyspark.errors import (
     SparkConnectException,
     SparkConnectGrpcException,
@@ -429,23 +427,6 @@ class SparkConnectClient(object):
         self._stub = grpc_lib.SparkConnectServiceStub(self._channel)
         # Configure logging for the SparkConnect client.
 
-    def register_udf(
-        self, function: Any, return_type: Union[str, pyspark.sql.types.DataType]
-    ) -> str:
-        """Create a temporary UDF in the session catalog on the other side. We generate a
-        temporary name for it."""
-        name = f"fun_{uuid.uuid4().hex}"
-        fun = pb2.CreateScalarFunction()
-        fun.parts.append(name)
-        logger.info(f"Registering UDF: {self._proto_to_string(fun)}")
-        fun.serialized_function = cloudpickle.dumps((function, return_type))
-
-        req = self._execute_plan_request_with_metadata()
-        req.plan.command.create_function.CopyFrom(fun)
-
-        self._execute(req)
-        return name
-
     def _build_metrics(self, metrics: "pb2.ExecutePlanResponse.Metrics") -> List[PlanMetrics]:
         return [
             PlanMetrics(
diff --git a/python/pyspark/sql/connect/function_builder.py b/python/pyspark/sql/connect/function_builder.py
deleted file mode 100644
index f3416e78286..00000000000
--- a/python/pyspark/sql/connect/function_builder.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-from pyspark.sql.connect import check_dependencies
-
-check_dependencies(__name__, __file__)
-
-import functools
-from typing import TYPE_CHECKING, Optional, Any, Iterable, Union
-
-import pyspark.sql.types
-
-import pyspark.sql.connect.proto as proto
-from pyspark.sql.connect.column import Column
-from pyspark.sql.connect.expressions import Expression
-from pyspark.sql.connect.functions import _invoke_function_over_columns
-
-
-if TYPE_CHECKING:
-    from pyspark.sql.connect._typing import (
-        ColumnOrName,
-        UserDefinedFunctionCallable,
-    )
-    from pyspark.sql.connect.client import SparkConnectClient
-
-
-class UserDefinedFunction(Expression):
-    """A user defied function is an expression that has a reference to the actual
-    Python callable attached. During plan generation, the client sends a command to
-    the server to register the UDF before execution. The expression object can be
-    reused and is not attached to a specific execution. If the internal name of
-    the temporary function is set, it is assumed that the registration has already
-    happened."""
-
-    def __init__(
-        self,
-        func: Any,
-        return_type: Union[str, pyspark.sql.types.DataType] = pyspark.sql.types.StringType(),
-        args: Optional[Iterable[Any]] = None,
-    ) -> None:
-        super().__init__()
-
-        self._func_ref = func
-        self._return_type = return_type
-        if args is not None:
-            self._args = list(args)
-        else:
-            self._args = []
-        self._func_name = None
-
-    def to_plan(self, session: "SparkConnectClient") -> proto.Expression:
-        if session is None:
-            raise Exception("CAnnot create UDF without remote Session.")
-        # Needs to materialize the UDF to the server
-        # Only do this once per session
-        func_name = session.register_udf(self._func_ref, self._return_type)
-        # Func name is used for the actual reference
-        return _invoke_function_over_columns(func_name, *self._args).to_plan(session)
-
-    def __str__(self) -> str:
-        return f"UserDefinedFunction({self._func_name})"
-
-
-def _create_udf(
-    function: Any, return_type: Union[str, pyspark.sql.types.DataType]
-) -> "UserDefinedFunctionCallable":
-    def wrapper(*cols: "ColumnOrName") -> "Column":
-        return Column(UserDefinedFunction(func=function, return_type=return_type, args=cols))
-
-    return wrapper
-
-
-def udf(
-    function: Any, return_type: pyspark.sql.types.DataType = pyspark.sql.types.StringType()
-) -> Any:
-    """
-    Returns a callable that represents the column once arguments are applied
-
-    Parameters
-    ----------
-    function
-    return_type
-
-    Returns
-    -------
-
-    """
-    # This is when @udf / @udf(DataType()) is used
-    if function is None or isinstance(function, (str, pyspark.sql.types.DataType)):
-        actual_return_type = function or return_type
-        # Overload with
-        if actual_return_type is None:
-            actual_return_type = pyspark.sql.types.StringType()
-        return functools.partial(_create_udf, return_type=actual_return_type)
-    else:
-        return _create_udf(function, return_type)
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py
index 482e7dd4fcc..c9a51b04bb6 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.py
+++ b/python/pyspark/sql/connect/proto/commands_pb2.py
@@ -36,12 +36,11 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\x9c\x03\n\x07\x43ommand\x12N\n\x0f\x63reate_function\x18\x01 \x01(\x0b\x32#.spark.connect.CreateScalarFunctionH\x00R\x0e\x63reateFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x0 [...]
+    b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\xcc\x02\n\x07\x43ommand\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\ [...]
 )
 
 
 _COMMAND = DESCRIPTOR.message_types_by_name["Command"]
-_CREATESCALARFUNCTION = DESCRIPTOR.message_types_by_name["CreateScalarFunction"]
 _CREATEDATAFRAMEVIEWCOMMAND = DESCRIPTOR.message_types_by_name["CreateDataFrameViewCommand"]
 _WRITEOPERATION = DESCRIPTOR.message_types_by_name["WriteOperation"]
 _WRITEOPERATION_OPTIONSENTRY = _WRITEOPERATION.nested_types_by_name["OptionsEntry"]
@@ -51,9 +50,6 @@ _WRITEOPERATIONV2_OPTIONSENTRY = _WRITEOPERATIONV2.nested_types_by_name["Options
 _WRITEOPERATIONV2_TABLEPROPERTIESENTRY = _WRITEOPERATIONV2.nested_types_by_name[
     "TablePropertiesEntry"
 ]
-_CREATESCALARFUNCTION_FUNCTIONLANGUAGE = _CREATESCALARFUNCTION.enum_types_by_name[
-    "FunctionLanguage"
-]
 _WRITEOPERATION_SAVEMODE = _WRITEOPERATION.enum_types_by_name["SaveMode"]
 _WRITEOPERATIONV2_MODE = _WRITEOPERATIONV2.enum_types_by_name["Mode"]
 Command = _reflection.GeneratedProtocolMessageType(
@@ -67,17 +63,6 @@ Command = _reflection.GeneratedProtocolMessageType(
 )
 _sym_db.RegisterMessage(Command)
 
-CreateScalarFunction = _reflection.GeneratedProtocolMessageType(
-    "CreateScalarFunction",
-    (_message.Message,),
-    {
-        "DESCRIPTOR": _CREATESCALARFUNCTION,
-        "__module__": "spark.connect.commands_pb2"
-        # @@protoc_insertion_point(class_scope:spark.connect.CreateScalarFunction)
-    },
-)
-_sym_db.RegisterMessage(CreateScalarFunction)
-
 CreateDataFrameViewCommand = _reflection.GeneratedProtocolMessageType(
     "CreateDataFrameViewCommand",
     (_message.Message,),
@@ -162,27 +147,23 @@ if _descriptor._USE_C_DESCRIPTORS == False:
     _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._options = None
     _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_options = b"8\001"
     _COMMAND._serialized_start = 166
-    _COMMAND._serialized_end = 578
-    _CREATESCALARFUNCTION._serialized_start = 581
-    _CREATESCALARFUNCTION._serialized_end = 1116
-    _CREATESCALARFUNCTION_FUNCTIONLANGUAGE._serialized_start = 954
-    _CREATESCALARFUNCTION_FUNCTIONLANGUAGE._serialized_end = 1093
-    _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 1119
-    _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 1269
-    _WRITEOPERATION._serialized_start = 1272
-    _WRITEOPERATION._serialized_end = 2014
-    _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1710
-    _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1768
-    _WRITEOPERATION_BUCKETBY._serialized_start = 1770
-    _WRITEOPERATION_BUCKETBY._serialized_end = 1861
-    _WRITEOPERATION_SAVEMODE._serialized_start = 1864
-    _WRITEOPERATION_SAVEMODE._serialized_end = 2001
-    _WRITEOPERATIONV2._serialized_start = 2017
-    _WRITEOPERATIONV2._serialized_end = 2812
-    _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1710
-    _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1768
-    _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 2584
-    _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 2650
-    _WRITEOPERATIONV2_MODE._serialized_start = 2653
-    _WRITEOPERATIONV2_MODE._serialized_end = 2812
+    _COMMAND._serialized_end = 498
+    _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 501
+    _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 651
+    _WRITEOPERATION._serialized_start = 654
+    _WRITEOPERATION._serialized_end = 1396
+    _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1092
+    _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1150
+    _WRITEOPERATION_BUCKETBY._serialized_start = 1152
+    _WRITEOPERATION_BUCKETBY._serialized_end = 1243
+    _WRITEOPERATION_SAVEMODE._serialized_start = 1246
+    _WRITEOPERATION_SAVEMODE._serialized_end = 1383
+    _WRITEOPERATIONV2._serialized_start = 1399
+    _WRITEOPERATIONV2._serialized_end = 2194
+    _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1092
+    _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1150
+    _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 1966
+    _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 2032
+    _WRITEOPERATIONV2_MODE._serialized_start = 2035
+    _WRITEOPERATIONV2_MODE._serialized_end = 2194
 # @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi
index 2447be6b467..8a1f2ffb122 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi
@@ -42,7 +42,6 @@ import google.protobuf.internal.enum_type_wrapper
 import google.protobuf.message
 import pyspark.sql.connect.proto.expressions_pb2
 import pyspark.sql.connect.proto.relations_pb2
-import pyspark.sql.connect.proto.types_pb2
 import sys
 import typing
 
@@ -60,14 +59,11 @@ class Command(google.protobuf.message.Message):
 
     DESCRIPTOR: google.protobuf.descriptor.Descriptor
 
-    CREATE_FUNCTION_FIELD_NUMBER: builtins.int
     WRITE_OPERATION_FIELD_NUMBER: builtins.int
     CREATE_DATAFRAME_VIEW_FIELD_NUMBER: builtins.int
     WRITE_OPERATION_V2_FIELD_NUMBER: builtins.int
     EXTENSION_FIELD_NUMBER: builtins.int
     @property
-    def create_function(self) -> global___CreateScalarFunction: ...
-    @property
     def write_operation(self) -> global___WriteOperation: ...
     @property
     def create_dataframe_view(self) -> global___CreateDataFrameViewCommand: ...
@@ -81,7 +77,6 @@ class Command(google.protobuf.message.Message):
     def __init__(
         self,
         *,
-        create_function: global___CreateScalarFunction | None = ...,
         write_operation: global___WriteOperation | None = ...,
         create_dataframe_view: global___CreateDataFrameViewCommand | None = ...,
         write_operation_v2: global___WriteOperationV2 | None = ...,
@@ -94,8 +89,6 @@ class Command(google.protobuf.message.Message):
             b"command_type",
             "create_dataframe_view",
             b"create_dataframe_view",
-            "create_function",
-            b"create_function",
             "extension",
             b"extension",
             "write_operation",
@@ -111,8 +104,6 @@ class Command(google.protobuf.message.Message):
             b"command_type",
             "create_dataframe_view",
             b"create_dataframe_view",
-            "create_function",
-            b"create_function",
             "extension",
             b"extension",
             "write_operation",
@@ -124,131 +115,11 @@ class Command(google.protobuf.message.Message):
     def WhichOneof(
         self, oneof_group: typing_extensions.Literal["command_type", b"command_type"]
     ) -> typing_extensions.Literal[
-        "create_function",
-        "write_operation",
-        "create_dataframe_view",
-        "write_operation_v2",
-        "extension",
+        "write_operation", "create_dataframe_view", "write_operation_v2", "extension"
     ] | None: ...
 
 global___Command = Command
 
-class CreateScalarFunction(google.protobuf.message.Message):
-    """Simple message that is used to create a scalar function based on the provided function body.
-
-    This message is used to register for example a Python UDF in the session catalog by providing
-    the serialized method body.
-
-    TODO(SPARK-40532) It is required to add the interpreter / language version to the command
-      parameters.
-    """
-
-    DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-    class _FunctionLanguage:
-        ValueType = typing.NewType("ValueType", builtins.int)
-        V: typing_extensions.TypeAlias = ValueType
-
-    class _FunctionLanguageEnumTypeWrapper(
-        google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[
-            CreateScalarFunction._FunctionLanguage.ValueType
-        ],
-        builtins.type,
-    ):  # noqa: F821
-        DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
-        FUNCTION_LANGUAGE_UNSPECIFIED: CreateScalarFunction._FunctionLanguage.ValueType  # 0
-        FUNCTION_LANGUAGE_SQL: CreateScalarFunction._FunctionLanguage.ValueType  # 1
-        FUNCTION_LANGUAGE_PYTHON: CreateScalarFunction._FunctionLanguage.ValueType  # 2
-        FUNCTION_LANGUAGE_SCALA: CreateScalarFunction._FunctionLanguage.ValueType  # 3
-
-    class FunctionLanguage(_FunctionLanguage, metaclass=_FunctionLanguageEnumTypeWrapper): ...
-    FUNCTION_LANGUAGE_UNSPECIFIED: CreateScalarFunction.FunctionLanguage.ValueType  # 0
-    FUNCTION_LANGUAGE_SQL: CreateScalarFunction.FunctionLanguage.ValueType  # 1
-    FUNCTION_LANGUAGE_PYTHON: CreateScalarFunction.FunctionLanguage.ValueType  # 2
-    FUNCTION_LANGUAGE_SCALA: CreateScalarFunction.FunctionLanguage.ValueType  # 3
-
-    PARTS_FIELD_NUMBER: builtins.int
-    LANGUAGE_FIELD_NUMBER: builtins.int
-    TEMPORARY_FIELD_NUMBER: builtins.int
-    ARGUMENT_TYPES_FIELD_NUMBER: builtins.int
-    RETURN_TYPE_FIELD_NUMBER: builtins.int
-    SERIALIZED_FUNCTION_FIELD_NUMBER: builtins.int
-    LITERAL_STRING_FIELD_NUMBER: builtins.int
-    @property
-    def parts(
-        self,
-    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
-        """(Required) Fully qualified name of the function including the catalog / schema names."""
-    language: global___CreateScalarFunction.FunctionLanguage.ValueType
-    """(Required) the function language."""
-    temporary: builtins.bool
-    """(Required) if this is a temporary function."""
-    @property
-    def argument_types(
-        self,
-    ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
-        pyspark.sql.connect.proto.types_pb2.DataType
-    ]:
-        """(Optional) A list of argument types. Can be empty when the function does not take an argument."""
-    @property
-    def return_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
-        """(Required) the return type of the function."""
-    serialized_function: builtins.bytes
-    """As a raw string serialized:"""
-    literal_string: builtins.str
-    """As a code literal"""
-    def __init__(
-        self,
-        *,
-        parts: collections.abc.Iterable[builtins.str] | None = ...,
-        language: global___CreateScalarFunction.FunctionLanguage.ValueType = ...,
-        temporary: builtins.bool = ...,
-        argument_types: collections.abc.Iterable[pyspark.sql.connect.proto.types_pb2.DataType]
-        | None = ...,
-        return_type: pyspark.sql.connect.proto.types_pb2.DataType | None = ...,
-        serialized_function: builtins.bytes = ...,
-        literal_string: builtins.str = ...,
-    ) -> None: ...
-    def HasField(
-        self,
-        field_name: typing_extensions.Literal[
-            "function_definition",
-            b"function_definition",
-            "literal_string",
-            b"literal_string",
-            "return_type",
-            b"return_type",
-            "serialized_function",
-            b"serialized_function",
-        ],
-    ) -> builtins.bool: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "argument_types",
-            b"argument_types",
-            "function_definition",
-            b"function_definition",
-            "language",
-            b"language",
-            "literal_string",
-            b"literal_string",
-            "parts",
-            b"parts",
-            "return_type",
-            b"return_type",
-            "serialized_function",
-            b"serialized_function",
-            "temporary",
-            b"temporary",
-        ],
-    ) -> None: ...
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["function_definition", b"function_definition"]
-    ) -> typing_extensions.Literal["serialized_function", "literal_string"] | None: ...
-
-global___CreateScalarFunction = CreateScalarFunction
-
 class CreateDataFrameViewCommand(google.protobuf.message.Message):
     """A command that can create DataFrame global temp view or local temp view."""
 
diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py
index 399fdea1cf4..b93bbffc999 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -453,9 +453,6 @@ class SparkSession:
         """
         return self._client
 
-    def register_udf(self, function: Any, return_type: Union[str, DataType]) -> str:
-        return self._client.register_udf(function, return_type)
-
     @staticmethod
     def _start_connect_server(master: str, opts: Dict[str, Any]) -> None:
         """
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 6be2a07bc4b..4957710524a 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -65,7 +65,6 @@ if should_test_connect:
     from pyspark.sql.connect.readwriter import DataFrameWriterV2
     from pyspark.sql.dataframe import DataFrame
     from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
-    from pyspark.sql.connect.function_builder import udf
     from pyspark.sql import functions as SF
     from pyspark.sql.connect import functions as CF
     from pyspark.sql.connect.client import Retrying
@@ -421,15 +420,6 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
             .schema,
         )
 
-    def test_simple_udf(self):
-        def conv_udf(x) -> str:
-            return "Martin"
-
-        u = udf(conv_udf)
-        df = self.connect.read.table(self.tbl_name)
-        result = df.select(u(df.id)).toPandas()
-        self.assertIsNotNone(result)
-
     def test_with_local_data(self):
         """SPARK-41114: Test creating a dataframe using local data"""
         pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
diff --git a/python/pyspark/sql/tests/connect/test_connect_plan.py b/python/pyspark/sql/tests/connect/test_connect_plan.py
index 6c6c3b5908e..9cb7c5dac1e 100644
--- a/python/pyspark/sql/tests/connect/test_connect_plan.py
+++ b/python/pyspark/sql/tests/connect/test_connect_plan.py
@@ -32,7 +32,6 @@ if should_test_connect:
     from pyspark.sql.connect.plan import WriteOperation, Read
     from pyspark.sql.connect.readwriter import DataFrameReader
     from pyspark.sql.connect.functions import col, lit
-    from pyspark.sql.connect.function_builder import UserDefinedFunction, udf
     from pyspark.sql.connect.types import pyspark_types_to_proto_types
     from pyspark.sql.types import (
         StringType,
@@ -495,15 +494,6 @@ class SparkConnectPlanTests(PlanOnlyTestFixture):
         self.assertEqual(len(data_source.paths), 1)
         self.assertEqual(data_source.paths[0], "test_path")
 
-    def test_simple_udf(self):
-        u = udf(lambda x: "Martin", StringType())
-        self.assertIsNotNone(u)
-        expr = u("ThisCol", "ThatCol", "OtherCol")
-        self.assertTrue(isinstance(expr, Column))
-        self.assertTrue(isinstance(expr._expr, UserDefinedFunction))
-        u_plan = expr.to_plan(self.connect)
-        self.assertIsNotNone(u_plan)
-
     def test_all_the_plans(self):
         df = self.connect.readTable(table_name=self.tbl_name)
         df = df.select(df.col1).filter(df.col2 == 2).sort(df.col3.asc())
diff --git a/python/pyspark/testing/connectutils.py b/python/pyspark/testing/connectutils.py
index 764f49aa432..662a7d1446e 100644
--- a/python/pyspark/testing/connectutils.py
+++ b/python/pyspark/testing/connectutils.py
@@ -122,7 +122,6 @@ class PlanOnlyTestFixture(unittest.TestCase):
         cls.session = SparkSession.builder.remote().getOrCreate()
         cls.tbl_name = "test_connect_plan_only_table_1"
 
-        cls.connect.set_hook("register_udf", cls._udf_mock)
         cls.connect.set_hook("readTable", cls._read_table)
         cls.connect.set_hook("range", cls._session_range)
         cls.connect.set_hook("sql", cls._session_sql)
@@ -130,7 +129,6 @@ class PlanOnlyTestFixture(unittest.TestCase):
 
     @classmethod
     def tearDownClass(cls):
-        cls.connect.drop_hook("register_udf")
         cls.connect.drop_hook("readTable")
         cls.connect.drop_hook("range")
         cls.connect.drop_hook("sql")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org