You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/02/07 05:31:23 UTC
[spark] branch branch-3.4 updated: [SPARK-42363][CONNECT] Remove SparkSession.register_udf
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 47839097a7e [SPARK-42363][CONNECT] Remove SparkSession.register_udf
47839097a7e is described below
commit 47839097a7e49da961c10c6399b20e79fe2dd264
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Mon Feb 6 21:31:02 2023 -0800
[SPARK-42363][CONNECT] Remove SparkSession.register_udf
### What changes were proposed in this pull request?
This PR proposes to remove `SparkSession.register_udf` and `pyspark.sql.connect.function_builder`.
### Why are the changes needed?
- For API parity w/ regular PySpark.
- They have to be implemented via `SparkSession.udf` (SPARK-41843).
- For non-registered Python and Pandas UDFs, they are implemented at SPARK-42124 and SPARK-42125.
- We implemented all functions so we do not need `pyspark.sql.connect.function_builder` workaround anymore.
### Does this PR introduce _any_ user-facing change?
No to end users.
Yee to the dev because it removes an API `SparkSession.register_udf` and `pyspark.sql.connect.function_builder`.
### How was this patch tested?
CI in this PR should verify them
Closes #39916 from HyukjinKwon/remove-register-udf.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 9fbdb2dc095131ebd5be41f61e9481574a40be4d)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../src/main/protobuf/spark/connect/commands.proto | 40 -------
.../sql/connect/planner/SparkConnectPlanner.scala | 39 +-----
python/pyspark/sql/connect/client.py | 19 ---
python/pyspark/sql/connect/function_builder.py | 109 -----------------
python/pyspark/sql/connect/proto/commands_pb2.py | 59 ++++------
python/pyspark/sql/connect/proto/commands_pb2.pyi | 131 +--------------------
python/pyspark/sql/connect/session.py | 3 -
.../sql/tests/connect/test_connect_basic.py | 10 --
.../pyspark/sql/tests/connect/test_connect_plan.py | 10 --
python/pyspark/testing/connectutils.py | 2 -
10 files changed, 22 insertions(+), 400 deletions(-)
diff --git a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
index da81da844e7..05c91d2c992 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto
@@ -31,7 +31,6 @@ option java_package = "org.apache.spark.connect.proto";
// produce a relational result.
message Command {
oneof command_type {
- CreateScalarFunction create_function = 1;
WriteOperation write_operation = 2;
CreateDataFrameViewCommand create_dataframe_view = 3;
WriteOperationV2 write_operation_v2 = 4;
@@ -43,45 +42,6 @@ message Command {
}
}
-// Simple message that is used to create a scalar function based on the provided function body.
-//
-// This message is used to register for example a Python UDF in the session catalog by providing
-// the serialized method body.
-//
-// TODO(SPARK-40532) It is required to add the interpreter / language version to the command
-// parameters.
-message CreateScalarFunction {
- // (Required) Fully qualified name of the function including the catalog / schema names.
- repeated string parts = 1;
-
- // (Required) the function language.
- FunctionLanguage language = 2;
-
- // (Required) if this is a temporary function.
- bool temporary = 3;
-
- // (Optional) A list of argument types. Can be empty when the function does not take an argument.
- repeated DataType argument_types = 4;
-
- // (Required) the return type of the function.
- DataType return_type = 5;
-
- // How the function body is defined:
- oneof function_definition {
- // As a raw string serialized:
- bytes serialized_function = 6;
- // As a code literal
- string literal_string = 7;
- }
-
- enum FunctionLanguage {
- FUNCTION_LANGUAGE_UNSPECIFIED = 0;
- FUNCTION_LANGUAGE_SQL = 1;
- FUNCTION_LANGUAGE_PYTHON = 2;
- FUNCTION_LANGUAGE_SCALA = 3;
- }
-}
-
// A command that can create DataFrame global temp view or local temp view.
message CreateDataFrameViewCommand {
// (Required) The relation that this view will be built on.
diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 08df2274840..28071acf8a4 100644
--- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -24,7 +24,7 @@ import com.google.common.collect.{Lists, Maps}
import com.google.protobuf.{Any => ProtoAny}
import org.apache.spark.TaskContext
-import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction}
+import org.apache.spark.api.python.SimplePythonFunction
import org.apache.spark.connect.proto
import org.apache.spark.sql.{Column, Dataset, Encoders, SparkSession}
import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier}
@@ -44,7 +44,6 @@ import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.execution.QueryExecution
import org.apache.spark.sql.execution.arrow.ArrowConverters
import org.apache.spark.sql.execution.command.CreateViewCommand
-import org.apache.spark.sql.execution.python.UserDefinedPythonFunction
import org.apache.spark.sql.functions.{col, expr}
import org.apache.spark.sql.internal.CatalogImpl
import org.apache.spark.sql.types._
@@ -1403,8 +1402,6 @@ class SparkConnectPlanner(val session: SparkSession) {
def process(command: proto.Command): Unit = {
command.getCommandTypeCase match {
- case proto.Command.CommandTypeCase.CREATE_FUNCTION =>
- handleCreateScalarFunction(command.getCreateFunction)
case proto.Command.CommandTypeCase.WRITE_OPERATION =>
handleWriteOperation(command.getWriteOperation)
case proto.Command.CommandTypeCase.CREATE_DATAFRAME_VIEW =>
@@ -1429,40 +1426,6 @@ class SparkConnectPlanner(val session: SparkSession) {
.getOrElse(throw InvalidPlanInput("No handler found for extension"))
}
- /**
- * This is a helper function that registers a new Python function in the SparkSession.
- *
- * Right now this function is very rudimentary and bare-bones just to showcase how it is
- * possible to remotely serialize a Python function and execute it on the Spark cluster. If the
- * Python version on the client and server diverge, the execution of the function that is
- * serialized will most likely fail.
- *
- * @param cf
- */
- private def handleCreateScalarFunction(cf: proto.CreateScalarFunction): Unit = {
- val function = SimplePythonFunction(
- cf.getSerializedFunction.toByteArray,
- // Empty environment variables
- Maps.newHashMap(),
- // No imported Python libraries
- Lists.newArrayList(),
- pythonExec,
- "3.9", // TODO(SPARK-40532) This needs to be an actual Python version.
- // Empty broadcast variables
- Lists.newArrayList(),
- // Null accumulator
- null)
-
- val udf = UserDefinedPythonFunction(
- cf.getPartsList.asScala.head,
- function,
- StringType,
- PythonEvalType.SQL_BATCHED_UDF,
- udfDeterministic = false)
-
- session.udf.registerPython(cf.getPartsList.asScala.head, udf)
- }
-
private def handleCreateViewCommand(createView: proto.CreateDataFrameViewCommand): Unit = {
val viewType = if (createView.getIsGlobal) GlobalTempView else LocalTempView
diff --git a/python/pyspark/sql/connect/client.py b/python/pyspark/sql/connect/client.py
index 133f58ea21a..36f1328b1bd 100644
--- a/python/pyspark/sql/connect/client.py
+++ b/python/pyspark/sql/connect/client.py
@@ -58,8 +58,6 @@ from google.rpc import error_details_pb2
import pyspark.sql.connect.proto as pb2
import pyspark.sql.connect.proto.base_pb2_grpc as grpc_lib
import pyspark.sql.connect.types as types
-import pyspark.sql.types
-from pyspark import cloudpickle
from pyspark.errors import (
SparkConnectException,
SparkConnectGrpcException,
@@ -429,23 +427,6 @@ class SparkConnectClient(object):
self._stub = grpc_lib.SparkConnectServiceStub(self._channel)
# Configure logging for the SparkConnect client.
- def register_udf(
- self, function: Any, return_type: Union[str, pyspark.sql.types.DataType]
- ) -> str:
- """Create a temporary UDF in the session catalog on the other side. We generate a
- temporary name for it."""
- name = f"fun_{uuid.uuid4().hex}"
- fun = pb2.CreateScalarFunction()
- fun.parts.append(name)
- logger.info(f"Registering UDF: {self._proto_to_string(fun)}")
- fun.serialized_function = cloudpickle.dumps((function, return_type))
-
- req = self._execute_plan_request_with_metadata()
- req.plan.command.create_function.CopyFrom(fun)
-
- self._execute(req)
- return name
-
def _build_metrics(self, metrics: "pb2.ExecutePlanResponse.Metrics") -> List[PlanMetrics]:
return [
PlanMetrics(
diff --git a/python/pyspark/sql/connect/function_builder.py b/python/pyspark/sql/connect/function_builder.py
deleted file mode 100644
index f3416e78286..00000000000
--- a/python/pyspark/sql/connect/function_builder.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-from pyspark.sql.connect import check_dependencies
-
-check_dependencies(__name__, __file__)
-
-import functools
-from typing import TYPE_CHECKING, Optional, Any, Iterable, Union
-
-import pyspark.sql.types
-
-import pyspark.sql.connect.proto as proto
-from pyspark.sql.connect.column import Column
-from pyspark.sql.connect.expressions import Expression
-from pyspark.sql.connect.functions import _invoke_function_over_columns
-
-
-if TYPE_CHECKING:
- from pyspark.sql.connect._typing import (
- ColumnOrName,
- UserDefinedFunctionCallable,
- )
- from pyspark.sql.connect.client import SparkConnectClient
-
-
-class UserDefinedFunction(Expression):
- """A user defied function is an expression that has a reference to the actual
- Python callable attached. During plan generation, the client sends a command to
- the server to register the UDF before execution. The expression object can be
- reused and is not attached to a specific execution. If the internal name of
- the temporary function is set, it is assumed that the registration has already
- happened."""
-
- def __init__(
- self,
- func: Any,
- return_type: Union[str, pyspark.sql.types.DataType] = pyspark.sql.types.StringType(),
- args: Optional[Iterable[Any]] = None,
- ) -> None:
- super().__init__()
-
- self._func_ref = func
- self._return_type = return_type
- if args is not None:
- self._args = list(args)
- else:
- self._args = []
- self._func_name = None
-
- def to_plan(self, session: "SparkConnectClient") -> proto.Expression:
- if session is None:
- raise Exception("CAnnot create UDF without remote Session.")
- # Needs to materialize the UDF to the server
- # Only do this once per session
- func_name = session.register_udf(self._func_ref, self._return_type)
- # Func name is used for the actual reference
- return _invoke_function_over_columns(func_name, *self._args).to_plan(session)
-
- def __str__(self) -> str:
- return f"UserDefinedFunction({self._func_name})"
-
-
-def _create_udf(
- function: Any, return_type: Union[str, pyspark.sql.types.DataType]
-) -> "UserDefinedFunctionCallable":
- def wrapper(*cols: "ColumnOrName") -> "Column":
- return Column(UserDefinedFunction(func=function, return_type=return_type, args=cols))
-
- return wrapper
-
-
-def udf(
- function: Any, return_type: pyspark.sql.types.DataType = pyspark.sql.types.StringType()
-) -> Any:
- """
- Returns a callable that represents the column once arguments are applied
-
- Parameters
- ----------
- function
- return_type
-
- Returns
- -------
-
- """
- # This is when @udf / @udf(DataType()) is used
- if function is None or isinstance(function, (str, pyspark.sql.types.DataType)):
- actual_return_type = function or return_type
- # Overload with
- if actual_return_type is None:
- actual_return_type = pyspark.sql.types.StringType()
- return functools.partial(_create_udf, return_type=actual_return_type)
- else:
- return _create_udf(function, return_type)
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py
index 482e7dd4fcc..c9a51b04bb6 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.py
+++ b/python/pyspark/sql/connect/proto/commands_pb2.py
@@ -36,12 +36,11 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\x9c\x03\n\x07\x43ommand\x12N\n\x0f\x63reate_function\x18\x01 \x01(\x0b\x32#.spark.connect.CreateScalarFunctionH\x00R\x0e\x63reateFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x0 [...]
+ b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\xcc\x02\n\x07\x43ommand\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\ [...]
)
_COMMAND = DESCRIPTOR.message_types_by_name["Command"]
-_CREATESCALARFUNCTION = DESCRIPTOR.message_types_by_name["CreateScalarFunction"]
_CREATEDATAFRAMEVIEWCOMMAND = DESCRIPTOR.message_types_by_name["CreateDataFrameViewCommand"]
_WRITEOPERATION = DESCRIPTOR.message_types_by_name["WriteOperation"]
_WRITEOPERATION_OPTIONSENTRY = _WRITEOPERATION.nested_types_by_name["OptionsEntry"]
@@ -51,9 +50,6 @@ _WRITEOPERATIONV2_OPTIONSENTRY = _WRITEOPERATIONV2.nested_types_by_name["Options
_WRITEOPERATIONV2_TABLEPROPERTIESENTRY = _WRITEOPERATIONV2.nested_types_by_name[
"TablePropertiesEntry"
]
-_CREATESCALARFUNCTION_FUNCTIONLANGUAGE = _CREATESCALARFUNCTION.enum_types_by_name[
- "FunctionLanguage"
-]
_WRITEOPERATION_SAVEMODE = _WRITEOPERATION.enum_types_by_name["SaveMode"]
_WRITEOPERATIONV2_MODE = _WRITEOPERATIONV2.enum_types_by_name["Mode"]
Command = _reflection.GeneratedProtocolMessageType(
@@ -67,17 +63,6 @@ Command = _reflection.GeneratedProtocolMessageType(
)
_sym_db.RegisterMessage(Command)
-CreateScalarFunction = _reflection.GeneratedProtocolMessageType(
- "CreateScalarFunction",
- (_message.Message,),
- {
- "DESCRIPTOR": _CREATESCALARFUNCTION,
- "__module__": "spark.connect.commands_pb2"
- # @@protoc_insertion_point(class_scope:spark.connect.CreateScalarFunction)
- },
-)
-_sym_db.RegisterMessage(CreateScalarFunction)
-
CreateDataFrameViewCommand = _reflection.GeneratedProtocolMessageType(
"CreateDataFrameViewCommand",
(_message.Message,),
@@ -162,27 +147,23 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_WRITEOPERATIONV2_TABLEPROPERTIESENTRY._options = None
_WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_options = b"8\001"
_COMMAND._serialized_start = 166
- _COMMAND._serialized_end = 578
- _CREATESCALARFUNCTION._serialized_start = 581
- _CREATESCALARFUNCTION._serialized_end = 1116
- _CREATESCALARFUNCTION_FUNCTIONLANGUAGE._serialized_start = 954
- _CREATESCALARFUNCTION_FUNCTIONLANGUAGE._serialized_end = 1093
- _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 1119
- _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 1269
- _WRITEOPERATION._serialized_start = 1272
- _WRITEOPERATION._serialized_end = 2014
- _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1710
- _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1768
- _WRITEOPERATION_BUCKETBY._serialized_start = 1770
- _WRITEOPERATION_BUCKETBY._serialized_end = 1861
- _WRITEOPERATION_SAVEMODE._serialized_start = 1864
- _WRITEOPERATION_SAVEMODE._serialized_end = 2001
- _WRITEOPERATIONV2._serialized_start = 2017
- _WRITEOPERATIONV2._serialized_end = 2812
- _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1710
- _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1768
- _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 2584
- _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 2650
- _WRITEOPERATIONV2_MODE._serialized_start = 2653
- _WRITEOPERATIONV2_MODE._serialized_end = 2812
+ _COMMAND._serialized_end = 498
+ _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 501
+ _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 651
+ _WRITEOPERATION._serialized_start = 654
+ _WRITEOPERATION._serialized_end = 1396
+ _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1092
+ _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1150
+ _WRITEOPERATION_BUCKETBY._serialized_start = 1152
+ _WRITEOPERATION_BUCKETBY._serialized_end = 1243
+ _WRITEOPERATION_SAVEMODE._serialized_start = 1246
+ _WRITEOPERATION_SAVEMODE._serialized_end = 1383
+ _WRITEOPERATIONV2._serialized_start = 1399
+ _WRITEOPERATIONV2._serialized_end = 2194
+ _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1092
+ _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1150
+ _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 1966
+ _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 2032
+ _WRITEOPERATIONV2_MODE._serialized_start = 2035
+ _WRITEOPERATIONV2_MODE._serialized_end = 2194
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi
index 2447be6b467..8a1f2ffb122 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi
@@ -42,7 +42,6 @@ import google.protobuf.internal.enum_type_wrapper
import google.protobuf.message
import pyspark.sql.connect.proto.expressions_pb2
import pyspark.sql.connect.proto.relations_pb2
-import pyspark.sql.connect.proto.types_pb2
import sys
import typing
@@ -60,14 +59,11 @@ class Command(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
- CREATE_FUNCTION_FIELD_NUMBER: builtins.int
WRITE_OPERATION_FIELD_NUMBER: builtins.int
CREATE_DATAFRAME_VIEW_FIELD_NUMBER: builtins.int
WRITE_OPERATION_V2_FIELD_NUMBER: builtins.int
EXTENSION_FIELD_NUMBER: builtins.int
@property
- def create_function(self) -> global___CreateScalarFunction: ...
- @property
def write_operation(self) -> global___WriteOperation: ...
@property
def create_dataframe_view(self) -> global___CreateDataFrameViewCommand: ...
@@ -81,7 +77,6 @@ class Command(google.protobuf.message.Message):
def __init__(
self,
*,
- create_function: global___CreateScalarFunction | None = ...,
write_operation: global___WriteOperation | None = ...,
create_dataframe_view: global___CreateDataFrameViewCommand | None = ...,
write_operation_v2: global___WriteOperationV2 | None = ...,
@@ -94,8 +89,6 @@ class Command(google.protobuf.message.Message):
b"command_type",
"create_dataframe_view",
b"create_dataframe_view",
- "create_function",
- b"create_function",
"extension",
b"extension",
"write_operation",
@@ -111,8 +104,6 @@ class Command(google.protobuf.message.Message):
b"command_type",
"create_dataframe_view",
b"create_dataframe_view",
- "create_function",
- b"create_function",
"extension",
b"extension",
"write_operation",
@@ -124,131 +115,11 @@ class Command(google.protobuf.message.Message):
def WhichOneof(
self, oneof_group: typing_extensions.Literal["command_type", b"command_type"]
) -> typing_extensions.Literal[
- "create_function",
- "write_operation",
- "create_dataframe_view",
- "write_operation_v2",
- "extension",
+ "write_operation", "create_dataframe_view", "write_operation_v2", "extension"
] | None: ...
global___Command = Command
-class CreateScalarFunction(google.protobuf.message.Message):
- """Simple message that is used to create a scalar function based on the provided function body.
-
- This message is used to register for example a Python UDF in the session catalog by providing
- the serialized method body.
-
- TODO(SPARK-40532) It is required to add the interpreter / language version to the command
- parameters.
- """
-
- DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
- class _FunctionLanguage:
- ValueType = typing.NewType("ValueType", builtins.int)
- V: typing_extensions.TypeAlias = ValueType
-
- class _FunctionLanguageEnumTypeWrapper(
- google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[
- CreateScalarFunction._FunctionLanguage.ValueType
- ],
- builtins.type,
- ): # noqa: F821
- DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
- FUNCTION_LANGUAGE_UNSPECIFIED: CreateScalarFunction._FunctionLanguage.ValueType # 0
- FUNCTION_LANGUAGE_SQL: CreateScalarFunction._FunctionLanguage.ValueType # 1
- FUNCTION_LANGUAGE_PYTHON: CreateScalarFunction._FunctionLanguage.ValueType # 2
- FUNCTION_LANGUAGE_SCALA: CreateScalarFunction._FunctionLanguage.ValueType # 3
-
- class FunctionLanguage(_FunctionLanguage, metaclass=_FunctionLanguageEnumTypeWrapper): ...
- FUNCTION_LANGUAGE_UNSPECIFIED: CreateScalarFunction.FunctionLanguage.ValueType # 0
- FUNCTION_LANGUAGE_SQL: CreateScalarFunction.FunctionLanguage.ValueType # 1
- FUNCTION_LANGUAGE_PYTHON: CreateScalarFunction.FunctionLanguage.ValueType # 2
- FUNCTION_LANGUAGE_SCALA: CreateScalarFunction.FunctionLanguage.ValueType # 3
-
- PARTS_FIELD_NUMBER: builtins.int
- LANGUAGE_FIELD_NUMBER: builtins.int
- TEMPORARY_FIELD_NUMBER: builtins.int
- ARGUMENT_TYPES_FIELD_NUMBER: builtins.int
- RETURN_TYPE_FIELD_NUMBER: builtins.int
- SERIALIZED_FUNCTION_FIELD_NUMBER: builtins.int
- LITERAL_STRING_FIELD_NUMBER: builtins.int
- @property
- def parts(
- self,
- ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
- """(Required) Fully qualified name of the function including the catalog / schema names."""
- language: global___CreateScalarFunction.FunctionLanguage.ValueType
- """(Required) the function language."""
- temporary: builtins.bool
- """(Required) if this is a temporary function."""
- @property
- def argument_types(
- self,
- ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[
- pyspark.sql.connect.proto.types_pb2.DataType
- ]:
- """(Optional) A list of argument types. Can be empty when the function does not take an argument."""
- @property
- def return_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
- """(Required) the return type of the function."""
- serialized_function: builtins.bytes
- """As a raw string serialized:"""
- literal_string: builtins.str
- """As a code literal"""
- def __init__(
- self,
- *,
- parts: collections.abc.Iterable[builtins.str] | None = ...,
- language: global___CreateScalarFunction.FunctionLanguage.ValueType = ...,
- temporary: builtins.bool = ...,
- argument_types: collections.abc.Iterable[pyspark.sql.connect.proto.types_pb2.DataType]
- | None = ...,
- return_type: pyspark.sql.connect.proto.types_pb2.DataType | None = ...,
- serialized_function: builtins.bytes = ...,
- literal_string: builtins.str = ...,
- ) -> None: ...
- def HasField(
- self,
- field_name: typing_extensions.Literal[
- "function_definition",
- b"function_definition",
- "literal_string",
- b"literal_string",
- "return_type",
- b"return_type",
- "serialized_function",
- b"serialized_function",
- ],
- ) -> builtins.bool: ...
- def ClearField(
- self,
- field_name: typing_extensions.Literal[
- "argument_types",
- b"argument_types",
- "function_definition",
- b"function_definition",
- "language",
- b"language",
- "literal_string",
- b"literal_string",
- "parts",
- b"parts",
- "return_type",
- b"return_type",
- "serialized_function",
- b"serialized_function",
- "temporary",
- b"temporary",
- ],
- ) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["function_definition", b"function_definition"]
- ) -> typing_extensions.Literal["serialized_function", "literal_string"] | None: ...
-
-global___CreateScalarFunction = CreateScalarFunction
-
class CreateDataFrameViewCommand(google.protobuf.message.Message):
"""A command that can create DataFrame global temp view or local temp view."""
diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py
index 399fdea1cf4..b93bbffc999 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -453,9 +453,6 @@ class SparkSession:
"""
return self._client
- def register_udf(self, function: Any, return_type: Union[str, DataType]) -> str:
- return self._client.register_udf(function, return_type)
-
@staticmethod
def _start_connect_server(master: str, opts: Dict[str, Any]) -> None:
"""
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 6be2a07bc4b..4957710524a 100644
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -65,7 +65,6 @@ if should_test_connect:
from pyspark.sql.connect.readwriter import DataFrameWriterV2
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.connect.dataframe import DataFrame as CDataFrame
- from pyspark.sql.connect.function_builder import udf
from pyspark.sql import functions as SF
from pyspark.sql.connect import functions as CF
from pyspark.sql.connect.client import Retrying
@@ -421,15 +420,6 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
.schema,
)
- def test_simple_udf(self):
- def conv_udf(x) -> str:
- return "Martin"
-
- u = udf(conv_udf)
- df = self.connect.read.table(self.tbl_name)
- result = df.select(u(df.id)).toPandas()
- self.assertIsNotNone(result)
-
def test_with_local_data(self):
"""SPARK-41114: Test creating a dataframe using local data"""
pdf = pd.DataFrame({"a": [1, 2, 3], "b": ["a", "b", "c"]})
diff --git a/python/pyspark/sql/tests/connect/test_connect_plan.py b/python/pyspark/sql/tests/connect/test_connect_plan.py
index 6c6c3b5908e..9cb7c5dac1e 100644
--- a/python/pyspark/sql/tests/connect/test_connect_plan.py
+++ b/python/pyspark/sql/tests/connect/test_connect_plan.py
@@ -32,7 +32,6 @@ if should_test_connect:
from pyspark.sql.connect.plan import WriteOperation, Read
from pyspark.sql.connect.readwriter import DataFrameReader
from pyspark.sql.connect.functions import col, lit
- from pyspark.sql.connect.function_builder import UserDefinedFunction, udf
from pyspark.sql.connect.types import pyspark_types_to_proto_types
from pyspark.sql.types import (
StringType,
@@ -495,15 +494,6 @@ class SparkConnectPlanTests(PlanOnlyTestFixture):
self.assertEqual(len(data_source.paths), 1)
self.assertEqual(data_source.paths[0], "test_path")
- def test_simple_udf(self):
- u = udf(lambda x: "Martin", StringType())
- self.assertIsNotNone(u)
- expr = u("ThisCol", "ThatCol", "OtherCol")
- self.assertTrue(isinstance(expr, Column))
- self.assertTrue(isinstance(expr._expr, UserDefinedFunction))
- u_plan = expr.to_plan(self.connect)
- self.assertIsNotNone(u_plan)
-
def test_all_the_plans(self):
df = self.connect.readTable(table_name=self.tbl_name)
df = df.select(df.col1).filter(df.col2 == 2).sort(df.col3.asc())
diff --git a/python/pyspark/testing/connectutils.py b/python/pyspark/testing/connectutils.py
index 764f49aa432..662a7d1446e 100644
--- a/python/pyspark/testing/connectutils.py
+++ b/python/pyspark/testing/connectutils.py
@@ -122,7 +122,6 @@ class PlanOnlyTestFixture(unittest.TestCase):
cls.session = SparkSession.builder.remote().getOrCreate()
cls.tbl_name = "test_connect_plan_only_table_1"
- cls.connect.set_hook("register_udf", cls._udf_mock)
cls.connect.set_hook("readTable", cls._read_table)
cls.connect.set_hook("range", cls._session_range)
cls.connect.set_hook("sql", cls._session_sql)
@@ -130,7 +129,6 @@ class PlanOnlyTestFixture(unittest.TestCase):
@classmethod
def tearDownClass(cls):
- cls.connect.drop_hook("register_udf")
cls.connect.drop_hook("readTable")
cls.connect.drop_hook("range")
cls.connect.drop_hook("sql")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org