You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/07/01 23:25:14 UTC
(spark) branch master updated: [SPARK-52539][CONNECT][FOLLOW-UP] Avoid using TypeAlias that is available from Python 3.10
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 47782d046225 [SPARK-52539][CONNECT][FOLLOW-UP] Avoid using TypeAlias that is available from Python 3.10
47782d046225 is described below
commit 47782d0462258edd2612c8d28c69115b5d89c4c6
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Wed Jul 2 08:25:03 2025 +0900
[SPARK-52539][CONNECT][FOLLOW-UP] Avoid using TypeAlias that is available from Python 3.10
### What changes were proposed in this pull request?
This PR proposes to remove the usage of `TypeAlias` that is available from Python 3.10. We have not dropped Python 3.9 yet.
### Why are the changes needed?
It fails builds, e.g., https://github.com/apache/spark/actions/runs/15969875397/job/45038592377
### Does this PR introduce _any_ user-facing change?
No, the main change has not been released out yet.
### How was this patch tested?
Manually
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51331 from HyukjinKwon/SPARK-52539-followup.
Lead-authored-by: Hyukjin Kwon <gu...@apache.org>
Co-authored-by: Hyukjin Kwon <gu...@gmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/session.py | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py
index fe923d471060..f2c06a2350a9 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -33,7 +33,6 @@ from typing import (
Dict,
List,
Tuple,
- TypeAlias,
Set,
cast,
overload,
@@ -132,7 +131,7 @@ class SparkSession:
def __init__(self) -> None:
self._options: Dict[str, Any] = {}
self._channel_builder: Optional[DefaultChannelBuilder] = None
- self._hook_factories: list["SparkSession.HookFactory"] = []
+ self._hook_factories: list["Callable[[SparkSession], SparkSession.Hook]"] = []
@overload
def config(self, key: str, value: Any) -> "SparkSession.Builder":
@@ -194,7 +193,9 @@ class SparkSession:
self._channel_builder = channelBuilder
return self
- def _registerHook(self, hook_factory: "SparkSession.HookFactory") -> "SparkSession.Builder":
+ def _registerHook(
+ self, hook_factory: "Callable[[SparkSession], SparkSession.Hook]"
+ ) -> "SparkSession.Builder":
with self._lock:
self._hook_factories.append(hook_factory)
return self
@@ -275,9 +276,6 @@ class SparkSession:
"""
return request
- HookFactory: TypeAlias = Callable[["SparkSession"], Hook]
- """A function that, given a session, returns a hook set up for it."""
-
_client: SparkConnectClient
# SPARK-47544: Explicitly declaring this as an identifier instead of a method.
@@ -289,7 +287,7 @@ class SparkSession:
self,
connection: Union[str, DefaultChannelBuilder],
userId: Optional[str] = None,
- hook_factories: Optional[list[HookFactory]] = None,
+ hook_factories: Optional[list["Callable[[SparkSession], Hook]"]] = None,
) -> None:
"""
Creates a new SparkSession for the Spark Connect interface.
@@ -305,7 +303,7 @@ class SparkSession:
isolate their Spark Sessions. If the `user_id` is not set, will default to
the $USER environment. Defining the user ID as part of the connection string
takes precedence.
- hook_factories: list[HookFactory], optional
+ hook_factories: list[Callable[[SparkSession], Hook]], optional
Optional list of hook factories for hooks that should be registered for this session.
"""
hook_factories = hook_factories or []
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org