[spark] branch master updated: [SPARK-37157][PYTHON] Inline type hints for python/pyspark/util.py

[gu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 8f20398  [SPARK-37157][PYTHON] Inline type hints for python/pyspark/util.py
8f20398 is described below

commit 8f203981d445e2d8ef73a3132f3195bc96775c1a
Author: dchvn <dg...@viettel.com.vn>
AuthorDate: Fri Nov 5 14:09:40 2021 +0900

    [SPARK-37157][PYTHON] Inline type hints for python/pyspark/util.py
    
    ### What changes were proposed in this pull request?
    Inline type hints for python/pyspark/util.py
    
    ### Why are the changes needed?
    We can take advantage of static type checking within the functions by inlining the type hints.
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Existing tests
    
    Closes #34438 from dchvn/SPARK-37157.
    
    Authored-by: dchvn <dg...@viettel.com.vn>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 python/pyspark/util.py  | 12 ++++++++----
 python/pyspark/util.pyi | 29 -----------------------------
 2 files changed, 8 insertions(+), 33 deletions(-)

diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index e075b04..4ab78ca 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -25,6 +25,7 @@ import sys
 import threading
 import traceback
 import types
+from collections import Callable
 
 from py4j.clientserver import ClientServer
 
@@ -41,7 +42,7 @@ class VersionUtils(object):
     Provides utility method to determine Spark versions with given input string.
     """
     @staticmethod
-    def majorMinorVersion(sparkVersion):
+    def majorMinorVersion(sparkVersion: str):
         """
         Given a Spark version string, return the (major version number, minor version number).
         E.g., for 2.0.1-SNAPSHOT, return (2, 0).
@@ -264,7 +265,7 @@ def _parse_memory(s):
     return int(float(s[:-1]) * units[s[-1].lower()])
 
 
-def inheritable_thread_target(f):
+def inheritable_thread_target(f: Callable) -> Callable:
     """
     Return thread target wrapper which is recommended to be used in PySpark when the
     pinned thread mode is enabled. The wrapper function, before calling original
@@ -310,13 +311,16 @@ def inheritable_thread_target(f):
     """
     from pyspark import SparkContext
 
-    if isinstance(SparkContext._gateway, ClientServer):
+    if isinstance(SparkContext._gateway, ClientServer):  # type: ignore[attr-defined]
         # Here's when the pinned-thread mode (PYSPARK_PIN_THREAD) is on.
 
         # NOTICE the internal difference vs `InheritableThread`. `InheritableThread`
         # copies local properties when the thread starts but `inheritable_thread_target`
         # copies when the function is wrapped.
-        properties = SparkContext._active_spark_context._jsc.sc().getLocalProperties().clone()
+        properties = (SparkContext
+                      ._active_spark_context  # type: ignore[attr-defined]
+                      ._jsc.sc()
+                      .getLocalProperties().clone())
 
         @functools.wraps(f)
         def wrapped(*args, **kwargs):
diff --git a/python/pyspark/util.pyi b/python/pyspark/util.pyi
deleted file mode 100644
index 06940bd..0000000
--- a/python/pyspark/util.pyi
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import threading
-from typing import Callable
-
-class VersionUtils(object):
-    @staticmethod
-    def majorMinorVersion(sparkVersion: str): ...
-
-def inheritable_thread_target(f: Callable) -> Callable: ...
-
-class InheritableThread(threading.Thread):
-    pass

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org