You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/12/04 10:37:57 UTC

(spark) branch master updated: [SPARK-46234][PYTHON] Introduce `PySparkKeyError` for PySpark error framework

This is an automated email from the ASF dual-hosted git repository.

ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 2423a4517ace [SPARK-46234][PYTHON] Introduce `PySparkKeyError` for PySpark error framework
2423a4517ace is described below

commit 2423a4517ace1d76938ee3fd285594900f272552
Author: Haejoon Lee <ha...@databricks.com>
AuthorDate: Mon Dec 4 18:37:36 2023 +0800

    [SPARK-46234][PYTHON] Introduce `PySparkKeyError` for PySpark error framework
    
    ### What changes were proposed in this pull request?
    
    This PR proposes to introduce `PySparkKeyError` for error framework, and migrate Python built-in `KeyError` into `PySparkKeyError`.
    
    ### Why are the changes needed?
    
    For better error handling.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No API changes, but it's improve the user-facing error messages.
    
    ### How was this patch tested?
    
    The existing CI should pass
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #44151 from itholic/pyspark_keyerror.
    
    Authored-by: Haejoon Lee <ha...@databricks.com>
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
 python/docs/source/reference/pyspark.errors.rst | 1 +
 python/pyspark/errors/__init__.py               | 2 ++
 python/pyspark/errors/error_classes.py          | 5 +++++
 python/pyspark/errors/exceptions/base.py        | 6 ++++++
 python/pyspark/sql/types.py                     | 9 +++++++--
 5 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/python/docs/source/reference/pyspark.errors.rst b/python/docs/source/reference/pyspark.errors.rst
index 88cbd405b83d..a4997506b41e 100644
--- a/python/docs/source/reference/pyspark.errors.rst
+++ b/python/docs/source/reference/pyspark.errors.rst
@@ -38,6 +38,7 @@ Classes
     PySparkAssertionError
     PySparkAttributeError
     PySparkException
+    PySparkKeyError
     PySparkNotImplementedError
     PySparkPicklingError
     PySparkRuntimeError
diff --git a/python/pyspark/errors/__init__.py b/python/pyspark/errors/__init__.py
index 923cb665d112..07033d216432 100644
--- a/python/pyspark/errors/__init__.py
+++ b/python/pyspark/errors/__init__.py
@@ -46,6 +46,7 @@ from pyspark.errors.exceptions.base import (  # noqa: F401
     PySparkAssertionError,
     PySparkNotImplementedError,
     PySparkPicklingError,
+    PySparkKeyError,
 )
 
 
@@ -77,4 +78,5 @@ __all__ = [
     "PySparkAssertionError",
     "PySparkNotImplementedError",
     "PySparkPicklingError",
+    "PySparkKeyError",
 ]
diff --git a/python/pyspark/errors/error_classes.py b/python/pyspark/errors/error_classes.py
index d0c0d1c115b0..8ca73ca85de6 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -387,6 +387,11 @@ ERROR_CLASSES_JSON = """
       "Attribute `<attr_name>` is not supported in Spark Connect as it depends on the JVM. If you need to use this attribute, do not use Spark Connect when creating your session. Visit https://spark.apache.org/docs/latest/sql-getting-started.html#starting-point-sparksession for creating regular Spark Session in detail."
     ]
   },
+  "KEY_NOT_EXISTS" : {
+    "message" : [
+      "Key `<key>` is not exists."
+    ]
+  },
   "KEY_VALUE_PAIR_REQUIRED" : {
     "message" : [
       "Key-value pair or a list of pairs is required."
diff --git a/python/pyspark/errors/exceptions/base.py b/python/pyspark/errors/exceptions/base.py
index 4a2b31418e29..0f4001483b7f 100644
--- a/python/pyspark/errors/exceptions/base.py
+++ b/python/pyspark/errors/exceptions/base.py
@@ -264,3 +264,9 @@ class PySparkImportError(PySparkException, ImportError):
     """
     Wrapper class for ImportError to support error classes.
     """
+
+
+class PySparkKeyError(PySparkException, KeyError):
+    """
+    Wrapper class for KeyError to support error classes.
+    """
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 932a5a703ea9..cbfc4ab5df02 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -55,6 +55,7 @@ from pyspark.errors import (
     PySparkTypeError,
     PySparkValueError,
     PySparkIndexError,
+    PySparkKeyError,
 )
 
 if has_numpy:
@@ -1042,7 +1043,9 @@ class StructType(DataType):
             for field in self:
                 if field.name == key:
                     return field
-            raise KeyError("No StructField named {0}".format(key))
+            raise PySparkKeyError(
+                error_class="KEY_NOT_EXISTS", message_parameters={"key": str(key)}
+            )
         elif isinstance(key, int):
             try:
                 return self.fields[key]
@@ -2563,7 +2566,9 @@ class Row(tuple):
             idx = self.__fields__.index(item)
             return super(Row, self).__getitem__(idx)
         except IndexError:
-            raise KeyError(item)
+            raise PySparkKeyError(
+                error_class="KEY_NOT_EXISTS", message_parameters={"key": str(item)}
+            )
         except ValueError:
             raise PySparkValueError(item)
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org