You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/03/25 08:28:54 UTC
(spark) branch master updated: [SPARK-56179][PYTHON] Consolidate error classes for type mismatch - part 2
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 524d7e28cedb [SPARK-56179][PYTHON] Consolidate error classes for type mismatch - part 2
524d7e28cedb is described below
commit 524d7e28cedba3da5a3b4d95c3e73332e5740877
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Wed Mar 25 16:28:24 2026 +0800
[SPARK-56179][PYTHON] Consolidate error classes for type mismatch - part 2
### What changes were proposed in this pull request?
Consolidate the next 13 `NOT_*` error classes into `NOT_EXPECTED_TYPE`:
- `NOT_COLUMN_OR_INT`
- `NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE`
- `NOT_COLUMN_OR_INT_OR_STR`
- `NOT_COLUMN_OR_STR`
- `NOT_COLUMN_OR_STR_OR_STRUCT`
- `NOT_DATAFRAME`
- `NOT_DATATYPE_OR_STR`
- `NOT_DICT`
- `NOT_EXPRESSION`
- `NOT_FLOAT_OR_INT`
- `NOT_INT`
- `NOT_INT_OR_SLICE_OR_STR`
- `NOT_LIST`
This is a follow-up to #54978.
### Why are the changes needed?
There are around 40 error classes for type mismatch, they can be consolidated into a single `NOT_EXPECTED_TYPE` class.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
CI
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (claude-opus-4-6)
Closes #54997 from zhengruifeng/consolidate_no_class_part2.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/errors/error-conditions.json | 78 -------------
python/pyspark/pipelines/api.py | 9 +-
python/pyspark/sql/classic/column.py | 32 ++++--
python/pyspark/sql/classic/dataframe.py | 84 ++++++++++----
python/pyspark/sql/connect/column.py | 32 ++++--
python/pyspark/sql/connect/dataframe.py | 116 ++++++++++++++-----
python/pyspark/sql/connect/functions/builtin.py | 123 +++++++++++++++------
.../pyspark/sql/connect/functions/partitioning.py | 3 +-
python/pyspark/sql/connect/group.py | 8 +-
python/pyspark/sql/connect/readwriter.py | 3 +-
python/pyspark/sql/connect/udf.py | 11 +-
python/pyspark/sql/dataframe.py | 11 +-
python/pyspark/sql/functions/builtin.py | 88 +++++++++++----
python/pyspark/sql/functions/partitioning.py | 3 +-
python/pyspark/sql/readwriter.py | 3 +-
python/pyspark/sql/streaming/state.py | 3 +-
.../sql/tests/connect/test_connect_basic.py | 12 +-
.../sql/tests/connect/test_connect_column.py | 11 +-
.../sql/tests/connect/test_connect_error.py | 8 +-
.../sql/tests/connect/test_connect_function.py | 32 ++++--
.../pyspark/sql/tests/connect/test_connect_stat.py | 3 +-
python/pyspark/sql/tests/pandas/test_pandas_udf.py | 8 +-
python/pyspark/sql/tests/test_column.py | 24 +++-
python/pyspark/sql/tests/test_dataframe.py | 24 ++--
python/pyspark/sql/tests/test_functions.py | 107 +++++++++++++-----
python/pyspark/sql/tests/test_repartition.py | 24 +++-
python/pyspark/sql/tests/test_udf.py | 12 +-
python/pyspark/sql/types.py | 8 +-
python/pyspark/sql/udf.py | 11 +-
29 files changed, 600 insertions(+), 291 deletions(-)
diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json
index 5f7d2da6398c..ca7d97439e17 100644
--- a/python/pyspark/errors/error-conditions.json
+++ b/python/pyspark/errors/error-conditions.json
@@ -602,90 +602,18 @@
"Value for `<arg_name>` must be greater than or equal to 0, got '<arg_value>'."
]
},
- "NOT_COLUMN_OR_INT": {
- "message": [
- "Argument `<arg_name>` should be a Column or int, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE": {
- "message": [
- "Argument `<arg_name>` should be a Column, int, list, str or tuple, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_COLUMN_OR_INT_OR_STR": {
- "message": [
- "Argument `<arg_name>` should be a Column, int or str, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_COLUMN_OR_STR": {
- "message": [
- "Argument `<arg_name>` should be a Column or str, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_COLUMN_OR_STR_OR_STRUCT": {
- "message": [
- "Argument `<arg_name>` should be a StructType, Column or str, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_DATAFRAME": {
- "message": [
- "Argument `<arg_name>` should be a DataFrame, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_DATATYPE_OR_STR": {
- "message": [
- "Argument `<arg_name>` should be a DataType or str, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_DICT": {
- "message": [
- "Argument `<arg_name>` should be a dict, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
"NOT_EXPECTED_TYPE": {
"message": [
"Argument `<arg_name>` should be <expected_type>, got <arg_type>."
],
"sqlState": "42K09"
},
- "NOT_EXPRESSION": {
- "message": [
- "Argument `<arg_name>` should be an Expression, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_FLOAT_OR_INT": {
- "message": [
- "Argument `<arg_name>` should be a float or int, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
"NOT_IMPLEMENTED": {
"message": [
"<feature> is not implemented."
],
"sqlState": "0A000"
},
- "NOT_INT": {
- "message": [
- "Argument `<arg_name>` should be an int, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
- "NOT_INT_OR_SLICE_OR_STR": {
- "message": [
- "Argument `<arg_name>` should be an int, slice or str, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
"NOT_IN_BARRIER_STAGE": {
"message": [
"It is not in a barrier stage."
@@ -697,12 +625,6 @@
],
"sqlState": "42K09"
},
- "NOT_LIST": {
- "message": [
- "Argument `<arg_name>` should be a list, got <arg_type>."
- ],
- "sqlState": "42K09"
- },
"NOT_LIST_OF_COLUMN": {
"message": [
"Argument `<arg_name>` should be a list[Column]."
diff --git a/python/pyspark/pipelines/api.py b/python/pyspark/pipelines/api.py
index f9c642cadc3b..a188400b15ed 100644
--- a/python/pyspark/pipelines/api.py
+++ b/python/pyspark/pipelines/api.py
@@ -85,8 +85,9 @@ def _validate_stored_dataset_args(
)
if table_properties is not None and not isinstance(table_properties, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "dict",
"arg_name": "table_properties",
"arg_type": type(table_properties).__name__,
},
@@ -434,8 +435,9 @@ def create_streaming_table(
)
if table_properties is not None and not isinstance(table_properties, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "dict",
"arg_name": "table_properties",
"arg_type": type(table_properties).__name__,
},
@@ -484,8 +486,9 @@ def create_sink(
)
if options is not None and not isinstance(options, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "dict",
"arg_name": "options",
"arg_type": type(options).__name__,
},
diff --git a/python/pyspark/sql/classic/column.py b/python/pyspark/sql/classic/column.py
index 99ad057810ae..284ba8956edc 100644
--- a/python/pyspark/sql/classic/column.py
+++ b/python/pyspark/sql/classic/column.py
@@ -69,8 +69,12 @@ def _to_java_column(col: "ColumnOrName") -> "JavaObject":
jcol = _create_column_from_name(col)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": type(col).__name__,
+ },
)
return jcol
@@ -470,8 +474,12 @@ class Column(ParentColumn):
jc = self._jc.substr(startPos._jc, cast(ParentColumn, length)._jc)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT",
- messageParameters={"arg_name": "startPos", "arg_type": type(startPos).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or int",
+ "arg_name": "startPos",
+ "arg_type": type(startPos).__name__,
+ },
)
return Column(jc)
@@ -557,8 +565,12 @@ class Column(ParentColumn):
jc = self._jc.cast(jdt)
else:
raise PySparkTypeError(
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "dataType",
+ "arg_type": type(dataType).__name__,
+ },
)
return Column(jc)
@@ -573,8 +585,12 @@ class Column(ParentColumn):
jc = self._jc.try_cast(jdt)
else:
raise PySparkTypeError(
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "dataType",
+ "arg_type": type(dataType).__name__,
+ },
)
return Column(jc)
diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py
index 1d8a074e43f8..6ca0efd443b9 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -301,8 +301,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
) -> str:
if not isinstance(n, int) or isinstance(n, bool):
raise PySparkTypeError(
- errorClass="NOT_INT",
- messageParameters={"arg_name": "n", "arg_type": type(n).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int",
+ "arg_name": "n",
+ "arg_type": type(n).__name__,
+ },
)
if not isinstance(vertical, bool):
@@ -565,8 +569,9 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return DataFrame(self._jdf.repartition(self._jcols(*cols)), self.sparkSession)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or str",
"arg_name": "numPartitions",
"arg_type": type(numPartitions).__name__,
},
@@ -591,8 +596,9 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return DataFrame(self._jdf.repartitionByRange(self._jcols(*cols)), self.sparkSession)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column, int or str",
"arg_name": "numPartitions",
"arg_type": type(numPartitions).__name__,
},
@@ -603,8 +609,9 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
) -> ParentDataFrame:
if not isinstance(numPartitions, (int, bool)):
raise PySparkTypeError(
- errorClass="NOT_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "int",
"arg_name": "numPartitions",
"arg_type": type(numPartitions).__name__,
},
@@ -653,13 +660,21 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
col = Column(col)
elif not isinstance(col, Column):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": type(col).__name__,
+ },
)
if not isinstance(fractions, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "fractions", "arg_type": type(fractions).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "fractions",
+ "arg_type": type(fractions).__name__,
+ },
)
for k, v in fractions.items():
if not isinstance(k, (float, int, str)):
@@ -1069,8 +1084,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
jdf = self._jdf.filter(condition._jc)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "condition", "arg_type": type(condition).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "condition",
+ "arg_type": type(condition).__name__,
+ },
)
return DataFrame(jdf, self.sparkSession)
@@ -1544,8 +1563,9 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
if not isinstance(relativeError, (float, int)):
raise PySparkTypeError(
- errorClass="NOT_FLOAT_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "float or int",
"arg_name": "relativeError",
"arg_type": type(relativeError).__name__,
},
@@ -1649,8 +1669,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
if not isinstance(colsMap, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "colsMap", "arg_type": type(colsMap).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "colsMap",
+ "arg_type": type(colsMap).__name__,
+ },
)
col_names = list(colsMap.keys())
@@ -1679,8 +1703,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
def withColumnsRenamed(self, colsMap: Dict[str, str]) -> ParentDataFrame:
if not isinstance(colsMap, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "colsMap", "arg_type": type(colsMap).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "colsMap",
+ "arg_type": type(colsMap).__name__,
+ },
)
col_names: List[str] = []
@@ -1701,8 +1729,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
if not isinstance(metadata, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "metadata", "arg_type": type(metadata).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "metadata",
+ "arg_type": type(metadata).__name__,
+ },
)
sc = get_active_spark_context()
jmeta = cast(JVMView, sc._jvm).org.apache.spark.sql.types.Metadata.fromJson(
@@ -1721,8 +1753,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
java_columns.append(c._jc)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": type(c).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": type(c).__name__,
+ },
)
jdf = self._jdf
@@ -1756,8 +1792,12 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
def sameSemantics(self, other: ParentDataFrame) -> bool:
if not isinstance(other, DataFrame):
raise PySparkTypeError(
- errorClass="NOT_DATAFRAME",
- messageParameters={"arg_name": "other", "arg_type": type(other).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataFrame",
+ "arg_name": "other",
+ "arg_type": type(other).__name__,
+ },
)
return self._jdf.sameSemantics(other._jdf)
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index 1968ed6f9b2a..c792e8db20af 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -116,8 +116,12 @@ class Column(ParentColumn):
def __init__(self, expr: "Expression") -> None:
if not isinstance(expr, Expression):
raise PySparkTypeError(
- errorClass="NOT_EXPRESSION",
- messageParameters={"arg_name": "expr", "arg_type": type(expr).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Expression",
+ "arg_name": "expr",
+ "arg_type": type(expr).__name__,
+ },
)
self._expr = expr
@@ -377,8 +381,12 @@ class Column(ParentColumn):
start_expr = _to_expr(startPos)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT",
- messageParameters={"arg_name": "startPos", "arg_type": type(length).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or int",
+ "arg_name": "startPos",
+ "arg_type": type(length).__name__,
+ },
)
return Column(UnresolvedFunction("substr", [self._expr, start_expr, length_expr]))
@@ -434,8 +442,12 @@ class Column(ParentColumn):
return Column(CastExpression(expr=self._expr, data_type=dataType))
else:
raise PySparkTypeError(
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "dataType",
+ "arg_type": type(dataType).__name__,
+ },
)
astype = cast
@@ -451,8 +463,12 @@ class Column(ParentColumn):
)
else:
raise PySparkTypeError(
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "dataType", "arg_type": type(dataType).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "dataType",
+ "arg_type": type(dataType).__name__,
+ },
)
def __repr__(self) -> str:
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index cf2c3b0de1a2..2cf9a692cf0d 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -223,13 +223,21 @@ class DataFrame(ParentDataFrame):
def _raise() -> NoReturn:
if allow_ordinal:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": arg_name, "arg_type": type(c).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": arg_name,
+ "arg_type": type(c).__name__,
+ },
)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": arg_name, "arg_type": type(c).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": arg_name,
+ "arg_type": type(c).__name__,
+ },
)
if (
@@ -417,8 +425,9 @@ class DataFrame(ParentDataFrame):
)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or str",
"arg_name": "numPartitions",
"arg_type": type(numPartitions).__name__,
},
@@ -460,8 +469,9 @@ class DataFrame(ParentDataFrame):
)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column, int or str",
"arg_name": "numPartitions",
"arg_type": type(numPartitions).__name__,
},
@@ -474,8 +484,9 @@ class DataFrame(ParentDataFrame):
) -> ParentDataFrame:
if not isinstance(numPartitions, int) or isinstance(numPartitions, bool):
raise PySparkTypeError(
- errorClass="NOT_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "int",
"arg_name": "numPartitions",
"arg_type": type(numPartitions).__name__,
},
@@ -563,8 +574,12 @@ class DataFrame(ParentDataFrame):
_cols = list(cols)
if any(not isinstance(c, (str, Column)) for c in _cols):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "cols", "arg_type": type(cols).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "cols",
+ "arg_type": type(cols).__name__,
+ },
)
return DataFrame(
@@ -798,8 +813,12 @@ class DataFrame(ParentDataFrame):
def withColumnsRenamed(self, colsMap: Dict[str, str]) -> ParentDataFrame:
if not isinstance(colsMap, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "colsMap", "arg_type": type(colsMap).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "colsMap",
+ "arg_type": type(colsMap).__name__,
+ },
)
return DataFrame(plan.WithColumnsRenamed(self._plan, colsMap), self._session)
@@ -809,8 +828,12 @@ class DataFrame(ParentDataFrame):
) -> str:
if not isinstance(n, int) or isinstance(n, bool):
raise PySparkTypeError(
- errorClass="NOT_INT",
- messageParameters={"arg_name": "n", "arg_type": type(n).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int",
+ "arg_name": "n",
+ "arg_type": type(n).__name__,
+ },
)
if not isinstance(vertical, bool):
raise PySparkTypeError(
@@ -856,8 +879,12 @@ class DataFrame(ParentDataFrame):
if not isinstance(colsMap, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "colsMap", "arg_type": type(colsMap).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "colsMap",
+ "arg_type": type(colsMap).__name__,
+ },
)
names: List[str] = []
@@ -897,8 +924,12 @@ class DataFrame(ParentDataFrame):
def withMetadata(self, columnName: str, metadata: Dict[str, Any]) -> ParentDataFrame:
if not isinstance(metadata, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "metadata", "arg_type": type(metadata).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "metadata",
+ "arg_type": type(metadata).__name__,
+ },
)
return DataFrame(
@@ -1180,8 +1211,12 @@ class DataFrame(ParentDataFrame):
def where(self, condition: Union[Column, str]) -> ParentDataFrame:
if not isinstance(condition, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "condition", "arg_type": type(condition).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "condition",
+ "arg_type": type(condition).__name__,
+ },
)
return self.filter(condition)
@@ -1284,8 +1319,12 @@ class DataFrame(ParentDataFrame):
if thresh is not None:
if not isinstance(thresh, int):
raise PySparkTypeError(
- errorClass="NOT_INT",
- messageParameters={"arg_name": "thresh", "arg_type": type(thresh).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int",
+ "arg_name": "thresh",
+ "arg_type": type(thresh).__name__,
+ },
)
# 'thresh' overwrites 'how'
@@ -1567,8 +1606,9 @@ class DataFrame(ParentDataFrame):
if not isinstance(relativeError, (float, int)):
raise PySparkTypeError(
- errorClass="NOT_FLOAT_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "float or int",
"arg_name": "relativeError",
"arg_type": type(relativeError).__name__,
},
@@ -1633,13 +1673,21 @@ class DataFrame(ParentDataFrame):
) -> ParentDataFrame:
if not isinstance(col, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": type(col).__name__,
+ },
)
if not isinstance(fractions, dict):
raise PySparkTypeError(
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "fractions", "arg_type": type(fractions).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "fractions",
+ "arg_type": type(fractions).__name__,
+ },
)
_fractions = []
@@ -1751,8 +1799,12 @@ class DataFrame(ParentDataFrame):
return F.col(self.columns[item])
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE",
- messageParameters={"arg_name": "item", "arg_type": type(item).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int, list, str or tuple",
+ "arg_name": "item",
+ "arg_type": type(item).__name__,
+ },
)
def _col(self, name: str, is_metadata_column: bool = False) -> Column:
@@ -2162,8 +2214,12 @@ class DataFrame(ParentDataFrame):
def sameSemantics(self, other: ParentDataFrame) -> bool:
if not isinstance(other, DataFrame):
raise PySparkTypeError(
- errorClass="NOT_DATAFRAME",
- messageParameters={"arg_name": "other", "arg_type": type(other).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataFrame",
+ "arg_name": "other",
+ "arg_type": type(other).__name__,
+ },
)
other = self._check_same_session(other)
return self._session.client.same_semantics(
diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py
index 187b29b3aaad..f8c254a8eb7f 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -88,8 +88,12 @@ def _to_col(col: "ColumnOrName") -> Column:
return column(col)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": type(col).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": type(col).__name__,
+ },
)
@@ -325,8 +329,12 @@ def broadcast(df: "DataFrame") -> "DataFrame":
if not isinstance(df, DataFrame):
raise PySparkTypeError(
- errorClass="NOT_DATAFRAME",
- messageParameters={"arg_name": "df", "arg_type": type(df).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataFrame",
+ "arg_name": "df",
+ "arg_type": type(df).__name__,
+ },
)
return df.hint("broadcast")
@@ -1288,8 +1296,9 @@ def percentile(
) -> Column:
if not isinstance(frequency, (int, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or int",
"arg_name": "frequency",
"arg_type": type(frequency).__name__,
},
@@ -1937,8 +1946,12 @@ def from_csv(
_schema = lit(schema)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "schema", "arg_type": type(schema).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "schema",
+ "arg_type": type(schema).__name__,
+ },
)
if options is None:
@@ -1989,8 +2002,12 @@ def from_xml(
_schema = lit(schema.json())
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR_OR_STRUCT",
- messageParameters={"arg_name": "schema", "arg_type": type(schema).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "StructType, Column or str",
+ "arg_name": "schema",
+ "arg_type": type(schema).__name__,
+ },
)
if options is None:
@@ -2245,8 +2262,12 @@ def schema_of_csv(csv: Union[str, Column], options: Optional[Mapping[str, str]]
csv = _enum_to_value(csv)
if not isinstance(csv, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "csv", "arg_type": type(csv).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "csv",
+ "arg_type": type(csv).__name__,
+ },
)
if options is None:
@@ -2262,8 +2283,12 @@ def schema_of_json(json: Union[str, Column], options: Optional[Mapping[str, str]
json = _enum_to_value(json)
if not isinstance(json, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "json", "arg_type": type(json).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "json",
+ "arg_type": type(json).__name__,
+ },
)
if options is None:
@@ -2279,8 +2304,12 @@ def schema_of_xml(xml: Union[str, Column], options: Optional[Mapping[str, str]]
xml = _enum_to_value(xml)
if not isinstance(xml, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "xml", "arg_type": type(xml).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "xml",
+ "arg_type": type(xml).__name__,
+ },
)
if options is None:
@@ -2317,8 +2346,12 @@ def slice(
_start = lit(start)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "start", "arg_type": type(start).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "start",
+ "arg_type": type(start).__name__,
+ },
)
length = _enum_to_value(length)
@@ -2328,8 +2361,12 @@ def slice(
_length = lit(length)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "length", "arg_type": type(length).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "length",
+ "arg_type": type(length).__name__,
+ },
)
return _invoke_function_over_columns("slice", x, _start, _length)
@@ -2575,14 +2612,22 @@ def overlay(
pos = _enum_to_value(pos)
if not isinstance(pos, (int, str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "pos", "arg_type": type(pos).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "pos",
+ "arg_type": type(pos).__name__,
+ },
)
len = _enum_to_value(len)
if len is not None and not isinstance(len, (int, str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "len", "arg_type": type(len).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "len",
+ "arg_type": type(len).__name__,
+ },
)
if isinstance(pos, int):
@@ -3743,8 +3788,12 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str])
gapDuration = _enum_to_value(gapDuration)
if gapDuration is None or not isinstance(gapDuration, (Column, str)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "gapDuration", "arg_type": type(gapDuration).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "gapDuration",
+ "arg_type": type(gapDuration).__name__,
+ },
)
time_col = _to_col(timeColumn)
@@ -3981,8 +4030,12 @@ time_to_micros.__doc__ = pysparkfuncs.time_to_micros.__doc__
def _ensure_column_or_name(arg: Optional[Any]) -> "ColumnOrName":
if not isinstance(arg, (Column, str)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "arg", "arg_type": type(arg).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "arg",
+ "arg_type": type(arg).__name__,
+ },
)
return arg
@@ -4403,8 +4456,12 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None
return _invoke_function_over_columns("assert_true", col)
if not isinstance(errMsg, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "errMsg", "arg_type": type(errMsg).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "errMsg",
+ "arg_type": type(errMsg).__name__,
+ },
)
return _invoke_function_over_columns("assert_true", col, lit(errMsg))
@@ -4416,8 +4473,12 @@ def raise_error(errMsg: Union[Column, str]) -> Column:
errMsg = _enum_to_value(errMsg)
if not isinstance(errMsg, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "errMsg", "arg_type": type(errMsg).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "errMsg",
+ "arg_type": type(errMsg).__name__,
+ },
)
return _invoke_function_over_columns("raise_error", lit(errMsg))
diff --git a/python/pyspark/sql/connect/functions/partitioning.py b/python/pyspark/sql/connect/functions/partitioning.py
index 6c8097621a60..b5fd0442318d 100644
--- a/python/pyspark/sql/connect/functions/partitioning.py
+++ b/python/pyspark/sql/connect/functions/partitioning.py
@@ -33,8 +33,9 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column:
_numBuckets = numBuckets
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or int",
"arg_name": "numBuckets",
"arg_type": type(numBuckets).__name__,
},
diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py
index 2c8a4ed7330b..d8ee49a0171a 100644
--- a/python/pyspark/sql/connect/group.py
+++ b/python/pyspark/sql/connect/group.py
@@ -235,8 +235,12 @@ class GroupedData:
if values is not None:
if not isinstance(values, list):
raise PySparkTypeError(
- errorClass="NOT_LIST",
- messageParameters={"arg_name": "values", "arg_type": type(values).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "list",
+ "arg_name": "values",
+ "arg_type": type(values).__name__,
+ },
)
for v in values:
if not isinstance(v, (bool, float, int, str)):
diff --git a/python/pyspark/sql/connect/readwriter.py b/python/pyspark/sql/connect/readwriter.py
index 923b719a4894..c4d227a9d74f 100644
--- a/python/pyspark/sql/connect/readwriter.py
+++ b/python/pyspark/sql/connect/readwriter.py
@@ -557,8 +557,9 @@ class DataFrameWriter(OptionUtils):
if isinstance(col, (list, tuple)):
if cols:
raise PySparkValueError(
- errorClass="NOT_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "int",
"arg_list": "numBuckets",
},
)
diff --git a/python/pyspark/sql/connect/udf.py b/python/pyspark/sql/connect/udf.py
index 355c1a5bac78..c9848634eb30 100644
--- a/python/pyspark/sql/connect/udf.py
+++ b/python/pyspark/sql/connect/udf.py
@@ -152,8 +152,9 @@ class UserDefinedFunction:
if not isinstance(returnType, (DataType, str)):
raise PySparkTypeError(
- errorClass="NOT_DATATYPE_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "DataType or str",
"arg_name": "returnType",
"arg_type": type(returnType).__name__,
},
@@ -161,8 +162,12 @@ class UserDefinedFunction:
if not isinstance(evalType, int):
raise PySparkTypeError(
- errorClass="NOT_INT",
- messageParameters={"arg_name": "evalType", "arg_type": type(evalType).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int",
+ "arg_name": "evalType",
+ "arg_type": type(evalType).__name__,
+ },
)
self.func = func
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index d9528384c703..b87eefd9ca4e 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -3217,8 +3217,9 @@ class DataFrame:
return _to_col(c)
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column, int or str",
"arg_name": "col",
"arg_type": type(c).__name__,
},
@@ -3233,8 +3234,12 @@ class DataFrame:
_cols = [c if asc else c.desc() for asc, c in zip(ascending, _cols)]
else:
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "ascending", "arg_type": type(ascending).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "ascending",
+ "arg_type": type(ascending).__name__,
+ },
)
return _cols
diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py
index 5f10b995f132..f6d5e64c3886 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -13358,8 +13358,12 @@ def session_window(timeColumn: "ColumnOrName", gapDuration: Union[Column, str])
def check_field(field: Union[Column, str], fieldName: str) -> None:
if field is None or not isinstance(field, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": fieldName, "arg_type": type(field).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": fieldName,
+ "arg_type": type(field).__name__,
+ },
)
time_col = _to_java_column(timeColumn)
@@ -14103,8 +14107,12 @@ def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None
return _invoke_function_over_columns("assert_true", col)
if not isinstance(errMsg, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "errMsg", "arg_type": type(errMsg).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "errMsg",
+ "arg_type": type(errMsg).__name__,
+ },
)
return _invoke_function_over_columns("assert_true", col, lit(errMsg))
@@ -14144,8 +14152,12 @@ def raise_error(errMsg: Union[Column, str]) -> Column:
errMsg = _enum_to_value(errMsg)
if not isinstance(errMsg, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "errMsg", "arg_type": type(errMsg).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "errMsg",
+ "arg_type": type(errMsg).__name__,
+ },
)
return _invoke_function_over_columns("raise_error", lit(errMsg))
@@ -15061,14 +15073,22 @@ def overlay(
pos = _enum_to_value(pos)
if not isinstance(pos, (int, str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "pos", "arg_type": type(pos).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "pos",
+ "arg_type": type(pos).__name__,
+ },
)
len = _enum_to_value(len)
if len is not None and not isinstance(len, (int, str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "len", "arg_type": type(len).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "len",
+ "arg_type": type(len).__name__,
+ },
)
if isinstance(pos, int):
@@ -21430,8 +21450,12 @@ def schema_of_json(json: Union[Column, str], options: Optional[Mapping[str, str]
json = _enum_to_value(json)
if not isinstance(json, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "json", "arg_type": type(json).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "json",
+ "arg_type": type(json).__name__,
+ },
)
return _invoke_function("schema_of_json", _to_java_column(lit(json)), _options_to_str(options))
@@ -21597,8 +21621,12 @@ def from_xml(
schema = _to_java_column(schema)
elif not isinstance(schema, str):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR_OR_STRUCT",
- messageParameters={"arg_name": "schema", "arg_type": type(schema).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "StructType, Column or str",
+ "arg_name": "schema",
+ "arg_type": type(schema).__name__,
+ },
)
return _invoke_function("from_xml", _to_java_column(col), schema, _options_to_str(options))
@@ -21687,8 +21715,12 @@ def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]]
xml = _enum_to_value(xml)
if not isinstance(xml, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "xml", "arg_type": type(xml).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "xml",
+ "arg_type": type(xml).__name__,
+ },
)
return _invoke_function("schema_of_xml", _to_java_column(lit(xml)), _options_to_str(options))
@@ -21822,8 +21854,12 @@ def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]]
csv = _enum_to_value(csv)
if not isinstance(csv, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "csv", "arg_type": type(csv).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "csv",
+ "arg_type": type(csv).__name__,
+ },
)
return _invoke_function("schema_of_csv", _to_java_column(lit(csv)), _options_to_str(options))
@@ -23371,8 +23407,12 @@ def from_csv(
if not isinstance(schema, (str, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "schema", "arg_type": type(schema).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "schema",
+ "arg_type": type(schema).__name__,
+ },
)
return _invoke_function(
@@ -25091,8 +25131,12 @@ def time_to_micros(col: "ColumnOrName") -> Column:
def _ensure_column_or_name(arg: Optional[Any]) -> "ColumnOrName":
if not isinstance(arg, (Column, str)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "arg", "arg_type": type(arg).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "arg",
+ "arg_type": type(arg).__name__,
+ },
)
return arg
diff --git a/python/pyspark/sql/functions/partitioning.py b/python/pyspark/sql/functions/partitioning.py
index 3bf7745e49c5..74f1ea8126be 100644
--- a/python/pyspark/sql/functions/partitioning.py
+++ b/python/pyspark/sql/functions/partitioning.py
@@ -210,8 +210,9 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column:
if not isinstance(numBuckets, (int, Column)):
raise PySparkTypeError(
- errorClass="NOT_COLUMN_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or int",
"arg_name": "numBuckets",
"arg_type": type(numBuckets).__name__,
},
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index bb8e51b60134..9f0e2656aefc 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -1509,8 +1509,9 @@ class DataFrameWriter(OptionUtils):
if not isinstance(numBuckets, int):
raise PySparkTypeError(
- errorClass="NOT_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "int",
"arg_name": "numBuckets",
"arg_type": type(numBuckets).__name__,
},
diff --git a/python/pyspark/sql/streaming/state.py b/python/pyspark/sql/streaming/state.py
index cd067a8413e1..7db667c4314a 100644
--- a/python/pyspark/sql/streaming/state.py
+++ b/python/pyspark/sql/streaming/state.py
@@ -178,8 +178,9 @@ class GroupState:
if isinstance(durationMs, str):
# TODO(SPARK-40437): Support string representation of durationMs.
raise PySparkTypeError(
- errorClass="NOT_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "int",
"arg_name": "durationMs",
"arg_type": type(durationMs).__name__,
},
diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py
index 8fee80fe0c05..5749a380ae00 100755
--- a/python/pyspark/sql/tests/connect/test_connect_basic.py
+++ b/python/pyspark/sql/tests/connect/test_connect_basic.py
@@ -248,8 +248,9 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column, int, list, str or tuple",
"arg_name": "item",
"arg_type": "float",
},
@@ -260,8 +261,9 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column, int, list, str or tuple",
"arg_name": "item",
"arg_type": "NoneType",
},
@@ -272,8 +274,9 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_LIST_OR_STR_OR_TUPLE",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column, int, list, str or tuple",
"arg_name": "item",
"arg_type": "DataFrame",
},
@@ -1270,8 +1273,9 @@ class SparkConnectBasicTests(SparkConnectSQLTestCase):
self.check_error(
exception=pe.exception,
- errorClass="NOT_DICT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "dict",
"arg_name": "metadata",
"arg_type": "list",
},
diff --git a/python/pyspark/sql/tests/connect/test_connect_column.py b/python/pyspark/sql/tests/connect/test_connect_column.py
index 7dee614ddaa7..2cdcba5406db 100644
--- a/python/pyspark/sql/tests/connect/test_connect_column.py
+++ b/python/pyspark/sql/tests/connect/test_connect_column.py
@@ -137,8 +137,9 @@ class SparkConnectColumnTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or int",
"arg_name": "startPos",
"arg_type": "float",
},
@@ -554,8 +555,12 @@ class SparkConnectColumnTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
self.check_error(
exception=pe.exception,
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "dataType", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "dataType",
+ "arg_type": "int",
+ },
)
def test_isin(self):
diff --git a/python/pyspark/sql/tests/connect/test_connect_error.py b/python/pyspark/sql/tests/connect/test_connect_error.py
index c8f203c71e6d..2213d1b1ed29 100644
--- a/python/pyspark/sql/tests/connect/test_connect_error.py
+++ b/python/pyspark/sql/tests/connect/test_connect_error.py
@@ -232,8 +232,12 @@ class SparkConnectErrorTests(ReusedConnectTestCase):
self.check_error(
exception=e1.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "cols", "arg_type": "NoneType"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "cols",
+ "arg_type": "NoneType",
+ },
)
def test_ym_interval_in_collect(self):
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py b/python/pyspark/sql/tests/connect/test_connect_function.py
index dbdc745c25ae..db9b7319c15c 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -143,8 +143,12 @@ class SparkConnectFunctionTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
self.check_error(
exception=pe.exception,
- errorClass="NOT_DATAFRAME",
- messageParameters={"arg_name": "df", "arg_type": "Column"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataFrame",
+ "arg_name": "df",
+ "arg_type": "Column",
+ },
)
def test_normal_functions(self):
@@ -1217,8 +1221,12 @@ class SparkConnectFunctionTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "start", "arg_type": "float"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "start",
+ "arg_type": "float",
+ },
)
with self.assertRaises(PySparkTypeError) as pe:
@@ -1226,8 +1234,12 @@ class SparkConnectFunctionTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "length", "arg_type": "float"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "length",
+ "arg_type": "float",
+ },
)
# test sort_array
@@ -1984,8 +1996,12 @@ class SparkConnectFunctionTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR_OR_STRUCT",
- messageParameters={"arg_name": "schema", "arg_type": "list"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "StructType, Column or str",
+ "arg_name": "schema",
+ "arg_type": "list",
+ },
)
# test schema_of_xml
diff --git a/python/pyspark/sql/tests/connect/test_connect_stat.py b/python/pyspark/sql/tests/connect/test_connect_stat.py
index 1b432f7b3cf7..9123b7a10f01 100644
--- a/python/pyspark/sql/tests/connect/test_connect_stat.py
+++ b/python/pyspark/sql/tests/connect/test_connect_stat.py
@@ -275,8 +275,9 @@ class SparkConnectStatTests(SparkConnectSQLTestCase):
self.check_error(
exception=pe.exception,
- errorClass="NOT_FLOAT_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "float or int",
"arg_name": "relativeError",
"arg_type": "str",
},
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf.py b/python/pyspark/sql/tests/pandas/test_pandas_udf.py
index 752ec743d412..ae9d231d9136 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf.py
@@ -244,8 +244,12 @@ class PandasUDFTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "returnType", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "returnType",
+ "arg_type": "int",
+ },
)
def test_stopiteration_in_udf(self):
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index aa650187f742..d6b0c6eca279 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -58,8 +58,12 @@ class ColumnTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": "int",
+ },
)
class A:
@@ -73,8 +77,12 @@ class ColumnTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": "NoneType"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": "NoneType",
+ },
)
self.assertRaises(TypeError, lambda: to_json(1))
@@ -293,8 +301,12 @@ class ColumnTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "dataType", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "dataType",
+ "arg_type": "int",
+ },
)
def test_over_negative(self):
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 9fb767b2ed66..725c0850ad1a 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -311,8 +311,8 @@ class DataFrameTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "colsMap", "arg_type": "tuple"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={"expected_type": "dict", "arg_name": "colsMap", "arg_type": "tuple"},
)
def test_with_columns_renamed_with_duplicated_names(self):
@@ -801,8 +801,12 @@ class DataFrameTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_DATAFRAME",
- messageParameters={"arg_name": "other", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataFrame",
+ "arg_name": "other",
+ "arg_type": "int",
+ },
)
def test_input_files(self):
@@ -837,8 +841,8 @@ class DataFrameTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_INT",
- messageParameters={"arg_name": "n", "arg_type": "bool"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={"expected_type": "int", "arg_name": "n", "arg_type": "bool"},
)
with self.assertRaises(PySparkTypeError) as pe:
@@ -1006,8 +1010,12 @@ class DataFrameTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "condition", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "condition",
+ "arg_type": "int",
+ },
)
def test_duplicate_field_names(self):
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index 2de49d1f4fc6..33c109d60c0c 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -236,8 +236,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "col", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "col",
+ "arg_type": "int",
+ },
)
with self.assertRaises(PySparkTypeError) as pe:
@@ -245,8 +249,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_DICT",
- messageParameters={"arg_name": "fractions", "arg_type": "list"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "dict",
+ "arg_name": "fractions",
+ "arg_type": "list",
+ },
)
with self.assertRaises(PySparkTypeError) as pe:
@@ -916,8 +924,9 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "Column or int",
"arg_name": "startPos",
"arg_type": "str",
},
@@ -1709,8 +1718,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "pos", "arg_type": "float"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "pos",
+ "arg_type": "float",
+ },
)
with self.assertRaises(PySparkTypeError) as pe:
@@ -1718,8 +1731,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "len", "arg_type": "float"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "len",
+ "arg_type": "float",
+ },
)
def test_percentile(self):
@@ -3127,8 +3144,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "errMsg", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "errMsg",
+ "arg_type": "int",
+ },
)
def test_raise_error(self):
@@ -3148,8 +3169,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "errMsg", "arg_type": "NoneType"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "errMsg",
+ "arg_type": "NoneType",
+ },
)
def test_sum_distinct(self):
@@ -3437,8 +3462,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "json", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "json",
+ "arg_type": "int",
+ },
)
def test_try_parse_json(self):
@@ -3491,8 +3520,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "csv", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "csv",
+ "arg_type": "int",
+ },
)
def test_from_csv(self):
@@ -3502,8 +3535,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "schema", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "schema",
+ "arg_type": "int",
+ },
)
def test_schema_of_xml(self):
@@ -3512,8 +3549,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "xml", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "xml",
+ "arg_type": "int",
+ },
)
def test_from_xml(self):
@@ -3523,8 +3564,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR_OR_STRUCT",
- messageParameters={"arg_name": "schema", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "StructType, Column or str",
+ "arg_name": "schema",
+ "arg_type": "int",
+ },
)
def test_greatest(self):
@@ -3568,8 +3613,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "gapDuration", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "gapDuration",
+ "arg_type": "int",
+ },
)
def test_current_user(self):
@@ -3587,8 +3636,12 @@ class FunctionsTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT",
- messageParameters={"arg_name": "numBuckets", "arg_type": "str"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or int",
+ "arg_name": "numBuckets",
+ "arg_type": "str",
+ },
)
def test_to_time(self):
diff --git a/python/pyspark/sql/tests/test_repartition.py b/python/pyspark/sql/tests/test_repartition.py
index 2e9cfdced92b..707d58dba331 100644
--- a/python/pyspark/sql/tests/test_repartition.py
+++ b/python/pyspark/sql/tests/test_repartition.py
@@ -36,8 +36,12 @@ class DataFrameRepartitionTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_STR",
- messageParameters={"arg_name": "numPartitions", "arg_type": "list"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column or str",
+ "arg_name": "numPartitions",
+ "arg_type": "list",
+ },
)
def test_repartition_by_range(self):
@@ -79,8 +83,12 @@ class DataFrameRepartitionTestsMixin:
self.check_error(
exception=pe.exception,
- errorClass="NOT_COLUMN_OR_INT_OR_STR",
- messageParameters={"arg_name": "numPartitions", "arg_type": "list"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "Column, int or str",
+ "arg_name": "numPartitions",
+ "arg_type": "list",
+ },
)
def test_repartition_by_id(self):
@@ -151,8 +159,12 @@ class DataFrameRepartitionTestsMixin:
df.repartitionById("5", col("id").cast("int"))
self.check_error(
exception=pe.exception,
- errorClass="NOT_INT",
- messageParameters={"arg_name": "numPartitions", "arg_type": "str"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int",
+ "arg_name": "numPartitions",
+ "arg_type": "str",
+ },
)
with self.assertRaises(PySparkValueError) as pe:
diff --git a/python/pyspark/sql/tests/test_udf.py b/python/pyspark/sql/tests/test_udf.py
index 21dcb162d1e7..42a24d08b5ec 100644
--- a/python/pyspark/sql/tests/test_udf.py
+++ b/python/pyspark/sql/tests/test_udf.py
@@ -1277,8 +1277,12 @@ class BaseUDFTestsMixin(object):
self.check_error(
exception=pe.exception,
- errorClass="NOT_DATATYPE_OR_STR",
- messageParameters={"arg_name": "returnType", "arg_type": "int"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "DataType or str",
+ "arg_name": "returnType",
+ "arg_type": "int",
+ },
)
with self.assertRaises(PySparkTypeError) as pe:
@@ -1286,8 +1290,8 @@ class BaseUDFTestsMixin(object):
self.check_error(
exception=pe.exception,
- errorClass="NOT_INT",
- messageParameters={"arg_name": "evalType", "arg_type": "str"},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={"expected_type": "int", "arg_name": "evalType", "arg_type": "str"},
)
def test_timeout_util_with_udf(self):
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 4114b3006c82..f83db6aa825f 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1587,8 +1587,12 @@ class StructType(DataType):
return StructType(self.fields[key])
else:
raise PySparkTypeError(
- errorClass="NOT_INT_OR_SLICE_OR_STR",
- messageParameters={"arg_name": "key", "arg_type": type(key).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int, slice or str",
+ "arg_name": "key",
+ "arg_type": type(key).__name__,
+ },
)
def simpleString(self) -> str:
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index d046a3210178..e586c560deeb 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -178,8 +178,9 @@ class UserDefinedFunction:
if not isinstance(returnType, (DataType, str)):
raise PySparkTypeError(
- errorClass="NOT_DATATYPE_OR_STR",
+ errorClass="NOT_EXPECTED_TYPE",
messageParameters={
+ "expected_type": "DataType or str",
"arg_name": "returnType",
"arg_type": type(returnType).__name__,
},
@@ -187,8 +188,12 @@ class UserDefinedFunction:
if not isinstance(evalType, int):
raise PySparkTypeError(
- errorClass="NOT_INT",
- messageParameters={"arg_name": "evalType", "arg_type": type(evalType).__name__},
+ errorClass="NOT_EXPECTED_TYPE",
+ messageParameters={
+ "expected_type": "int",
+ "arg_name": "evalType",
+ "arg_type": type(evalType).__name__,
+ },
)
self.func = func
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org