You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2024/05/07 01:15:48 UTC
(spark) branch master updated: [SPARK-48142][PYTHON][CONNECT][TESTS] Enable `CogroupedApplyInPandasTests.test_wrong_args`
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2ef7246b9c5b [SPARK-48142][PYTHON][CONNECT][TESTS] Enable `CogroupedApplyInPandasTests.test_wrong_args`
2ef7246b9c5b is described below
commit 2ef7246b9c5b39b16cf9a37d7fc84a233362967c
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue May 7 09:15:31 2024 +0800
[SPARK-48142][PYTHON][CONNECT][TESTS] Enable `CogroupedApplyInPandasTests.test_wrong_args`
### What changes were proposed in this pull request?
Enable `CogroupedApplyInPandasTests.test_wrong_args` by including a missing check
### Why are the changes needed?
for test coverage
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #46397 from zhengruifeng/fix_pandas_udf_check.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/sql/connect/group.py | 2 ++
python/pyspark/sql/pandas/functions.py | 9 ++++++++-
.../sql/tests/connect/test_parity_pandas_cogrouped_map.py | 9 +++++----
3 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py
index 699dce76c4a1..c916e8acf3e4 100644
--- a/python/pyspark/sql/connect/group.py
+++ b/python/pyspark/sql/connect/group.py
@@ -398,7 +398,9 @@ class PandasCogroupedOps:
) -> "DataFrame":
from pyspark.sql.connect.udf import UserDefinedFunction
from pyspark.sql.connect.dataframe import DataFrame
+ from pyspark.sql.pandas.functions import _validate_pandas_udf # type: ignore[attr-defined]
+ _validate_pandas_udf(func, schema, PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF)
udf_obj = UserDefinedFunction(
func,
returnType=schema,
diff --git a/python/pyspark/sql/pandas/functions.py b/python/pyspark/sql/pandas/functions.py
index 62d365a3b2a1..5922a5ced863 100644
--- a/python/pyspark/sql/pandas/functions.py
+++ b/python/pyspark/sql/pandas/functions.py
@@ -431,7 +431,8 @@ def pandas_udf(f=None, returnType=None, functionType=None):
return _create_pandas_udf(f=f, returnType=return_type, evalType=eval_type)
-def _create_pandas_udf(f, returnType, evalType):
+# validate the pandas udf and return the adjusted eval type
+def _validate_pandas_udf(f, returnType, evalType) -> int:
argspec = getfullargspec(f)
# pandas UDF by type hints.
@@ -528,6 +529,12 @@ def _create_pandas_udf(f, returnType, evalType):
},
)
+ return evalType
+
+
+def _create_pandas_udf(f, returnType, evalType):
+ evalType = _validate_pandas_udf(f, returnType, evalType)
+
if is_remote():
from pyspark.sql.connect.udf import _create_udf as _create_connect_udf
diff --git a/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py b/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py
index 708960dd47d4..00d71bda2d93 100644
--- a/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py
@@ -20,10 +20,11 @@ from pyspark.sql.tests.pandas.test_pandas_cogrouped_map import CogroupedApplyInP
from pyspark.testing.connectutils import ReusedConnectTestCase
-class CogroupedApplyInPandasTests(CogroupedApplyInPandasTestsMixin, ReusedConnectTestCase):
- @unittest.skip("Fails in Spark Connect, should enable.")
- def test_wrong_args(self):
- self.check_wrong_args()
+class CogroupedApplyInPandasTests(
+ CogroupedApplyInPandasTestsMixin,
+ ReusedConnectTestCase,
+):
+ pass
if __name__ == "__main__":
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org