You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/12/07 22:43:18 UTC
(spark) branch master updated: [SPARK-54623][PYTHON][TEST] Add coverage test for UDTF null checker
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 44efa48dd95e [SPARK-54623][PYTHON][TEST] Add coverage test for UDTF null checker
44efa48dd95e is described below
commit 44efa48dd95e7c6a3f3c664566ec1dd4154b7851
Author: Tian Gao <ga...@hotmail.com>
AuthorDate: Mon Dec 8 07:43:06 2025 +0900
[SPARK-54623][PYTHON][TEST] Add coverage test for UDTF null checker
### What changes were proposed in this pull request?
Coverage tests are added for the null checker in UDTF.
### Why are the changes needed?
Improve test coverage and I need to refactor the code later so we need to make sure there's no regression.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
The test passed locally.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53362 from gaogaotiantian/test-null-checker.
Authored-by: Tian Gao <ga...@hotmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/tests/test_udtf.py | 105 ++++++++++++++++++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git a/python/pyspark/sql/tests/test_udtf.py b/python/pyspark/sql/tests/test_udtf.py
index 72bef492e23c..eeb07400b060 100644
--- a/python/pyspark/sql/tests/test_udtf.py
+++ b/python/pyspark/sql/tests/test_udtf.py
@@ -812,6 +812,111 @@ class BaseUDTFTestsMixin:
with self.assertRaisesRegex(err_type, expected):
func().collect()
+ def test_udtf_nullable_check(self):
+ for ret_type, value, expected in [
+ (
+ StructType([StructField("value", ArrayType(IntegerType(), False))]),
+ ([None],),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType([StructField("value", ArrayType(IntegerType(), True))]),
+ ([None],),
+ [Row(value=[None])],
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(), IntegerType(), False))]),
+ ({"a": None},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(), IntegerType(), True))]),
+ ({"a": None},),
+ [Row(value={"a": None})],
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(), IntegerType(), True))]),
+ ({None: 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType([StructField("value", MapType(StringType(), IntegerType(), False))]),
+ ({None: 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value", MapType(StringType(), ArrayType(IntegerType(), False), False)
+ )
+ ]
+ ),
+ ({"s": [None]},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value",
+ MapType(
+ StructType([StructField("value", StringType(), False)]),
+ IntegerType(),
+ False,
+ ),
+ )
+ ]
+ ),
+ ({(None,): 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value",
+ MapType(
+ StructType([StructField("value", StringType(), False)]),
+ IntegerType(),
+ True,
+ ),
+ )
+ ]
+ ),
+ ({(None,): 1},),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [StructField("value", StructType([StructField("value", StringType(), False)]))]
+ ),
+ ((None,),),
+ "PySparkRuntimeError",
+ ),
+ (
+ StructType(
+ [
+ StructField(
+ "value",
+ StructType(
+ [StructField("value", ArrayType(StringType(), False), False)]
+ ),
+ )
+ ]
+ ),
+ (([None],),),
+ "PySparkRuntimeError",
+ ),
+ ]:
+
+ class TestUDTF:
+ def eval(self):
+ yield value
+
+ with self.subTest(ret_type=ret_type, value=value):
+ self._check_result_or_exception(TestUDTF, ret_type, expected)
+
def test_numeric_output_type_casting(self):
class TestUDTF:
def eval(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org