You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/10/21 23:21:57 UTC
(spark) branch master updated: [SPARK-50052][PYTHON] Make NumpyArrayConverter support empty str ndarray
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 91ae1022198b [SPARK-50052][PYTHON] Make NumpyArrayConverter support empty str ndarray
91ae1022198b is described below
commit 91ae1022198ba4f5ed3204c7dd959c60f76ad3a4
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue Oct 22 08:21:42 2024 +0900
[SPARK-50052][PYTHON] Make NumpyArrayConverter support empty str ndarray
### What changes were proposed in this pull request?
Make `NumpyArrayConverter` support empty str ndarray
### Why are the changes needed?
existing implementation needs the str ndarray to be non-empty, so fails:
```
In [6]: spark.range(1).select(sf.lit(np.array(["a", "b"], np.str_)))
Out[6]: DataFrame[ARRAY('a', 'b'): array<string>]
In [7]: spark.range(1).select(sf.lit(np.array([], np.int32)))
Out[7]: DataFrame[ARRAY(): array<int>]
In [8]: spark.range(1).select(sf.lit(np.array([], np.str_)))
...
PySparkTypeError: [UNSUPPORTED_NUMPY_ARRAY_SCALAR] The type of array scalar '<U1' is not supported.
```
### Does this PR introduce _any_ user-facing change?
support empty string array
```
In [3]: spark.range(1).select(sf.lit(np.array([], np.str_)))
Out[3]: DataFrame[ARRAY(): array<string>]
```
### How was this patch tested?
added test
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #48575 from zhengruifeng/classic_np_array_conv.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../sql/tests/connect/test_parity_functions.py | 8 ++++++++
python/pyspark/sql/tests/test_functions.py | 23 ++++++++++++++++++++++
python/pyspark/sql/types.py | 17 ++++++++--------
3 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_functions.py b/python/pyspark/sql/tests/connect/test_parity_functions.py
index 0a77c5531082..2004c065659b 100644
--- a/python/pyspark/sql/tests/connect/test_parity_functions.py
+++ b/python/pyspark/sql/tests/connect/test_parity_functions.py
@@ -34,6 +34,14 @@ class FunctionsParityTests(FunctionsTestsMixin, ReusedConnectTestCase):
def test_input_file_name_reset_for_rdd(self):
super().test_input_file_name_reset_for_rdd()
+ @unittest.skip("SPARK-50050: Spark Connect should support str ndarray.")
+ def test_str_ndarray(self):
+ super().test_str_ndarray()
+
+ @unittest.skip("SPARK-50051: Spark Connect should empty ndarray.")
+ def test_empty_ndarray(self):
+ super().test_empty_ndarray()
+
if __name__ == "__main__":
from pyspark.sql.tests.connect.test_parity_functions import * # noqa: F401
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index cec6e2ababbd..657f0b337156 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -1263,6 +1263,29 @@ class FunctionsTestsMixin:
},
)
+ @unittest.skipIf(not have_numpy, "NumPy not installed")
+ def test_str_ndarray(self):
+ import numpy as np
+
+ for arr in [
+ np.array([], np.str_),
+ np.array(["a"], np.str_),
+ np.array([1, 2, 3], np.str_),
+ ]:
+ self.assertEqual(
+ [("a", "array<string>")],
+ self.spark.range(1).select(F.lit(arr).alias("a")).dtypes,
+ )
+
+ @unittest.skipIf(not have_numpy, "NumPy not installed")
+ def test_empty_ndarray(self):
+ import numpy as np
+
+ self.assertEqual(
+ [("a", "array<int>")],
+ self.spark.range(1).select(F.lit(np.array([], np.int32)).alias("a")).dtypes,
+ )
+
def test_binary_math_function(self):
funcs, expected = zip(
*[(F.atan2, 0.13664), (F.hypot, 8.07527), (F.pow, 2.14359), (F.pmod, 1.1)]
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index e6e2267af0ce..0f77e95cced1 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -3273,6 +3273,8 @@ class NumpyArrayConverter:
return gateway.jvm.double
elif nt == np.dtype("bool"):
return gateway.jvm.boolean
+ elif np.isdtype(nt, np.str_):
+ return gateway.jvm.String
return None
@@ -3286,15 +3288,12 @@ class NumpyArrayConverter:
assert gateway is not None
plist = obj.tolist()
- if len(obj) > 0 and isinstance(plist[0], str):
- jtpe = gateway.jvm.String
- else:
- jtpe = self._from_numpy_type_to_java_type(obj.dtype, gateway)
- if jtpe is None:
- raise PySparkTypeError(
- errorClass="UNSUPPORTED_NUMPY_ARRAY_SCALAR",
- messageParameters={"dtype": str(obj.dtype)},
- )
+ jtpe = self._from_numpy_type_to_java_type(obj.dtype, gateway)
+ if jtpe is None:
+ raise PySparkTypeError(
+ errorClass="UNSUPPORTED_NUMPY_ARRAY_SCALAR",
+ messageParameters={"dtype": str(obj.dtype)},
+ )
jarr = gateway.new_array(jtpe, len(obj))
for i in range(len(plist)):
jarr[i] = plist[i]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org