You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/02/14 12:57:56 UTC
[spark] branch master updated: [SPARK-42434][PYTHON][CONNECT] array_append` should accept `Any` value
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6e090eba4fd [SPARK-42434][PYTHON][CONNECT] array_append` should accept `Any` value
6e090eba4fd is described below
commit 6e090eba4fd0f4f841177ffeefc3a376242075a2
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue Feb 14 21:57:42 2023 +0900
[SPARK-42434][PYTHON][CONNECT] array_append` should accept `Any` value
### What changes were proposed in this pull request?
`array_append` should accept `Any` value
### Why are the changes needed?
make `array_append` consistent with other array functions in pyspark
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
added UT
Closes #40012 from zhengruifeng/py_array_append.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/connect/functions.py | 4 ++--
python/pyspark/sql/functions.py | 12 +++++++-----
python/pyspark/sql/tests/connect/test_connect_function.py | 4 ++++
3 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/python/pyspark/sql/connect/functions.py b/python/pyspark/sql/connect/functions.py
index e5305938797..7955f8932aa 100644
--- a/python/pyspark/sql/connect/functions.py
+++ b/python/pyspark/sql/connect/functions.py
@@ -1156,8 +1156,8 @@ def array(*cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName
array.__doc__ = pysparkfuncs.array.__doc__
-def array_append(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
- return _invoke_function_over_columns("array_append", col1, col2)
+def array_append(col: "ColumnOrName", value: Any) -> Column:
+ return _invoke_function("array_append", _to_col(col), lit(value))
array_append.__doc__ = pysparkfuncs.array_append.__doc__
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 39a3f036cc3..bfedf473b93 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -7839,7 +7839,7 @@ def array_compact(col: "ColumnOrName") -> Column:
@try_remote_functions
-def array_append(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
+def array_append(col: "ColumnOrName", value: Any) -> Column:
"""
Collection function: returns an array of the elements in col1 along
with the added element in col2 at the last of the array.
@@ -7851,10 +7851,10 @@ def array_append(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
Parameters
----------
- col1 : :class:`~pyspark.sql.Column` or str
+ col : :class:`~pyspark.sql.Column` or str
name of column containing array
- col2 : :class:`~pyspark.sql.Column` or str
- name of column containing element
+ value :
+ a literal value, or a :class:`~pyspark.sql.Column` expression.
Returns
-------
@@ -7867,8 +7867,10 @@ def array_append(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
>>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")])
>>> df.select(array_append(df.c1, df.c2)).collect()
[Row(array_append(c1, c2)=['b', 'a', 'c', 'c'])]
+ >>> df.select(array_append(df.c1, 'x')).collect()
+ [Row(array_append(c1, x)=['b', 'a', 'c', 'x'])]
"""
- return _invoke_function_over_columns("array_append", col1, col2)
+ return _invoke_function_over_columns("array_append", col, lit(value))
@try_remote_functions
diff --git a/python/pyspark/sql/tests/connect/test_connect_function.py b/python/pyspark/sql/tests/connect/test_connect_function.py
index 243153be083..fbf2eab445a 100644
--- a/python/pyspark/sql/tests/connect/test_connect_function.py
+++ b/python/pyspark/sql/tests/connect/test_connect_function.py
@@ -1072,6 +1072,10 @@ class SparkConnectFunctionTests(ReusedConnectTestCase, PandasOnSparkTestUtils, S
)
# test array_append
+ self.assert_eq(
+ cdf.select(CF.array_append(cdf.a, "xyz")).toPandas(),
+ sdf.select(SF.array_append(sdf.a, "xyz")).toPandas(),
+ )
self.assert_eq(
cdf.select(CF.array_append(cdf.a, CF.lit("ab"))).toPandas(),
sdf.select(SF.array_append(sdf.a, SF.lit("ab"))).toPandas(),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org