You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2021/01/13 00:57:43 UTC
[spark] branch branch-3.1 updated:
[SPARK-32338][SQL][PYSPARK][FOLLOW-UP][TEST] Add more tests for slice
function
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 5dd94c1 [SPARK-32338][SQL][PYSPARK][FOLLOW-UP][TEST] Add more tests for slice function
5dd94c1 is described below
commit 5dd94c175d20e1a9614bd5bce9449901dbbfff60
Author: Takuya UESHIN <ue...@databricks.com>
AuthorDate: Wed Jan 13 09:56:38 2021 +0900
[SPARK-32338][SQL][PYSPARK][FOLLOW-UP][TEST] Add more tests for slice function
### What changes were proposed in this pull request?
This PR is a follow-up of #29138 and #29195 to add more tests for `slice` function.
### Why are the changes needed?
The original PRs are missing tests with column-based arguments instead of literals.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added tests and existing tests.
Closes #31159 from ueshin/issues/SPARK-32338/slice_tests.
Authored-by: Takuya UESHIN <ue...@databricks.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit ad8e40e2ab80d0fb5a88fcc991c15267316b57d7)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
python/pyspark/sql/tests/test_functions.py | 11 ++++++++++-
.../scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala | 8 ++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/sql/tests/test_functions.py b/python/pyspark/sql/tests/test_functions.py
index 58599a9..053164a 100644
--- a/python/pyspark/sql/tests/test_functions.py
+++ b/python/pyspark/sql/tests/test_functions.py
@@ -350,7 +350,7 @@ class FunctionsTests(ReusedSQLTestCase):
self.assertEqual(result[0], '')
def test_slice(self):
- from pyspark.sql.functions import slice, lit
+ from pyspark.sql.functions import lit, size, slice
df = self.spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x'])
@@ -359,6 +359,15 @@ class FunctionsTests(ReusedSQLTestCase):
df.select(slice(df.x, lit(2), lit(2)).alias("sliced")).collect(),
)
+ self.assertEqual(
+ df.select(slice(df.x, size(df.x) - 1, lit(1)).alias("sliced")).collect(),
+ [Row(sliced=[2]), Row(sliced=[4])]
+ )
+ self.assertEqual(
+ df.select(slice(df.x, lit(1), size(df.x) - 1).alias("sliced")).collect(),
+ [Row(sliced=[1, 2]), Row(sliced=[4])]
+ )
+
def test_array_repeat(self):
from pyspark.sql.functions import array_repeat, lit
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index 585a835..aa1678e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -943,6 +943,14 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
checkAnswer(df.select(slice(df("x"), -1, 1)), answerNegative)
checkAnswer(df.select(slice(df("x"), lit(-1), lit(1))), answerNegative)
checkAnswer(df.selectExpr("slice(x, -1, 1)"), answerNegative)
+
+ val answerStartExpr = Seq(Row(Seq(2)), Row(Seq(4)))
+ checkAnswer(df.select(slice(df("x"), size($"x") - 1, lit(1))), answerStartExpr)
+ checkAnswer(df.selectExpr("slice(x, size(x) - 1, 1)"), answerStartExpr)
+
+ val answerLengthExpr = Seq(Row(Seq(1, 2)), Row(Seq(4)))
+ checkAnswer(df.select(slice(df("x"), lit(1), size($"x") - 1)), answerLengthExpr)
+ checkAnswer(df.selectExpr("slice(x, 1, size(x) - 1)"), answerLengthExpr)
}
test("array_join function") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org