You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/06/10 02:18:35 UTC
[spark] branch master updated: [SPARK-43916][CONNECT][TESTS] Adds the test case that percentile with frequency for Scala API
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c30b411e412 [SPARK-43916][CONNECT][TESTS] Adds the test case that percentile with frequency for Scala API
c30b411e412 is described below
commit c30b411e412a76bf85d6e438e6b0c39014476a0c
Author: Jiaan Geng <be...@163.com>
AuthorDate: Sat Jun 10 10:18:07 2023 +0800
[SPARK-43916][CONNECT][TESTS] Adds the test case that percentile with frequency for Scala API
### What changes were proposed in this pull request?
https://github.com/apache/spark/pull/41444 added `percentile` to Scala, Python and Connect API.
But missing the test case that `percentile` with `frequency` for Scala API.
### Why are the changes needed?
Adds the test case that `percentile` with `frequency` for Scala API.
### Does this PR introduce _any_ user-facing change?
'No'.
New feature.
### How was this patch tested?
N/A
Closes #41538 from beliefer/SPARK-43916_followup.
Authored-by: Jiaan Geng <be...@163.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../apache/spark/sql/PlanGenerationTestSuite.scala | 6 +++-
.../function_percentile_with_frequency.explain | 2 ++
.../function_percentile_without_frequency.explain | 2 ++
.../function_percentile_with_frequency.json | 33 +++++++++++++++++++++
.../function_percentile_with_frequency.proto.bin | Bin 0 -> 198 bytes
.../function_percentile_without_frequency.json | 29 ++++++++++++++++++
...function_percentile_without_frequency.proto.bin | Bin 0 -> 192 bytes
7 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 109219603b9..18223c56588 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -986,10 +986,14 @@ class PlanGenerationTestSuite
fn.min_by(fn.col("a"), fn.col("b"))
}
- functionTest("percentile") {
+ functionTest("percentile without frequency") {
fn.percentile(fn.col("a"), fn.lit(0.3))
}
+ functionTest("percentile with frequency") {
+ fn.percentile(fn.col("a"), fn.lit(0.3), fn.lit(2))
+ }
+
functionTest("percentile_approx") {
fn.percentile_approx(fn.col("a"), fn.lit(0.3), fn.lit(20))
}
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_percentile_with_frequency.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_percentile_with_frequency.explain
new file mode 100644
index 00000000000..263c6d0bac9
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_percentile_with_frequency.explain
@@ -0,0 +1,2 @@
+Aggregate [percentile(a#0, 0.3, 2, 0, 0, false) AS percentile(a, 0.3, 2)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_percentile_without_frequency.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_percentile_without_frequency.explain
new file mode 100644
index 00000000000..901eaf745ef
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_percentile_without_frequency.explain
@@ -0,0 +1,2 @@
+Aggregate [percentile(a#0, 0.3, 1, 0, 0, false) AS percentile(a, 0.3, 1)#0]
++- LocalRelation <empty>, [id#0L, a#0, b#0, d#0, e#0, f#0, g#0]
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json
new file mode 100644
index 00000000000..f5780442664
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.json
@@ -0,0 +1,33 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "percentile",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "a"
+ }
+ }, {
+ "literal": {
+ "double": 0.3
+ }
+ }, {
+ "literal": {
+ "integer": 2
+ }
+ }]
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin
new file mode 100644
index 00000000000..91d6279f9bd
Binary files /dev/null and b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_with_frequency.proto.bin differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json
new file mode 100644
index 00000000000..44e2c98a4dc
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.json
@@ -0,0 +1,29 @@
+{
+ "common": {
+ "planId": "1"
+ },
+ "project": {
+ "input": {
+ "common": {
+ "planId": "0"
+ },
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double,d:struct\u003cid:bigint,a:int,b:double\u003e,e:array\u003cint\u003e,f:map\u003cstring,struct\u003cid:bigint,a:int,b:double\u003e\u003e,g:string\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "percentile",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "a"
+ }
+ }, {
+ "literal": {
+ "double": 0.3
+ }
+ }]
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin
new file mode 100644
index 00000000000..45b807e5ffb
Binary files /dev/null and b/connector/connect/common/src/test/resources/query-tests/queries/function_percentile_without_frequency.proto.bin differ
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org