You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2023/02/16 01:43:47 UTC
[spark] branch branch-3.4 updated: [SPARK-42453][CONNECT] Implement function max in Scala client
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 42337fe25d8 [SPARK-42453][CONNECT] Implement function max in Scala client
42337fe25d8 is described below
commit 42337fe25d86b18744ba38206518fabd18477d53
Author: Rui Wang <ru...@databricks.com>
AuthorDate: Thu Feb 16 09:43:12 2023 +0800
[SPARK-42453][CONNECT] Implement function max in Scala client
### What changes were proposed in this pull request?
This PR tries to implement functions.max for Scala client.
### Why are the changes needed?
API coverage.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing scala client testing framework.
Closes #40041 from amaliujia/rw-scala-count.
Authored-by: Rui Wang <ru...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 5d81a1cf91a53fd3fe7051f25fdeedf389413f09)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../main/scala/org/apache/spark/sql/functions.scala | 8 ++++++++
.../apache/spark/sql/PlanGenerationTestSuite.scala | 4 ++++
.../query-tests/explain-results/function_max.explain | 2 ++
.../resources/query-tests/queries/function_max.json | 19 +++++++++++++++++++
.../query-tests/queries/function_max.proto.bin | 4 ++++
5 files changed, 37 insertions(+)
diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
index 7045b2885d3..6f433245cf3 100644
--- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala
@@ -44,6 +44,14 @@ object functions {
*/
def col(colName: String): Column = Column(colName)
+ /**
+ * Aggregate function: returns the maximum value of the expression in a group.
+ *
+ * @group agg_funcs
+ * @since 3.4.0
+ */
+ def max(e: Column): Column = Column.fn("max", e)
+
private def createLiteral(f: proto.Expression.Literal.Builder => Unit): Column = Column {
builder =>
val literalBuilder = proto.Expression.Literal.newBuilder()
diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index 5a2a9485cf0..416a9b33886 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -731,6 +731,10 @@ class PlanGenerationTestSuite extends ConnectFunSuite with BeforeAndAfterAll wit
select(fn.col("id"))
}
+ test("function max") {
+ select(fn.max(Column("id")))
+ }
+
test("function udf " + scala) {
// This test might be a bit tricky if different JVM
// versions are used to generate the golden files.
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_max.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_max.explain
new file mode 100644
index 00000000000..077ba5498b1
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_max.explain
@@ -0,0 +1,2 @@
+'Project [unresolvedalias('max('id), None)]
++- LocalRelation <empty>, [none#0L, none#1, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_max.json b/connector/connect/common/src/test/resources/query-tests/queries/function_max.json
new file mode 100644
index 00000000000..2aba75d4337
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_max.json
@@ -0,0 +1,19 @@
+{
+ "project": {
+ "input": {
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
+ }
+ },
+ "expressions": [{
+ "unresolvedFunction": {
+ "functionName": "max",
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }]
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin
new file mode 100644
index 00000000000..34d523d28b8
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_max.proto.bin
@@ -0,0 +1,4 @@
+7
+$Z" struct<id:bigint,a:int,b:double>
+max
+id
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org