You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2023/02/16 18:47:30 UTC
[spark] branch branch-3.4 updated: [SPARK-42464][CONNECT] Fix ProtoToPlanTestSuite for Scala 2.13
This is an automated email from the ASF dual-hosted git repository.
hvanhovell pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 1c70f5f2d73 [SPARK-42464][CONNECT] Fix ProtoToPlanTestSuite for Scala 2.13
1c70f5f2d73 is described below
commit 1c70f5f2d73331360b53993d0f2e73e7a7c8da4a
Author: Herman van Hovell <he...@databricks.com>
AuthorDate: Thu Feb 16 14:47:05 2023 -0400
[SPARK-42464][CONNECT] Fix ProtoToPlanTestSuite for Scala 2.13
### What changes were proposed in this pull request?
The `ProtoToPlanTestSuite` were broken for Scala 2.13 . This was caused by the following two problems:
- Explain output between 2.12 and 2.13 is not stable because we render collection implementations as well. For this I changed the rendering of the offending classes to be version agnostic.
- UDF code had deserialization issues. This was always the risk. I have removed those tests, we will work on improving UDF coverage in a follow-up.
### Why are the changes needed?
We want to test Scala 2.13 as well.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Test only change.
Closes #40055 from hvanhovell/SPARK-42464.
Authored-by: Herman van Hovell <he...@databricks.com>
Signed-off-by: Herman van Hovell <he...@databricks.com>
(cherry picked from commit 229807efd06562fc07fce9a257062d8a66068761)
Signed-off-by: Herman van Hovell <he...@databricks.com>
---
.../apache/spark/sql/PlanGenerationTestSuite.scala | 21 -----
.../explain-results/column_as_multi.explain | 2 +-
.../column_star_with_target.explain | 2 +-
.../explain-results/function_udf.explain | 2 -
.../explain-results/function_udf_2.12.explain | 2 -
.../explain-results/function_udf_2.13.explain | 2 -
.../join_inner_using_multiple_col_array.explain | 2 +-
.../join_inner_using_multiple_col_seq.explain | 2 +-
.../join_inner_using_single_col.explain | 2 +-
.../join_using_multiple_col_array.explain | 2 +-
.../join_using_multiple_col_seq.explain | 2 +-
.../explain-results/join_using_single_col.explain | 2 +-
.../query-tests/queries/function_udf.json | 96 ---------------------
.../query-tests/queries/function_udf.proto.bin | Bin 11257 -> 0 bytes
.../query-tests/queries/function_udf_2.12.json | 96 ---------------------
.../queries/function_udf_2.12.proto.bin | Bin 11257 -> 0 bytes
.../query-tests/queries/function_udf_2.13.json | 96 ---------------------
.../queries/function_udf_2.13.proto.bin | Bin 12092 -> 0 bytes
.../sql/connect/ProtoToParsedPlanTestSuite.scala | 5 --
.../spark/sql/catalyst/analysis/unresolved.scala | 4 +-
.../spark/sql/catalyst/plans/joinTypes.scala | 1 +
21 files changed, 11 insertions(+), 330 deletions(-)
diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
index f8bae27cc0e..10aa4b658da 100644
--- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
+++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala
@@ -20,7 +20,6 @@ import java.nio.file.{Files, Path}
import scala.collection.mutable
import scala.util.{Failure, Success, Try}
-import scala.util.Properties.versionNumberString
import com.google.protobuf.util.JsonFormat
import io.grpc.inprocess.InProcessChannelBuilder
@@ -58,8 +57,6 @@ class PlanGenerationTestSuite extends ConnectFunSuite with BeforeAndAfterAll wit
// Borrowed from SparkFunSuite
private val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
- private val scala = versionNumberString.substring(0, versionNumberString.indexOf(".", 2))
-
// Borrowed from SparkFunSuite
private def getWorkspaceFilePath(first: String, more: String*): Path = {
if (!(sys.props.contains("spark.test.home") || sys.env.contains("SPARK_HOME"))) {
@@ -787,24 +784,6 @@ class PlanGenerationTestSuite extends ConnectFunSuite with BeforeAndAfterAll wit
select(fn.max(Column("id")))
}
- test("function udf " + scala) {
- // This test might be a bit tricky if different JVM
- // versions are used to generate the golden files.
- val functions = Seq(
- fn.udf(TestUDFs.udf0)
- .asNonNullable()
- .asNondeterministic(),
- fn.udf(TestUDFs.udf1).withName("foo"),
- fn.udf(TestUDFs.udf2).withName("f3"),
- fn.udf(TestUDFs.udf3).withName("bar"),
- fn.udf(TestUDFs.udf4).withName("f_four"))
- val id = fn.col("id")
- val columns = functions.zipWithIndex.map { case (udf, i) =>
- udf(Seq.fill(i)(id): _*)
- }
- select(columns: _*)
- }
-
test("function lit") {
select(
fn.lit(fn.col("id")),
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain
index 0afa06a3c1b..34d098320fa 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/column_as_multi.explain
@@ -1,2 +1,2 @@
-'Project ['d AS Buffer(v1, v2, v3)]
+'Project ['d AS (v1, v2, v3)]
+- LocalRelation <empty>, [none#0L, none#1, none#2, none#3, none#4, none#5, none#6]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain
index 116393cfb02..f3366709711 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/column_star_with_target.explain
@@ -1,2 +1,2 @@
-'Project [ArrayBuffer(str).*]
+'Project [str.*]
+- LocalRelation <empty>, [none#0L, none#1, none#2, none#3, none#4, none#5, none#6]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain
deleted file mode 100644
index cdcf089bf7a..00000000000
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain
+++ /dev/null
@@ -1,2 +0,0 @@
-'Project [unresolvedalias((), None), unresolvedalias(foo('id), None), unresolvedalias(f3('id, 'id), None), unresolvedalias(bar('id, 'id, 'id), None), unresolvedalias(f_four('id, 'id, 'id, 'id), None)]
-+- LocalRelation <empty>, [none#0L, none#1, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain
deleted file mode 100644
index cdcf089bf7a..00000000000
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain
+++ /dev/null
@@ -1,2 +0,0 @@
-'Project [unresolvedalias((), None), unresolvedalias(foo('id), None), unresolvedalias(f3('id, 'id), None), unresolvedalias(bar('id, 'id, 'id), None), unresolvedalias(f_four('id, 'id, 'id, 'id), None)]
-+- LocalRelation <empty>, [none#0L, none#1, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain
deleted file mode 100644
index cdcf089bf7a..00000000000
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain
+++ /dev/null
@@ -1,2 +0,0 @@
-'Project [unresolvedalias((), None), unresolvedalias(foo('id), None), unresolvedalias(f3('id, 'id), None), unresolvedalias(bar('id, 'id, 'id), None), unresolvedalias(f_four('id, 'id, 'id, 'id), None)]
-+- LocalRelation <empty>, [none#0L, none#1, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain
index 715fa26ba2e..8df79323f39 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_array.explain
@@ -1,3 +1,3 @@
-'Join UsingJoin(Inner,Buffer(id, a))
+'Join UsingJoin(Inner, [id, a])
:- LocalRelation <empty>, [none#0L, none#1, none#2]
+- LocalRelation <empty>, [none#0, none#1L, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain
index 715fa26ba2e..8df79323f39 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_multiple_col_seq.explain
@@ -1,3 +1,3 @@
-'Join UsingJoin(Inner,Buffer(id, a))
+'Join UsingJoin(Inner, [id, a])
:- LocalRelation <empty>, [none#0L, none#1, none#2]
+- LocalRelation <empty>, [none#0, none#1L, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain
index bfcdcffbff9..8f83aaf56c1 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_inner_using_single_col.explain
@@ -1,3 +1,3 @@
-'Join UsingJoin(Inner,Buffer(id))
+'Join UsingJoin(Inner, [id])
:- LocalRelation <empty>, [none#0L, none#1, none#2]
+- LocalRelation <empty>, [none#0, none#1L, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain
index 92056649124..4b7bfe92868 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_array.explain
@@ -1,3 +1,3 @@
-'Join UsingJoin(FullOuter,Buffer(id, a))
+'Join UsingJoin(FullOuter, [id, a])
:- LocalRelation <empty>, [none#0L, none#1, none#2]
+- LocalRelation <empty>, [none#0, none#1L, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain
index 577f544646e..7fb3e44a238 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_multiple_col_seq.explain
@@ -1,3 +1,3 @@
-'Join UsingJoin(RightOuter,Buffer(id, a))
+'Join UsingJoin(RightOuter, [id, a])
:- LocalRelation <empty>, [none#0L, none#1, none#2]
+- LocalRelation <empty>, [none#0, none#1L, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain
index 224790686d5..a5f9651e8be 100644
--- a/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain
+++ b/connector/connect/common/src/test/resources/query-tests/explain-results/join_using_single_col.explain
@@ -1,3 +1,3 @@
-'Join UsingJoin(LeftSemi,Buffer(id))
+'Join UsingJoin(LeftSemi, [id])
:- LocalRelation <empty>, [none#0L, none#1, none#2]
+- LocalRelation <empty>, [none#0, none#1L, none#2]
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json
deleted file mode 100644
index 76738354e15..00000000000
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
- "project": {
- "input": {
- "localRelation": {
- "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
- }
- },
- "expressions": [{
- "commonInlineUserDefinedFunction": {
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQxBwIoVHJpGRUCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "foo",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQyWQlE7Ce2cPkCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "f3",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQzyD4NN4Grh74CAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "bar",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ009cpyjjQtFMCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "f_four",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ1hQsS9jxAO/gCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }]
- }
-}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin
deleted file mode 100644
index aa1971d3112..00000000000
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin and /dev/null differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json
deleted file mode 100644
index 76738354e15..00000000000
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
- "project": {
- "input": {
- "localRelation": {
- "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
- }
- },
- "expressions": [{
- "commonInlineUserDefinedFunction": {
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQxBwIoVHJpGRUCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "foo",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQyWQlE7Ce2cPkCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "f3",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQzyD4NN4Grh74CAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "bar",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ009cpyjjQtFMCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "f_four",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAFkxzY2FsYS9jb2xsZWN0aW9uL1NlcTtMAA1vdXRwdXRFbmNvZGVydAA4TG9yZy9hcGFjaGUvc3Bhcmsvc3FsL2NhdGFseXN0L2VuY29kZXJzL0Fnbm9zdGljRW5jb2Rlcjt4cHNyACVvcmcuYXBhY2hlLnNwYXJrLnNxbC5UZXN0VURGcyQkYW5vbiQ1hQsS9jxAO/gCAAB4cHNyACRzY2FsYS5jb2xsZWN0aW9uLm11dGFibGUuQXJyYXlCdWZmZXIVOLBTg4KOcwIAA0kAC2luaXRpYWxTaXplSQAFc2l6ZTBbAAVhcnJheXQAE1 [...]
- "nullable": true
- }
- }
- }]
- }
-}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin
deleted file mode 100644
index aa1971d3112..00000000000
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin and /dev/null differ
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json
deleted file mode 100644
index 23aeb078543..00000000000
--- a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
- "project": {
- "input": {
- "localRelation": {
- "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
- }
- },
- "expressions": [{
- "commonInlineUserDefinedFunction": {
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMQjucFe5xZ3dAgAAeHBzcgAyc2NhbGEuY29sbGVjdGlvbi5nZW5lcmljLkRlZmF1bHRTZXJpYWxpemF0aW9uUHJveHkAAAAAAAAAAwMAAUwAB2ZhY3Rvcn [...]
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "foo",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMmifcExfceVGAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "f3",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kM6Rv/pH6bPiLAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "bar",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNAv76myB5zZ4AgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
- "nullable": true
- }
- }
- }, {
- "commonInlineUserDefinedFunction": {
- "functionName": "f_four",
- "deterministic": true,
- "arguments": [{
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }, {
- "unresolvedAttribute": {
- "unparsedIdentifier": "id"
- }
- }],
- "scalarScalaUdf": {
- "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNbf+MdXT5ejEAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
- "nullable": true
- }
- }
- }]
- }
-}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin
deleted file mode 100644
index d7b3c84579c..00000000000
Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin and /dev/null differ
diff --git a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala
index 1626d9278ff..8f47b7b7843 100644
--- a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala
+++ b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala
@@ -21,7 +21,6 @@ import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor}
import java.nio.file.attribute.BasicFileAttributes
import scala.util.{Failure, Success, Try}
-import scala.util.Properties.versionNumberString
import org.apache.spark.SparkFunSuite
import org.apache.spark.connect.proto
@@ -75,10 +74,6 @@ class ProtoToParsedPlanTestSuite extends SparkFunSuite with SharedSparkSession {
private def createTest(file: Path): Unit = {
val relativePath = inputFilePath.relativize(file)
val fileName = relativePath.getFileName.toString
- if ((versionNumberString.startsWith("2.12") && fileName.endsWith("2.13.proto.bin")) ||
- (versionNumberString.startsWith("2.13") && fileName.endsWith("2.12.proto.bin"))) {
- return
- }
if (!fileName.endsWith(".proto.bin")) {
logError(s"Skipping $fileName")
return
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
index 5e20f12747b..04bc513aa99 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/unresolved.scala
@@ -459,7 +459,7 @@ case class UnresolvedStar(target: Option[Seq[String]]) extends Star with Unevalu
}
}
- override def toString: String = target.map(_ + ".").getOrElse("") + "*"
+ override def toString: String = target.map(_.mkString("", ".", ".")).getOrElse("") + "*"
}
/**
@@ -516,7 +516,7 @@ case class MultiAlias(child: Expression, names: Seq[String])
override lazy val resolved = false
- override def toString: String = s"$child AS $names"
+ override def toString: String = s"$child AS ${names.mkString("(", ", ", ")")}"
override protected def withNewChildInternal(newChild: Expression): MultiAlias =
copy(child = newChild)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
index eeec3cd765a..0c5f620d722 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/joinTypes.scala
@@ -105,6 +105,7 @@ case class UsingJoin(tpe: JoinType, usingColumns: Seq[String]) extends JoinType
require(Seq(Inner, LeftOuter, LeftSemi, RightOuter, FullOuter, LeftAnti, Cross).contains(tpe),
"Unsupported using join type " + tpe)
override def sql: String = "USING " + tpe.sql
+ override def toString: String = s"UsingJoin($tpe, ${usingColumns.mkString("[", ", ", "]")})"
}
object LeftExistence {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org