You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/02/13 01:31:46 UTC
[spark] branch branch-3.4 updated: [SPARK-42410][CONNECT][TESTS] Support Scala 2.12/2.13 tests in `connect` module
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new d3b59aad100 [SPARK-42410][CONNECT][TESTS] Support Scala 2.12/2.13 tests in `connect` module
d3b59aad100 is described below
commit d3b59aad1000820d7fcbd78e0e1810d9bf81b8cc
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Feb 12 17:31:26 2023 -0800
[SPARK-42410][CONNECT][TESTS] Support Scala 2.12/2.13 tests in `connect` module
### What changes were proposed in this pull request?
This PR aims to support both Scala 2.12/13 tests in `connect` module by splitting the golden files.
### Why are the changes needed?
After #39933, Scala 2.13 CIs are broken .
- **master**: https://github.com/apache/spark/actions/runs/4157806226/jobs/7192575602
- **branch-3.4**: https://github.com/apache/spark/actions/runs/4155578848/jobs/7188777977
```
[info] - function_udf *** FAILED *** (29 milliseconds)
[info] java.io.InvalidClassException: org.apache.spark.sql.TestUDFs$$anon$1; local class incompatible: stream classdesc serialVersionUID = 505010451380771093, local class serialVersionUID = 643575318841761245
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs and do Scala 2.13 tests manually.
```
$ dev/change-scala-version.sh 2.13
$ build/sbt -Dscala.version=2.13.8 -Pscala-2.13 -Phadoop-3 assembly/package "connect/test"
...
[info] - function_udf_2.13 (19 milliseconds)
[info] Run completed in 8 seconds, 964 milliseconds.
[info] Total number of tests run: 119
[info] Suites: completed 8, aborted 0
[info] Tests: succeeded 119, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
[success] Total time: 18 s, completed Feb 12, 2023, 3:33:40 PM
```
Closes #39982 from dongjoon-hyun/SPARK-42410.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit a5db9b976d8f37d22f8a91f461c05cbb20601d8a)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
...ction_udf.explain => function_udf_2.12.explain} | 0
...ction_udf.explain => function_udf_2.13.explain} | 0
.../{function_udf.json => function_udf_2.12.json} | 0
...n_udf.proto.bin => function_udf_2.12.proto.bin} | Bin
.../query-tests/queries/function_udf_2.13.json | 96 +++++++++++++++++++++
.../queries/function_udf_2.13.proto.bin | Bin 0 -> 12092 bytes
.../sql/connect/ProtoToParsedPlanTestSuite.scala | 5 ++
7 files changed, 101 insertions(+)
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain
similarity index 100%
copy from connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain
copy to connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.12.explain
diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain
similarity index 100%
rename from connector/connect/common/src/test/resources/query-tests/explain-results/function_udf.explain
rename to connector/connect/common/src/test/resources/query-tests/explain-results/function_udf_2.13.explain
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json
similarity index 100%
rename from connector/connect/common/src/test/resources/query-tests/queries/function_udf.json
rename to connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.json
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin
similarity index 100%
rename from connector/connect/common/src/test/resources/query-tests/queries/function_udf.proto.bin
rename to connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.12.proto.bin
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json
new file mode 100644
index 00000000000..23aeb078543
--- /dev/null
+++ b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.json
@@ -0,0 +1,96 @@
+{
+ "project": {
+ "input": {
+ "localRelation": {
+ "schema": "struct\u003cid:bigint,a:int,b:double\u003e"
+ }
+ },
+ "expressions": [{
+ "commonInlineUserDefinedFunction": {
+ "scalarScalaUdf": {
+ "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMQjucFe5xZ3dAgAAeHBzcgAyc2NhbGEuY29sbGVjdGlvbi5nZW5lcmljLkRlZmF1bHRTZXJpYWxpemF0aW9uUHJveHkAAAAAAAAAAwMAAUwAB2ZhY3Rvcn [...]
+ }
+ }
+ }, {
+ "commonInlineUserDefinedFunction": {
+ "functionName": "foo",
+ "deterministic": true,
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }],
+ "scalarScalaUdf": {
+ "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kMmifcExfceVGAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
+ "nullable": true
+ }
+ }
+ }, {
+ "commonInlineUserDefinedFunction": {
+ "functionName": "f3",
+ "deterministic": true,
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }, {
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }],
+ "scalarScalaUdf": {
+ "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kM6Rv/pH6bPiLAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
+ "nullable": true
+ }
+ }
+ }, {
+ "commonInlineUserDefinedFunction": {
+ "functionName": "bar",
+ "deterministic": true,
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }, {
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }, {
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }],
+ "scalarScalaUdf": {
+ "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNAv76myB5zZ4AgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
+ "nullable": true
+ }
+ }
+ }, {
+ "commonInlineUserDefinedFunction": {
+ "functionName": "f_four",
+ "deterministic": true,
+ "arguments": [{
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }, {
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }, {
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }, {
+ "unresolvedAttribute": {
+ "unparsedIdentifier": "id"
+ }
+ }],
+ "scalarScalaUdf": {
+ "payload": "rO0ABXNyAC1vcmcuYXBhY2hlLnNwYXJrLnNxbC5jb25uZWN0LmNvbW1vbi5VZGZQYWNrZXR7DRDpFmMz1QIAA0wACGZ1bmN0aW9udAASTGphdmEvbGFuZy9PYmplY3Q7TAANaW5wdXRFbmNvZGVyc3QAIExzY2FsYS9jb2xsZWN0aW9uL2ltbXV0YWJsZS9TZXE7TAANb3V0cHV0RW5jb2RlcnQAOExvcmcvYXBhY2hlL3NwYXJrL3NxbC9jYXRhbHlzdC9lbmNvZGVycy9BZ25vc3RpY0VuY29kZXI7eHBzcgAlb3JnLmFwYWNoZS5zcGFyay5zcWwuVGVzdFVERnMkJGFub24kNbf+MdXT5ejEAgAAeHBzcgApc2NhbGEuY29sbGVjdGlvbi5pbW11dGFibGUuQXJyYXlTZXEkb2ZSZWYAAAAAAAAAAwIAAVsAC3Vuc2FmZUFycmF5dAATW0 [...]
+ "nullable": true
+ }
+ }
+ }]
+ }
+}
\ No newline at end of file
diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin
new file mode 100644
index 00000000000..d7b3c84579c
Binary files /dev/null and b/connector/connect/common/src/test/resources/query-tests/queries/function_udf_2.13.proto.bin differ
diff --git a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala
index 8f47b7b7843..1626d9278ff 100644
--- a/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala
+++ b/connector/connect/server/src/test/scala/org/apache/spark/sql/connect/ProtoToParsedPlanTestSuite.scala
@@ -21,6 +21,7 @@ import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor}
import java.nio.file.attribute.BasicFileAttributes
import scala.util.{Failure, Success, Try}
+import scala.util.Properties.versionNumberString
import org.apache.spark.SparkFunSuite
import org.apache.spark.connect.proto
@@ -74,6 +75,10 @@ class ProtoToParsedPlanTestSuite extends SparkFunSuite with SharedSparkSession {
private def createTest(file: Path): Unit = {
val relativePath = inputFilePath.relativize(file)
val fileName = relativePath.getFileName.toString
+ if ((versionNumberString.startsWith("2.12") && fileName.endsWith("2.13.proto.bin")) ||
+ (versionNumberString.startsWith("2.13") && fileName.endsWith("2.12.proto.bin"))) {
+ return
+ }
if (!fileName.endsWith(".proto.bin")) {
logError(s"Skipping $fileName")
return
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org