You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/11/18 02:38:05 UTC
[spark] branch master updated: [SPARK-41177][PROTOBUF][TESTS] Fix maven test failed of `protobuf` module
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8cf82700a4f [SPARK-41177][PROTOBUF][TESTS] Fix maven test failed of `protobuf` module
8cf82700a4f is described below
commit 8cf82700a4f09e54b3d872bf4902a41df5a151cb
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Fri Nov 18 11:37:48 2022 +0900
[SPARK-41177][PROTOBUF][TESTS] Fix maven test failed of `protobuf` module
### What changes were proposed in this pull request?
Maven and sbt use different protobuf plugin, this causes the contents of `connector/protobuf/src/test/resources/protobuf` to be compiled into different directories in `scala-2.12/test-classes` by different tools:
- Maven : compiled to `connector/protobuf/target/scala-2.12/test-classes/protobuf `
- sbt: compiled to `connector/protobuf/target/scala-2.12/test-classes/`
The files in the `resources/protobuf` directory are read in the test case of the `protobuf` module, which can pass with sbt test, but will fail in the maven test because the `protobuf/` prefix should be added when using maven.
So this pr add a helper method to passing in two alternative directories to make maven test also successful.
### Why are the changes needed?
Fix maven test failed of `protobuf` module.
### Does this PR introduce _any_ user-facing change?
No, just for test
### How was this patch tested?
- Pass GA
- Manual test
```
mvn clean install -pl connector/protobuf -am -DskipTests
mvn test -pl connector/protobuf
```
**Before**
```
ProtobufSerdeSuite:
org.apache.spark.sql.protobuf.ProtobufSerdeSuite *** ABORTED ***
java.lang.RuntimeException: Unable to load a Suite class org.apache.spark.sql.protobuf.ProtobufSerdeSuite that was discovered in the runpath: null
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:80)
at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
...
Cause: java.lang.NullPointerException:
at org.apache.spark.sql.test.SQLTestUtilsBase.testFile(SQLTestUtils.scala:466)
at org.apache.spark.sql.test.SQLTestUtilsBase.testFile$(SQLTestUtils.scala:465)
at org.apache.spark.sql.protobuf.ProtobufSerdeSuite.testFile(ProtobufSerdeSuite.scala:34)
at org.apache.spark.sql.protobuf.ProtobufSerdeSuite.<init>(ProtobufSerdeSuite.scala:39)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:66)
...
ProtobufFunctionsSuite:
org.apache.spark.sql.protobuf.ProtobufFunctionsSuite *** ABORTED ***
java.lang.RuntimeException: Unable to load a Suite class org.apache.spark.sql.protobuf.ProtobufFunctionsSuite that was discovered in the runpath: null
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:80)
at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
...
Cause: java.lang.NullPointerException:
at org.apache.spark.sql.test.SQLTestUtilsBase.testFile(SQLTestUtils.scala:466)
at org.apache.spark.sql.test.SQLTestUtilsBase.testFile$(SQLTestUtils.scala:465)
at org.apache.spark.sql.protobuf.ProtobufFunctionsSuite.testFile(ProtobufFunctionsSuite.scala:35)
at org.apache.spark.sql.protobuf.ProtobufFunctionsSuite.<init>(ProtobufFunctionsSuite.scala:39)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:66)
...
ProtobufCatalystDataConversionSuite:
org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite *** ABORTED ***
java.lang.RuntimeException: Unable to load a Suite class org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite that was discovered in the runpath: null
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:80)
at org.scalatest.tools.DiscoverySuite.$anonfun$nestedSuites$1(DiscoverySuite.scala:38)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
...
Cause: java.lang.NullPointerException:
at org.apache.spark.sql.test.SQLTestUtilsBase.testFile(SQLTestUtils.scala:466)
at org.apache.spark.sql.test.SQLTestUtilsBase.testFile$(SQLTestUtils.scala:465)
at org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite.testFile(ProtobufCatalystDataConversionSuite.scala:34)
at org.apache.spark.sql.protobuf.ProtobufCatalystDataConversionSuite.<init>(ProtobufCatalystDataConversionSuite.scala:39)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at org.scalatest.tools.DiscoverySuite$.getSuiteInstance(DiscoverySuite.scala:66)
...
Run completed in 717 milliseconds.
Total number of tests run: 0
Suites: completed 1, aborted 3
Tests: succeeded 0, failed 0, canceled 0, ignored 0, pending 0
*** 3 SUITES ABORTED ***
```
**After**
```
[info] ProtobufFunctionsSuite:
[info] - roundtrip in to_protobuf and from_protobuf - struct (4 seconds, 168 milliseconds)
[info] - roundtrip in from_protobuf and to_protobuf - Repeated (570 milliseconds)
[info] - roundtrip in from_protobuf and to_protobuf - Repeated Message Once (340 milliseconds)
[info] - roundtrip in from_protobuf and to_protobuf - Repeated Message Twice (215 milliseconds)
[info] - roundtrip in from_protobuf and to_protobuf - Map (589 milliseconds)
[info] - roundtrip in from_protobuf and to_protobuf - Enum (187 milliseconds)
[info] - roundtrip in from_protobuf and to_protobuf - Multiple Message (346 milliseconds)
[info] - Handle recursive fields in Protobuf schema, A->B->A (117 milliseconds)
[info] - Handle recursive fields in Protobuf schema, C->D->Array(C) (12 milliseconds)
[info] - Handle extra fields : oldProducer -> newConsumer (91 milliseconds)
[info] - Handle extra fields : newProducer -> oldConsumer (7 milliseconds)
[info] - roundtrip in to_protobuf and from_protobuf - with nulls (457 milliseconds)
[info] - from_protobuf filter to_protobuf (356 milliseconds)
[info] - Handle TimestampType between to_protobuf and from_protobuf (560 milliseconds)
[info] - Handle DayTimeIntervalType between to_protobuf and from_protobuf (547 milliseconds)
[info] - raise cannot construct protobuf descriptor error (13 milliseconds)
[info] ProtobufCatalystDataConversionSuite:
[info] - single StructType(StructField(int32_type,IntegerType,true)) with seed 989 (76 milliseconds)
[info] - single StructType(StructField(double_type,DoubleType,true)) with seed 65 (51 milliseconds)
[info] - single StructType(StructField(float_type,FloatType,true)) with seed 43 (32 milliseconds)
[info] - single StructType(StructField(bytes_type,BinaryType,true)) with seed 657 (62 milliseconds)
[info] - single StructType(StructField(string_type,StringType,true)) with seed 965 (43 milliseconds)
[info] - Handle unsupported input of message type (31 milliseconds)
[info] - filter push-down to Protobuf deserializer (23 milliseconds)
[info] - ProtobufDeserializer with binary type (1 millisecond)
[info] - Full names for message using descriptor file (1 millisecond)
[info] ProtobufSerdeSuite:
[info] - Test basic conversion (7 milliseconds)
[info] - Fail to convert with field type mismatch (3 milliseconds)
[info] - Fail to convert with missing nested Protobuf fields for serializer (2 milliseconds)
[info] - Fail to convert with deeply nested field type mismatch (8 milliseconds)
[info] - Fail to convert with missing Catalyst fields (3 milliseconds)
[info] - raise cannot parse and construct protobuf descriptor error (2 milliseconds)
[info] Run completed in 12 seconds, 167 milliseconds.
[info] Total number of tests run: 31
[info] Suites: completed 3, aborted 0
[info] Tests: succeeded 31, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```
Closes #38690 from LuciferYang/protobuf-maven-test.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../ProtobufCatalystDataConversionSuite.scala | 5 +--
.../sql/protobuf/ProtobufFunctionsSuite.scala | 12 ++++---
.../spark/sql/protobuf/ProtobufSerdeSuite.scala | 8 ++---
.../spark/sql/protobuf/ProtobufTestBase.scala | 37 ++++++++++++++++++++++
4 files changed, 51 insertions(+), 11 deletions(-)
diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
index ec75ebb5507..3e9273835e3 100644
--- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
+++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufCatalystDataConversionSuite.scala
@@ -34,9 +34,10 @@ import org.apache.spark.unsafe.types.UTF8String
class ProtobufCatalystDataConversionSuite
extends SparkFunSuite
with SharedSparkSession
- with ExpressionEvalHelper {
+ with ExpressionEvalHelper
+ with ProtobufTestBase {
- private val testFileDesc = testFile("catalyst_types.desc").replace("file:/", "/")
+ private val testFileDesc = testFile("catalyst_types.desc", "protobuf/catalyst_types.desc")
private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.CatalystTypes$"
private def checkResultWithEval(
diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
index 00ec56f90a6..e493bc66ca7 100644
--- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
+++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
@@ -32,11 +32,12 @@ import org.apache.spark.sql.protobuf.utils.ProtobufUtils
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{DayTimeIntervalType, IntegerType, StringType, StructField, StructType, TimestampType}
-class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Serializable {
+class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with ProtobufTestBase
+ with Serializable {
import testImplicits._
- val testFileDesc = testFile("functions_suite.desc").replace("file:/", "/")
+ val testFileDesc = testFile("functions_suite.desc", "protobuf/functions_suite.desc")
private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.SimpleMessageProtos$"
/**
@@ -458,7 +459,7 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri
}
test("Handle extra fields : oldProducer -> newConsumer") {
- val testFileDesc = testFile("catalyst_types.desc").replace("file:/", "/")
+ val testFileDesc = testFile("catalyst_types.desc", "protobuf/catalyst_types.desc")
val oldProducer = ProtobufUtils.buildDescriptor(testFileDesc, "oldProducer")
val newConsumer = ProtobufUtils.buildDescriptor(testFileDesc, "newConsumer")
@@ -498,7 +499,7 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri
}
test("Handle extra fields : newProducer -> oldConsumer") {
- val testFileDesc = testFile("catalyst_types.desc").replace("file:/", "/")
+ val testFileDesc = testFile("catalyst_types.desc", "protobuf/catalyst_types.desc")
val newProducer = ProtobufUtils.buildDescriptor(testFileDesc, "newProducer")
val oldConsumer = ProtobufUtils.buildDescriptor(testFileDesc, "oldConsumer")
@@ -680,7 +681,8 @@ class ProtobufFunctionsSuite extends QueryTest with SharedSparkSession with Seri
test("raise cannot construct protobuf descriptor error") {
val df = Seq(ByteString.empty().toByteArray).toDF("value")
- val testFileDescriptor = testFile("basicmessage_noimports.desc").replace("file:/", "/")
+ val testFileDescriptor =
+ testFile("basicmessage_noimports.desc", "protobuf/basicmessage_noimports.desc")
val e = intercept[AnalysisException] {
df.select(functions.from_protobuf($"value", "BasicMessage", testFileDescriptor) as 'sample)
diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
index 22b9d58bbd4..87ed5340943 100644
--- a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
+++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
@@ -31,12 +31,12 @@ import org.apache.spark.sql.types.{IntegerType, StructType}
* Tests for [[ProtobufSerializer]] and [[ProtobufDeserializer]] with a more specific focus on
* those classes.
*/
-class ProtobufSerdeSuite extends SharedSparkSession {
+class ProtobufSerdeSuite extends SharedSparkSession with ProtobufTestBase {
import ProtoSerdeSuite._
import ProtoSerdeSuite.MatchType._
- val testFileDesc = testFile("serde_suite.desc").replace("file:/", "/")
+ val testFileDesc = testFile("serde_suite.desc", "protobuf/serde_suite.desc")
private val javaClassNamePrefix = "org.apache.spark.sql.protobuf.protos.SerdeSuiteProtos$"
test("Test basic conversion") {
@@ -179,7 +179,7 @@ class ProtobufSerdeSuite extends SharedSparkSession {
test("raise cannot parse and construct protobuf descriptor error") {
// passing serde_suite.proto instead serde_suite.desc
- var testFileDesc = testFile("serde_suite.proto").replace("file:/", "/")
+ var testFileDesc = testFile("serde_suite.proto", "protobuf/serde_suite.proto")
val e1 = intercept[AnalysisException] {
ProtobufUtils.buildDescriptor(testFileDesc, "FieldMissingInSQLRoot")
}
@@ -189,7 +189,7 @@ class ProtobufSerdeSuite extends SharedSparkSession {
errorClass = "CANNOT_PARSE_PROTOBUF_DESCRIPTOR",
parameters = Map("descFilePath" -> testFileDesc))
- testFileDesc = testFile("basicmessage_noimports.desc").replace("file:/", "/")
+ testFileDesc = testFile("basicmessage_noimports.desc", "protobuf/basicmessage_noimports.desc")
val e2 = intercept[AnalysisException] {
ProtobufUtils.buildDescriptor(testFileDesc, "FieldMissingInSQLRoot")
}
diff --git a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufTestBase.scala b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufTestBase.scala
new file mode 100644
index 00000000000..831b4a26c06
--- /dev/null
+++ b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufTestBase.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.protobuf
+
+import org.apache.spark.sql.test.SQLTestUtils
+
+trait ProtobufTestBase extends SQLTestUtils {
+
+ /**
+ * Returns full path to the given file in the resource folder,
+ * if the first choice throw NPE, try to return the full path of alternative.
+ * The result path doesn't contain the `file:/` protocol part.
+ */
+ protected def testFile(fileName: String, alternateFileName: String): String = {
+ val ret = try {
+ testFile(fileName)
+ } catch {
+ case _: NullPointerException => testFile(alternateFileName)
+ }
+ ret.replace("file:/", "/")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org