You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/09/06 04:10:54 UTC
spark git commit: [SPARK-25337][SQL][TEST] runSparkSubmit` should
provide non-testing mode
Repository: spark
Updated Branches:
refs/heads/master 3d6b68b03 -> 0a5a49a51
[SPARK-25337][SQL][TEST] runSparkSubmit` should provide non-testing mode
## What changes were proposed in this pull request?
`HiveExternalCatalogVersionsSuite` Scala-2.12 test has been failing due to class path issue. It is marked as `ABORTED` because it fails at `beforeAll` during data population stage.
- https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test%20(Dashboard)/job/spark-master-test-maven-hadoop-2.7-ubuntu-scala-2.12/
```
org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite *** ABORTED ***
Exception encountered when invoking run on a nested suite - spark-submit returned with exit code 1.
```
The root cause of the failure is that `runSparkSubmit` mixes 2.4.0-SNAPSHOT classes and old Spark (2.1.3/2.2.2/2.3.1) together during `spark-submit`. This PR aims to provide `non-test` mode execution mode to `runSparkSubmit` by removing the followings.
- SPARK_TESTING
- SPARK_SQL_TESTING
- SPARK_PREPEND_CLASSES
- SPARK_DIST_CLASSPATH
Previously, in the class path, new Spark classes are behind the old Spark classes. So, new ones are unseen. However, Spark 2.4.0 reveals this bug due to the recent data source class changes.
## How was this patch tested?
Manual test. After merging, it will be tested via Jenkins.
```scala
$ dev/change-scala-version.sh 2.12
$ build/mvn -DskipTests -Phive -Pscala-2.12 clean package
$ build/mvn -Phive -Pscala-2.12 -Dtest=none -DwildcardSuites=org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite test
...
HiveExternalCatalogVersionsSuite:
- backward compatibility
...
Tests: succeeded 1, failed 0, canceled 0, ignored 0, pending 0
All tests passed.
```
Closes #22340 from dongjoon-hyun/SPARK-25337.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Sean Owen <se...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0a5a49a5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0a5a49a5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0a5a49a5
Branch: refs/heads/master
Commit: 0a5a49a51c85d2c81c38104d3fcc8e0fa330ccc5
Parents: 3d6b68b
Author: Dongjoon Hyun <do...@apache.org>
Authored: Wed Sep 5 21:10:51 2018 -0700
Committer: Sean Owen <se...@databricks.com>
Committed: Wed Sep 5 21:10:51 2018 -0700
----------------------------------------------------------------------
.../sql/hive/HiveExternalCatalogVersionsSuite.scala | 2 +-
.../apache/spark/sql/hive/SparkSubmitTestUtils.scala | 14 ++++++++++++--
2 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0a5a49a5/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 5103aa8..25df333 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -181,7 +181,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
"--conf", s"spark.sql.test.version.index=$index",
"--driver-java-options", s"-Dderby.system.home=${wareHousePath.getCanonicalPath}",
tempPyFile.getCanonicalPath)
- runSparkSubmit(args, Some(sparkHome.getCanonicalPath))
+ runSparkSubmit(args, Some(sparkHome.getCanonicalPath), false)
}
tempPyFile.delete()
http://git-wip-us.apache.org/repos/asf/spark/blob/0a5a49a5/sql/hive/src/test/scala/org/apache/spark/sql/hive/SparkSubmitTestUtils.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/SparkSubmitTestUtils.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/SparkSubmitTestUtils.scala
index 68ed97d..889f81b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/SparkSubmitTestUtils.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/SparkSubmitTestUtils.scala
@@ -38,7 +38,10 @@ trait SparkSubmitTestUtils extends SparkFunSuite with TimeLimits {
// NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly.
// This is copied from org.apache.spark.deploy.SparkSubmitSuite
- protected def runSparkSubmit(args: Seq[String], sparkHomeOpt: Option[String] = None): Unit = {
+ protected def runSparkSubmit(
+ args: Seq[String],
+ sparkHomeOpt: Option[String] = None,
+ isSparkTesting: Boolean = true): Unit = {
val sparkHome = sparkHomeOpt.getOrElse(
sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!")))
val history = ArrayBuffer.empty[String]
@@ -53,7 +56,14 @@ trait SparkSubmitTestUtils extends SparkFunSuite with TimeLimits {
val builder = new ProcessBuilder(commands: _*).directory(new File(sparkHome))
val env = builder.environment()
- env.put("SPARK_TESTING", "1")
+ if (isSparkTesting) {
+ env.put("SPARK_TESTING", "1")
+ } else {
+ env.remove("SPARK_TESTING")
+ env.remove("SPARK_SQL_TESTING")
+ env.remove("SPARK_PREPEND_CLASSES")
+ env.remove("SPARK_DIST_CLASSPATH")
+ }
env.put("SPARK_HOME", sparkHome)
def captureOutput(source: String)(line: String): Unit = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org