You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2019/04/18 12:50:05 UTC
[spark] branch master updated: [SPARK-27460][TESTS] Running slowest
test suites in their own forked JVMs for higher parallelism
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9c238b8 [SPARK-27460][TESTS] Running slowest test suites in their own forked JVMs for higher parallelism
9c238b8 is described below
commit 9c238b8a462c483644237bd038edcf58f42a7db9
Author: Gengliang Wang <ge...@databricks.com>
AuthorDate: Thu Apr 18 20:49:36 2019 +0800
[SPARK-27460][TESTS] Running slowest test suites in their own forked JVMs for higher parallelism
## What changes were proposed in this pull request?
This patch modifies SparkBuild so that the largest / slowest test suites (or collections of suites) can run in their own forked JVMs, allowing them to be run in parallel with each other. This opt-in / whitelisting approach allows us to increase parallelism without having to fix a long-tail of flakiness / brittleness issues in tests which aren't performance bottlenecks.
See comments in SparkBuild.scala for information on the details, including a summary of why we sometimes opt to run entire groups of tests in a single forked JVM .
The time of full new pull request test in Jenkins is reduced by around 53%:
before changes: 4hr 40min
after changes: 2hr 13min
## How was this patch tested?
Unit test
Closes #24373 from gengliangwang/parallelTest.
Authored-by: Gengliang Wang <ge...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
project/SparkBuild.scala | 94 +++++++++++++++++++++-
.../org/apache/spark/sql/SQLQueryTestSuite.scala | 5 +-
.../apache/spark/sql/test/SharedSparkSession.scala | 7 +-
3 files changed, 100 insertions(+), 6 deletions(-)
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 94f014a..656071d 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -430,6 +430,83 @@ object SparkBuild extends PomBuild {
else x.settings(Seq[Setting[_]](): _*)
} ++ Seq[Project](OldDeps.project)
}
+
+ if (!sys.env.contains("SERIAL_SBT_TESTS")) {
+ allProjects.foreach(enable(SparkParallelTestGrouping.settings))
+ }
+}
+
+object SparkParallelTestGrouping {
+ // Settings for parallelizing tests. The basic strategy here is to run the slowest suites (or
+ // collections of suites) in their own forked JVMs, allowing us to gain parallelism within a
+ // SBT project. Here, we take a whitelisting approach where the default behavior is to run all
+ // tests sequentially in a single JVM, requiring us to manually opt-in to the extra parallelism.
+ //
+ // There are a reasons why such a whitelist approach is good:
+ //
+ // 1. Launching one JVM per suite adds significant overhead for short-running suites. In
+ // addition to JVM startup time and JIT warmup, it appears that initialization of Derby
+ // metastores can be very slow so creating a fresh warehouse per suite is inefficient.
+ //
+ // 2. When parallelizing within a project we need to give each forked JVM a different tmpdir
+ // so that the metastore warehouses do not collide. Unfortunately, it seems that there are
+ // some tests which have an overly tight dependency on the default tmpdir, so those fragile
+ // tests need to continue re-running in the default configuration (or need to be rewritten).
+ // Fixing that problem would be a huge amount of work for limited payoff in most cases
+ // because most test suites are short-running.
+ //
+
+ private val testsWhichShouldRunInTheirOwnDedicatedJvm = Set(
+ "org.apache.spark.DistributedSuite",
+ "org.apache.spark.sql.catalyst.expressions.DateExpressionsSuite",
+ "org.apache.spark.sql.catalyst.expressions.HashExpressionsSuite",
+ "org.apache.spark.sql.catalyst.expressions.CastSuite",
+ "org.apache.spark.sql.catalyst.expressions.MathExpressionsSuite",
+ "org.apache.spark.sql.hive.HiveExternalCatalogSuite",
+ "org.apache.spark.sql.hive.StatisticsSuite",
+ "org.apache.spark.sql.hive.execution.HiveCompatibilitySuite",
+ "org.apache.spark.sql.hive.client.VersionsSuite",
+ "org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite",
+ "org.apache.spark.ml.classification.LogisticRegressionSuite",
+ "org.apache.spark.ml.classification.LinearSVCSuite",
+ "org.apache.spark.sql.SQLQueryTestSuite"
+ )
+
+ private val DEFAULT_TEST_GROUP = "default_test_group"
+
+ private def testNameToTestGroup(name: String): String = name match {
+ case _ if testsWhichShouldRunInTheirOwnDedicatedJvm.contains(name) => name
+ case _ => DEFAULT_TEST_GROUP
+ }
+
+ lazy val settings = Seq(
+ testGrouping in Test := {
+ val tests: Seq[TestDefinition] = (definedTests in Test).value
+ val defaultForkOptions = ForkOptions(
+ bootJars = Nil,
+ javaHome = javaHome.value,
+ connectInput = connectInput.value,
+ outputStrategy = outputStrategy.value,
+ runJVMOptions = (javaOptions in Test).value,
+ workingDirectory = Some(baseDirectory.value),
+ envVars = (envVars in Test).value
+ )
+ tests.groupBy(test => testNameToTestGroup(test.name)).map { case (groupName, groupTests) =>
+ val forkOptions = {
+ if (groupName == DEFAULT_TEST_GROUP) {
+ defaultForkOptions
+ } else {
+ defaultForkOptions.copy(runJVMOptions = defaultForkOptions.runJVMOptions ++
+ Seq(s"-Djava.io.tmpdir=${baseDirectory.value}/target/tmp/$groupName"))
+ }
+ }
+ new Tests.Group(
+ name = groupName,
+ tests = groupTests,
+ runPolicy = Tests.SubProcess(forkOptions))
+ }
+ }.toSeq
+ )
}
object Core {
@@ -911,8 +988,14 @@ object TestSettings {
testOptions in Test += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
// Enable Junit testing.
libraryDependencies += "com.novocode" % "junit-interface" % "0.11" % "test",
- // Only allow one test at a time, even across projects, since they run in the same JVM
- parallelExecution in Test := false,
+ // `parallelExecutionInTest` controls whether test suites belonging to the same SBT project
+ // can run in parallel with one another. It does NOT control whether tests execute in parallel
+ // within the same JVM (which is controlled by `testForkedParallel`) or whether test cases
+ // within the same suite can run in parallel (which is a ScalaTest runner option which is passed
+ // to the underlying runner but is not a SBT-level configuration). This needs to be `true` in
+ // order for the extra parallelism enabled by `SparkParallelTestGrouping` to take effect.
+ // The `SERIAL_SBT_TESTS` check is here so the extra parallelism can be feature-flagged.
+ parallelExecution in Test := { if (sys.env.contains("SERIAL_SBT_TESTS")) false else true },
// Make sure the test temp directory exists.
resourceGenerators in Test += Def.macroValueI(resourceManaged in Test map { outDir: File =>
var dir = new File(testTempDir)
@@ -933,7 +1016,12 @@ object TestSettings {
}
Seq.empty[File]
}).value,
- concurrentRestrictions in Global += Tags.limit(Tags.Test, 1)
+ concurrentRestrictions in Global := {
+ // The number of concurrent test groups is empirically chosen based on experience
+ // with Jenkins flakiness.
+ if (sys.env.contains("SERIAL_SBT_TESTS")) (concurrentRestrictions in Global).value
+ else Seq(Tags.limit(Tags.ForkedTestGroup, 4))
+ }
)
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index def99c8..d1842d3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -291,10 +291,13 @@ class SQLQueryTestSuite extends QueryTest with SharedSQLContext {
val df = session.sql(sql)
val schema = df.schema
val notIncludedMsg = "[not included in comparison]"
+ val clsName = this.getClass.getCanonicalName
// Get answer, but also get rid of the #1234 expression ids that show up in explain plans
val answer = hiveResultString(df.queryExecution.executedPlan)
.map(_.replaceAll("#\\d+", "#x")
- .replaceAll("Location.*/sql/core/", s"Location ${notIncludedMsg}sql/core/")
+ .replaceAll(
+ s"Location.*/sql/core/spark-warehouse/$clsName/",
+ s"Location ${notIncludedMsg}sql/core/spark-warehouse/")
.replaceAll("Created By.*", s"Created By $notIncludedMsg")
.replaceAll("Created Time.*", s"Created Time $notIncludedMsg")
.replaceAll("Last Access.*", s"Last Access $notIncludedMsg")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
index 8734639..ff6211b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SharedSparkSession.scala
@@ -26,7 +26,7 @@ import org.apache.spark.{DebugFilesystem, SparkConf}
import org.apache.spark.internal.config.UNSAFE_EXCEPTION_ON_MEMORY_LEAK
import org.apache.spark.sql.{SparkSession, SQLContext}
import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
/**
* Helper trait for SQL test suites where all tests share a single [[TestSparkSession]].
@@ -37,7 +37,7 @@ trait SharedSparkSession
with Eventually { self: Suite =>
protected def sparkConf = {
- new SparkConf()
+ val conf = new SparkConf()
.set("spark.hadoop.fs.file.impl", classOf[DebugFilesystem].getName)
.set(UNSAFE_EXCEPTION_ON_MEMORY_LEAK, true)
.set(SQLConf.CODEGEN_FALLBACK.key, "false")
@@ -46,6 +46,9 @@ trait SharedSparkSession
// this rule may potentially block testing of other optimization rules such as
// ConstantPropagation etc.
.set(SQLConf.OPTIMIZER_EXCLUDED_RULES.key, ConvertToLocalRelation.ruleName)
+ conf.set(
+ StaticSQLConf.WAREHOUSE_PATH,
+ conf.get(StaticSQLConf.WAREHOUSE_PATH) + "/" + getClass.getCanonicalName)
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org