You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/05/04 05:53:30 UTC
[spark] branch branch-3.0 updated: [SPARK-31267][SQL] Flaky test:
WholeStageCodegenSparkSubmitSuite.Generated code on driver should not embed
platform-specific constant
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7d48215 [SPARK-31267][SQL] Flaky test: WholeStageCodegenSparkSubmitSuite.Generated code on driver should not embed platform-specific constant
7d48215 is described below
commit 7d482150f14bb95ed923a39951e2b474ea604e43
Author: Tianshi Zhu <zh...@gmail.com>
AuthorDate: Mon May 4 14:50:38 2020 +0900
[SPARK-31267][SQL] Flaky test: WholeStageCodegenSparkSubmitSuite.Generated code on driver should not embed platform-specific constant
### What changes were proposed in this pull request?
Allow customized timeouts for `runSparkSubmit`, which will make flaky tests more likely to pass by using a larger timeout value.
I was able to reproduce the test failure on my laptop, which took 1.5 - 2 minutes to finish the test. After increasing the timeout, the test now can pass locally.
### Why are the changes needed?
This allows slow tests to use a larger timeout, so they are more likely to succeed.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
The test was able to pass on my local env after the change.
Closes #28438 from tianshizz/SPARK-31267.
Authored-by: Tianshi Zhu <zh...@gmail.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit a222644e1df907d0aba19634a166e146dfb4f551)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 5 +++--
.../spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala | 3 ++-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 9d47368..0715087 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileStatus, FSDataInputStream, Path}
import org.scalatest.{BeforeAndAfterEach, Matchers}
import org.scalatest.concurrent.{Signaler, ThreadSignaler, TimeLimits}
+import org.scalatest.time.Span
import org.scalatest.time.SpanSugar._
import org.apache.spark._
@@ -1417,7 +1418,7 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits {
implicit val defaultSignaler: Signaler = ThreadSignaler
// NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly.
- def runSparkSubmit(args: Seq[String], root: String = ".."): Unit = {
+ def runSparkSubmit(args: Seq[String], root: String = "..", timeout: Span = 1.minute): Unit = {
val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
val sparkSubmitFile = if (Utils.isWindows) {
new File(s"$root\\bin\\spark-submit.cmd")
@@ -1430,7 +1431,7 @@ object SparkSubmitSuite extends SparkFunSuite with TimeLimits {
Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome))
try {
- val exitCode = failAfter(1.minute) { process.waitFor() }
+ val exitCode = failAfter(timeout) { process.waitFor() }
if (exitCode != 0) {
fail(s"Process returned with exit code $exitCode. See the log4j logs for more detail.")
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
index f6814d8..c5a01de 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSparkSubmitSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution
import org.scalatest.{Assertions, BeforeAndAfterEach, Matchers}
import org.scalatest.concurrent.TimeLimits
+import org.scalatest.time.SpanSugar._
import org.apache.spark.{SparkFunSuite, TestUtils}
import org.apache.spark.deploy.SparkSubmitSuite
@@ -50,7 +51,7 @@ class WholeStageCodegenSparkSubmitSuite extends SparkFunSuite
"--conf", "spark.executor.extraJavaOptions=-XX:+UseCompressedOops",
"--conf", "spark.sql.adaptive.enabled=false",
unusedJar.toString)
- SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..")
+ SparkSubmitSuite.runSparkSubmit(argsForSparkSubmit, "../..", 3.minutes)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org