You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/05/03 07:15:16 UTC
spark git commit: [SPARK-23489][SQL][TEST]
HiveExternalCatalogVersionsSuite should verify the downloaded file
Repository: spark
Updated Branches:
refs/heads/master bf4352ca6 -> c9bfd1c6f
[SPARK-23489][SQL][TEST] HiveExternalCatalogVersionsSuite should verify the downloaded file
## What changes were proposed in this pull request?
Although [SPARK-22654](https://issues.apache.org/jira/browse/SPARK-22654) made `HiveExternalCatalogVersionsSuite` download from Apache mirrors three times, it has been flaky because it didn't verify the downloaded file. Some Apache mirrors terminate the downloading abnormally, the *corrupted* file shows the following errors.
```
gzip: stdin: not in gzip format
tar: Child returned status 1
tar: Error is not recoverable: exiting now
22:46:32.700 WARN org.apache.spark.sql.hive.HiveExternalCatalogVersionsSuite:
===== POSSIBLE THREAD LEAK IN SUITE o.a.s.sql.hive.HiveExternalCatalogVersionsSuite, thread names: Keep-Alive-Timer =====
*** RUN ABORTED ***
java.io.IOException: Cannot run program "./bin/spark-submit" (in directory "/tmp/test-spark/spark-2.2.0"): error=2, No such file or directory
```
This has been reported weirdly in two ways. For example, the above case is reported as Case 2 `no failures`.
- Case 1. [Test Result (1 failure / +1)](https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test%20(Dashboard)/job/spark-master-test-sbt-hadoop-2.7/4389/)
- Case 2. [Test Result (no failures)](https://amplab.cs.berkeley.edu/jenkins/view/Spark%20QA%20Test%20(Dashboard)/job/spark-master-test-maven-hadoop-2.6/4811/)
This PR aims to make `HiveExternalCatalogVersionsSuite` more robust by verifying the downloaded `tgz` file by extracting and checking the existence of `bin/spark-submit`. If it turns out that the file is empty or corrupted, `HiveExternalCatalogVersionsSuite` will do retry logic like the download failure.
## How was this patch tested?
Pass the Jenkins.
Author: Dongjoon Hyun <do...@apache.org>
Closes #21210 from dongjoon-hyun/SPARK-23489.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c9bfd1c6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c9bfd1c6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c9bfd1c6
Branch: refs/heads/master
Commit: c9bfd1c6f8d16890ea1e5bc2bcb654a3afb32591
Parents: bf4352c
Author: Dongjoon Hyun <do...@apache.org>
Authored: Thu May 3 15:15:05 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu May 3 15:15:05 2018 +0800
----------------------------------------------------------------------
.../hive/HiveExternalCatalogVersionsSuite.scala | 32 ++++++++++----------
1 file changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c9bfd1c6/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 6ca58e6..ea86ab9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -67,7 +67,21 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
logInfo(s"Downloading Spark $version from $url")
try {
getFileFromUrl(url, path, filename)
- return
+ val downloaded = new File(sparkTestingDir, filename).getCanonicalPath
+ val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath
+
+ Seq("mkdir", targetDir).!
+ val exitCode = Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").!
+ Seq("rm", downloaded).!
+
+ // For a corrupted file, `tar` returns non-zero values. However, we also need to check
+ // the extracted file because `tar` returns 0 for empty file.
+ val sparkSubmit = new File(sparkTestingDir, s"spark-$version/bin/spark-submit")
+ if (exitCode == 0 && sparkSubmit.exists()) {
+ return
+ } else {
+ Seq("rm", "-rf", targetDir).!
+ }
} catch {
case ex: Exception => logWarning(s"Failed to download Spark $version from $url", ex)
}
@@ -75,20 +89,6 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
fail(s"Unable to download Spark $version")
}
-
- private def downloadSpark(version: String): Unit = {
- tryDownloadSpark(version, sparkTestingDir.getCanonicalPath)
-
- val downloaded = new File(sparkTestingDir, s"spark-$version-bin-hadoop2.7.tgz").getCanonicalPath
- val targetDir = new File(sparkTestingDir, s"spark-$version").getCanonicalPath
-
- Seq("mkdir", targetDir).!
-
- Seq("tar", "-xzf", downloaded, "-C", targetDir, "--strip-components=1").!
-
- Seq("rm", downloaded).!
- }
-
private def genDataDir(name: String): String = {
new File(tmpDataDir, name).getCanonicalPath
}
@@ -161,7 +161,7 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
PROCESS_TABLES.testingVersions.zipWithIndex.foreach { case (version, index) =>
val sparkHome = new File(sparkTestingDir, s"spark-$version")
if (!sparkHome.exists()) {
- downloadSpark(version)
+ tryDownloadSpark(version, sparkTestingDir.getCanonicalPath)
}
val args = Seq(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org