You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/07/16 04:26:52 UTC
spark git commit: [SPARK-24813][TESTS][HIVE][HOTFIX][BRANCH-2.2]
HiveExternalCatalogVersionsSuite still flaky; fall back to Apache archive
Repository: spark
Updated Branches:
refs/heads/branch-2.2 a8537a5ab -> 4bc4ccd63
[SPARK-24813][TESTS][HIVE][HOTFIX][BRANCH-2.2] HiveExternalCatalogVersionsSuite still flaky; fall back to Apache archive
## What changes were proposed in this pull request?
Try only unique ASF mirrors to download Spark release; fall back to Apache archive if no mirrors available or release is not mirrored
## How was this patch tested?
Existing HiveExternalCatalogVersionsSuite
Author: Sean Owen <sr...@gmail.com>
Closes #21779 from srowen/SPARK-24813.2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4bc4ccd6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4bc4ccd6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4bc4ccd6
Branch: refs/heads/branch-2.2
Commit: 4bc4ccd63b7d56fba1814505136b0d33517ed8f0
Parents: a8537a5
Author: Sean Owen <sr...@gmail.com>
Authored: Mon Jul 16 12:26:45 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Mon Jul 16 12:26:45 2018 +0800
----------------------------------------------------------------------
.../hive/HiveExternalCatalogVersionsSuite.scala | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4bc4ccd6/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 92c27e8..313059b 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -53,12 +53,22 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
}
private def tryDownloadSpark(version: String, path: String): Unit = {
- // Try mirrors a few times until one succeeds
- for (i <- 0 until 3) {
- val preferredMirror =
- Seq("wget", "https://www.apache.org/dyn/closer.lua?preferred=true", "-q", "-O", "-").!!.trim
+ // Try a few mirrors first; fall back to Apache archive
+ val mirrors =
+ (0 until 2).flatMap { _ =>
+ try {
+ Some(Seq("wget",
+ "https://www.apache.org/dyn/closer.lua?preferred=true", "-q", "-O", "-").!!.trim)
+ } catch {
+ // If we can't get a mirror URL, skip it. No retry.
+ case _: Exception => None
+ }
+ }
+ val sites = mirrors.distinct :+ "https://archive.apache.org/dist"
+ logInfo(s"Trying to download Spark $version from $sites")
+ for (site <- sites) {
val filename = s"spark-$version-bin-hadoop2.7.tgz"
- val url = s"$preferredMirror/spark/spark-$version/$filename"
+ val url = s"$site/spark/spark-$version/$filename"
logInfo(s"Downloading Spark $version from $url")
if (Seq("wget", url, "-q", "-P", path).! == 0) {
val downloaded = new File(sparkTestingDir, filename).getCanonicalPath
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org