You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/07/16 01:30:01 UTC
spark git commit: [SPARK-24813][TESTS][HIVE][HOTFIX]
HiveExternalCatalogVersionsSuite still flaky; fall back to Apache archive
Repository: spark
Updated Branches:
refs/heads/master 5d62a985d -> bbc2ffc8a
[SPARK-24813][TESTS][HIVE][HOTFIX] HiveExternalCatalogVersionsSuite still flaky; fall back to Apache archive
## What changes were proposed in this pull request?
Try only unique ASF mirrors to download Spark release; fall back to Apache archive if no mirrors available or release is not mirrored
## How was this patch tested?
Existing HiveExternalCatalogVersionsSuite
Author: Sean Owen <sr...@gmail.com>
Closes #21776 from srowen/SPARK-24813.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bbc2ffc8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bbc2ffc8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bbc2ffc8
Branch: refs/heads/master
Commit: bbc2ffc8ab27192384def9847c36b873efd87234
Parents: 5d62a98
Author: Sean Owen <sr...@gmail.com>
Authored: Mon Jul 16 09:29:51 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Mon Jul 16 09:29:51 2018 +0800
----------------------------------------------------------------------
.../hive/HiveExternalCatalogVersionsSuite.scala | 24 +++++++++++++-------
1 file changed, 16 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bbc2ffc8/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
index 5149218..f821268 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala
@@ -56,14 +56,21 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
}
private def tryDownloadSpark(version: String, path: String): Unit = {
- // Try mirrors a few times until one succeeds
- for (i <- 0 until 3) {
- // we don't retry on a failure to get mirror url. If we can't get a mirror url,
- // the test fails (getStringFromUrl will throw an exception)
- val preferredMirror =
- getStringFromUrl("https://www.apache.org/dyn/closer.lua?preferred=true")
+ // Try a few mirrors first; fall back to Apache archive
+ val mirrors =
+ (0 until 2).flatMap { _ =>
+ try {
+ Some(getStringFromUrl("https://www.apache.org/dyn/closer.lua?preferred=true"))
+ } catch {
+ // If we can't get a mirror URL, skip it. No retry.
+ case _: Exception => None
+ }
+ }
+ val sites = mirrors.distinct :+ "https://archive.apache.org/dist"
+ logInfo(s"Trying to download Spark $version from $sites")
+ for (site <- sites) {
val filename = s"spark-$version-bin-hadoop2.7.tgz"
- val url = s"$preferredMirror/spark/spark-$version/$filename"
+ val url = s"$site/spark/spark-$version/$filename"
logInfo(s"Downloading Spark $version from $url")
try {
getFileFromUrl(url, path, filename)
@@ -83,7 +90,8 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils {
Seq("rm", "-rf", targetDir).!
}
} catch {
- case ex: Exception => logWarning(s"Failed to download Spark $version from $url", ex)
+ case ex: Exception =>
+ logWarning(s"Failed to download Spark $version from $url: ${ex.getMessage}")
}
}
fail(s"Unable to download Spark $version")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org