incubator-zeppelin git commit: ZEPPELIN-783: fix Spark download on CI

[bz...@apache.org]

Repository: incubator-zeppelin
Updated Branches:
  refs/heads/master 5ad02a28f -> 715526824


ZEPPELIN-783: fix Spark download on CI

### What is this PR for?
Fix Spark download on CI

### What type of PR is it?
Hot Fix

### What is the Jira issue?
[ZEPPELIN-783](https://issues.apache.org/jira/browse/ZEPPELIN-783)

### How should this be tested?
CI must be green

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Alexander Bezzubov <bz...@apache.org>

Closes #818 from bzz/ZEPPELIN-783-stable-ci-part-2 and squashes the following commits:

b4d66b3 [Alexander Bezzubov] ZEPPELIN-783: advanced Spark download failover procedure


Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/71552682
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/71552682
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/71552682

Branch: refs/heads/master
Commit: 7155268247965b9c22946d4d521838f552c6fcee
Parents: 5ad02a2
Author: Alexander Bezzubov <bz...@apache.org>
Authored: Wed Apr 6 16:21:56 2016 +0900
Committer: Alexander Bezzubov <bz...@apache.org>
Committed: Fri Apr 8 12:23:25 2016 +0900

----------------------------------------------------------------------
 testing/downloadSpark.sh | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/71552682/testing/downloadSpark.sh
----------------------------------------------------------------------
diff --git a/testing/downloadSpark.sh b/testing/downloadSpark.sh
index d12580f..004e1dd 100755
--- a/testing/downloadSpark.sh
+++ b/testing/downloadSpark.sh
@@ -40,10 +40,28 @@ fi
 
 set -xe
 
-TIMEOUT_SEC=590
+MAX_DOWNLOAD_TIME_SEC=590
 FWDIR="$(dirname "${BASH_SOURCE-$0}")"
 ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
 
+#######################################
+# Downloads file from the givrn URL.
+# Ties 3 times with 1s delay, 20s read and 15s connection timeouts.
+# Globals:
+#   None
+# Arguments:
+#   url - source URL
+# Returns:
+#   None
+#######################################
+download_with_retry() {
+    local url="$1"
+    wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 3 "${url}"
+    if [[ "$?" -ne 0 ]]; then
+        echo "3 download attempts for ${url} failed"
+    fi
+}
+
 SPARK_CACHE=".spark-dist"
 SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
 export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
@@ -56,11 +74,13 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
         pwd
         ls -la .
         echo "${SPARK_CACHE} does not have ${SPARK_ARCHIVE} downloading ..."
+
         # download archive if not cached
         if [[ "${SPARK_VER_RANGE}" == "<=1.2" ]]; then
             # spark 1.1.x and spark 1.2.x can be downloaded from archive
             STARTTIME=`date +%s`
-            timeout -s KILL "${TIMEOUT_SEC}" wget -q "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
+            #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
+            download_with_retry "http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/${SPARK_ARCHIVE}.tgz"
             ENDTIME=`date +%s`
             DOWNLOADTIME="$((ENDTIME-STARTTIME))"
         else
@@ -72,7 +92,8 @@ if [[ ! -d "${SPARK_HOME}" ]]; then
             PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
 
             STARTTIME=`date +%s`
-            timeout -s KILL "${TIMEOUT_SEC}" wget -q "${PREFFERED}${PATHINFO}"
+            #timeout -s KILL "${MAX_DOWNLOAD_TIME_SEC}" wget -q "${PREFFERED}${PATHINFO}"
+            download_with_retry "${PREFFERED}${PATHINFO}"
             ENDTIME=`date +%s`
             DOWNLOADTIME="$((ENDTIME-STARTTIME))"
         fi