[incubator-uniffle] branch master updated: feat: support downloads latest hadoop archives and mirror url (#619)

[xi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xianjin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new 7814cb98 feat: support downloads latest hadoop archives and mirror url (#619)
7814cb98 is described below

commit 7814cb98943f2fb5e76cf393d51a0d4e974834bd
Author: advancedxy <xi...@apache.org>
AuthorDate: Fri Feb 17 15:29:35 2023 +0800

    feat: support downloads latest hadoop archives and mirror url (#619)
    
    ### What changes were proposed in this pull request?
    1. support download latest hadoop archive such as hadoop-2.10.2
    2. add `--apache-mirror` for build.sh
    
    ### Why are the changes needed?
    Fix: #618
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Manually tested.
---
 deploy/kubernetes/docker/build.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/deploy/kubernetes/docker/build.sh b/deploy/kubernetes/docker/build.sh
index 86972934..a318b860 100644
--- a/deploy/kubernetes/docker/build.sh
+++ b/deploy/kubernetes/docker/build.sh
@@ -24,6 +24,7 @@ function exit_with_usage() {
   echo "Usage:"
   echo "+------------------------------------------------------------------------------------------------------+"
   echo "| ./build.sh [--hadoop-version <hadoop version>] [--registry <registry url>] [--author <author name>]  |"
+  echo "|            [--apache-mirror <apache mirror url>]                                                     |"
   echo "+------------------------------------------------------------------------------------------------------+"
   exit 1
 }
@@ -33,6 +34,8 @@ UNKNOWN_REGISTRY="UNKNOWN_REGISTRY"
 REGISTRY=$UNKNOWN_REGISTRY
 HADOOP_VERSION=2.8.5
 AUTHOR=$(whoami)
+# If you are based in China, you could pass --apache-mirror <a_mirror_url> when building this.
+APACHE_MIRROR="https://dlcdn.apache.org"
 
 while (( "$#" )); do
   case $1 in
@@ -51,6 +54,10 @@ while (( "$#" )); do
     --help)
       exit_with_usage
       ;;
+    --apache-mirror)
+      APACHE_MIRROR="$2"
+      shift
+      ;;
     --*)
       echo "Error: $1 is not supported"
       exit_with_usage
@@ -71,10 +78,11 @@ if [ "$REGISTRY" == $UNKNOWN_REGISTRY ]; \
 fi
 
 HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
-HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
-echo "HADOOP_URL is $HADOOP_URL"
+ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
+HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
+echo "HADOOP_URL is either ${HADOOP_URL} or ${ARCHIVE_HADOOP_URL}"
 if [ ! -e "$HADOOP_FILE" ]; \
-  then wget "$HADOOP_URL"; \
+  then wget "${HADOOP_URL}" || wget "$ARCHIVE_HADOOP_URL"; \
   else echo "${HADOOP_FILE} has been downloaded"; \
 fi