You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by ti...@apache.org on 2023/11/07 02:15:04 UTC
(pulsar) branch master updated: [improve][build] Change UBUNTU_MIRROR default value (#21520)

This is an automated email from the ASF dual-hosted git repository.

tison pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new 44abba9922e [improve][build] Change UBUNTU_MIRROR default value (#21520)
44abba9922e is described below

commit 44abba9922e5e02099bab79591e2327845772f16
Author: Lari Hotari <lh...@users.noreply.github.com>
AuthorDate: Tue Nov 7 04:14:57 2023 +0200

    [improve][build] Change UBUNTU_MIRROR default value (#21520)
---
 build/pulsar_ci_tool.sh                        | 26 ++++++++++++++++++--------
 docker/pulsar/Dockerfile                       |  4 ++--
 docker/pulsar/pom.xml                          |  2 +-
 tests/docker-images/java-test-image/Dockerfile |  4 ++--
 4 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
index d946edd3957..ae33c3e9166 100755
--- a/build/pulsar_ci_tool.sh
+++ b/build/pulsar_ci_tool.sh
@@ -55,9 +55,11 @@ function ci_dependency_check() {
   _ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@"
 }
 
-function ci_pick_ubuntu_mirror() {
-  echo "Choosing fastest up-to-date ubuntu mirror based on download speed..."
-  UBUNTU_MIRROR=$({
+# Finds fastest up-to-date ubuntu mirror based on download speed
+function ci_find_fast_ubuntu_mirror() {
+  local ubuntu_release=${1:-"$(lsb_release -c 2>/dev/null | cut -f2 || echo "jammy")"}
+  local ubuntu_arch=${2:-"$(dpkg --print-architecture 2>/dev/null || echo "amd64")"}
+  {
     # choose mirrors that are up-to-date by checking the Last-Modified header for
     {
       # randomly choose up to 10 mirrors using http:// protocol
@@ -65,12 +67,20 @@ function ci_pick_ubuntu_mirror() {
       curl -s http://mirrors.ubuntu.com/mirrors.txt | grep '^http://' | shuf -n 10
       # also consider Azure's Ubuntu mirror
       echo http://azure.archive.ubuntu.com/ubuntu/
-    } | xargs -I {} sh -c 'echo "$(curl -m 5 -sI {}dists/$(lsb_release -c | cut -f2)-security/Contents-$(dpkg --print-architecture).gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }'
+    } | xargs -I {} sh -c "ubuntu_release=$ubuntu_release ubuntu_arch=$ubuntu_arch;"'echo "$(curl -m 5 -sI {}dists/${ubuntu_release}/Contents-${ubuntu_arch}.gz|sed s/\\r\$//|grep Last-Modified|awk -F": " "{ print \$2 }" | LANG=C date -f- -u +%s)" "{}"' | sort -rg | awk '{ if (NR==1) TS=$1; if ($1 == TS) print $2 }'
   } | xargs -I {} sh -c 'echo `curl -r 0-102400 -m 5 -s -w %{speed_download} -o /dev/null {}ls-lR.gz` {}' \
-    |sort -g -r |head -1| awk '{ print $2  }')
+    |sort -g -r |head -1| awk '{ print $2  }'
+}
+
+function ci_pick_ubuntu_mirror() {
+  echo "Choosing fastest up-to-date ubuntu mirror based on download speed..."
+  UBUNTU_MIRROR=$(ci_find_fast_ubuntu_mirror)
   if [ -z "$UBUNTU_MIRROR" ]; then
-      # fallback to full mirrors list
-      UBUNTU_MIRROR="mirror://mirrors.ubuntu.com/mirrors.txt"
+      # fallback to no mirror
+      UBUNTU_MIRROR="http://archive.ubuntu.com/ubuntu/"
+      UBUNTU_SECURITY_MIRROR="http://security.ubuntu.com/ubuntu/"
+  else
+      UBUNTU_SECURITY_MIRROR="${UBUNTU_MIRROR}"
   fi
   OLD_MIRROR=$(cat /etc/apt/sources.list | grep '^deb ' | head -1 | awk '{ print $2 }')
   echo "Picked '$UBUNTU_MIRROR'. Current mirror is '$OLD_MIRROR'."
@@ -81,7 +91,7 @@ function ci_pick_ubuntu_mirror() {
   # set the chosen mirror also in the UBUNTU_MIRROR and UBUNTU_SECURITY_MIRROR environment variables
   # that can be used by docker builds
   export UBUNTU_MIRROR
-  export UBUNTU_SECURITY_MIRROR=$UBUNTU_MIRROR
+  export UBUNTU_SECURITY_MIRROR
   # make environment variables available for later GitHub Actions steps
   if [ -n "$GITHUB_ENV" ]; then
     echo "UBUNTU_MIRROR=$UBUNTU_MIRROR" >> $GITHUB_ENV
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
index 4f55dd57ba3..2bd6d402f76 100644
--- a/docker/pulsar/Dockerfile
+++ b/docker/pulsar/Dockerfile
@@ -52,12 +52,12 @@ RUN chmod g+w /pulsar/trino
 FROM ubuntu:22.04
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt
+ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/
 ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/
 ARG JDK_MAJOR_VERSION=17
 
 # Install some utilities
-RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirrors.ubuntu.com/mirrors.txt}|g" \
+RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \
      -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \
      && echo 'Acquire::http::Timeout "30";\nAcquire::ftp::Timeout "30";\nAcquire::Retries "3";' > /etc/apt/apt.conf.d/99timeout_and_retries \
      && apt-get update \
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
index 21babe4aad2..1c29fa3f00c 100644
--- a/docker/pulsar/pom.xml
+++ b/docker/pulsar/pom.xml
@@ -48,7 +48,7 @@
   </dependencies>
 
   <properties>
-    <UBUNTU_MIRROR>mirror://mirrors.ubuntu.com/mirrors.txt</UBUNTU_MIRROR>
+    <UBUNTU_MIRROR>http://archive.ubuntu.com/ubuntu/</UBUNTU_MIRROR>
     <UBUNTU_SECURITY_MIRROR>http://security.ubuntu.com/ubuntu/</UBUNTU_SECURITY_MIRROR>
     <IMAGE_JDK_MAJOR_VERSION>17</IMAGE_JDK_MAJOR_VERSION>
   </properties>
diff --git a/tests/docker-images/java-test-image/Dockerfile b/tests/docker-images/java-test-image/Dockerfile
index e1616ecfbae..6a9c7d10331 100644
--- a/tests/docker-images/java-test-image/Dockerfile
+++ b/tests/docker-images/java-test-image/Dockerfile
@@ -33,11 +33,11 @@ RUN chmod a+rx /pulsar/bin/*
 WORKDIR /pulsar
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt
+ARG UBUNTU_MIRROR=http://archive.ubuntu.com/ubuntu/
 ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/
 ARG JDK_MAJOR_VERSION=17
 
-RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirrors.ubuntu.com/mirrors.txt}|g" \
+RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-http://archive.ubuntu.com/ubuntu/}|g" \
      -e "s|http://security\.ubuntu\.com/ubuntu/|${UBUNTU_SECURITY_MIRROR:-http://security.ubuntu.com/ubuntu/}|g" /etc/apt/sources.list \
      && apt-get update \
      && apt-get -y dist-upgrade \