You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by su...@apache.org on 2021/03/11 17:08:15 UTC
[druid] branch master updated: Apache archive mirror (#10979)
This is an automated email from the ASF dual-hosted git repository.
suneet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b0432be Apache archive mirror (#10979)
b0432be is described below
commit b0432be07aa449213f36280b89890f14272dc14a
Author: Vyatcheslav Mogilevsky <tr...@gmail.com>
AuthorDate: Thu Mar 11 20:07:51 2021 +0300
Apache archive mirror (#10979)
* Ability to use mirror of archive.apache.org
* Ability to use mirror of archive.apache.org: documentation
* Ability to use mirror of archive.apache.org: fix int test Dockerfile: missing COPY instruction
---
examples/quickstart/tutorial/hadoop/docker/Dockerfile | 3 ++-
integration-tests/README.md | 6 ++++++
integration-tests/docker/Dockerfile | 3 ++-
integration-tests/docker/base-setup.sh | 5 +++--
integration-tests/script/docker_build_containers.sh | 6 +++---
5 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile
index c9c30ac..0237d61 100644
--- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile
+++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile
@@ -45,7 +45,8 @@ ENV JAVA_HOME /usr/lib/jvm/zulu-8
ENV PATH $PATH:$JAVA_HOME/bin
# hadoop
-RUN curl -s https://archive.apache.org/dist/hadoop/core/hadoop-2.8.5/hadoop-2.8.5.tar.gz | tar -xz -C /usr/local/
+ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org
+RUN curl -s ${APACHE_ARCHIVE_MIRROR_HOST}/dist/hadoop/core/hadoop-2.8.5/hadoop-2.8.5.tar.gz | tar -xz -C /usr/local/
RUN cd /usr/local && ln -s ./hadoop-2.8.5 hadoop
ENV HADOOP_PREFIX /usr/local/hadoop
diff --git a/integration-tests/README.md b/integration-tests/README.md
index 3bba04c..1d00e56 100644
--- a/integration-tests/README.md
+++ b/integration-tests/README.md
@@ -46,6 +46,12 @@ environment variable to localhost on your system, as follows:
export DOCKER_IP=127.0.0.1
```
+Optionally, you can also set `APACHE_ARCHIVE_MIRROR_HOST` to override `https://archive.apache.org` host. This host is used to download archives such as hadoop and kafka during building docker images:
+
+```
+export APACHE_ARCHIVE_MIRROR_HOST=https://example.com/remote-generic-repo
+```
+
## Running tests
To run all tests from a test group using docker and mvn run the following command:
diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile
index 10e82b9..421d347 100644
--- a/integration-tests/docker/Dockerfile
+++ b/integration-tests/docker/Dockerfile
@@ -20,7 +20,8 @@ FROM openjdk:$JDK_VERSION as druidbase
# Otherwise docker's layered images mean that things are not actually deleted.
COPY base-setup.sh /root/base-setup.sh
-RUN /root/base-setup.sh && rm -f /root/base-setup.sh
+ARG APACHE_ARCHIVE_MIRROR_HOST=https://archive.apache.org
+RUN APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST} /root/base-setup.sh && rm -f /root/base-setup.sh
FROM druidbase
ARG MYSQL_VERSION
diff --git a/integration-tests/docker/base-setup.sh b/integration-tests/docker/base-setup.sh
index 186874a..e68f6bc 100755
--- a/integration-tests/docker/base-setup.sh
+++ b/integration-tests/docker/base-setup.sh
@@ -18,6 +18,7 @@ set -e
set -u
export DEBIAN_FRONTEND=noninteractive
+APACHE_ARCHIVE_MIRROR_HOST=${APACHE_ARCHIVE_MIRROR_HOST:-https://archive.apache.org}
apt-get update
@@ -33,7 +34,7 @@ apt-get install -y supervisor
# Zookeeper
install_zk() {
- wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
+ wget -q -O /tmp/$ZK_TAR.tar.gz "$APACHE_ARCHIVE_MIRROR_HOST/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
rm /tmp/$ZK_TAR.tar.gz
@@ -52,7 +53,7 @@ ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
# Kafka
# Match the version to the Kafka client used by KafkaSupervisor
KAFKA_VERSION=2.7.0
-wget -q -O /tmp/kafka_2.13-$KAFKA_VERSION.tgz "https://apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
+wget -q -O /tmp/kafka_2.13-$KAFKA_VERSION.tgz "$APACHE_ARCHIVE_MIRROR_HOST/dist/kafka/$KAFKA_VERSION/kafka_2.13-$KAFKA_VERSION.tgz"
tar -xzf /tmp/kafka_2.13-$KAFKA_VERSION.tgz -C /usr/local
ln -s /usr/local/kafka_2.13-$KAFKA_VERSION /usr/local/kafka
rm /tmp/kafka_2.13-$KAFKA_VERSION.tgz
diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh
index 6bff502..ef3df47 100755
--- a/integration-tests/script/docker_build_containers.sh
+++ b/integration-tests/script/docker_build_containers.sh
@@ -28,11 +28,11 @@ else
case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
8)
echo "Build druid-cluster with Java 8"
- docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
;;
11)
echo "Build druid-cluster with Java 11"
- docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION --build-arg APACHE_ARCHIVE_MIRROR_HOST $SHARED_DIR/docker
;;
*)
echo "Invalid JVM Runtime given. Stopping"
@@ -44,5 +44,5 @@ fi
# Build Hadoop docker if needed
if [ -n "$DRUID_INTEGRATION_TEST_BUILD_HADOOP_DOCKER" ] && [ "$DRUID_INTEGRATION_TEST_BUILD_HADOOP_DOCKER" == true ]
then
- docker build -t druid-it/hadoop:2.8.5 $HADOOP_DOCKER_DIR
+ docker build -t druid-it/hadoop:2.8.5 --build-arg APACHE_ARCHIVE_MIRROR_HOST $HADOOP_DOCKER_DIR
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org