You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by xv...@apache.org on 2021/01/31 16:36:06 UTC
[druid] branch master updated: move integration tests from
ZooKeeper 3.4.x to 3.5.x (#10786)
This is an automated email from the ASF dual-hosted git repository.
xvrl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new c346ce6 move integration tests from ZooKeeper 3.4.x to 3.5.x (#10786)
c346ce6 is described below
commit c346ce64b1497f8bbd547bee16fa98cc0d3c74f2
Author: Xavier Léauté <xv...@apache.org>
AuthorDate: Sun Jan 31 08:35:39 2021 -0800
move integration tests from ZooKeeper 3.4.x to 3.5.x (#10786)
* move integration tests from ZooKeeper 3.4.x to 3.5.x
* run a subset of our integration tests with ZK 3.4 for backwards compatibility testing.
* remove need to build separate docker-base image
- use multi-stage build for the base image
- use openjdk base image instead of building our own JDK base
- workaround Debian not including MySQL by using MariaDB
- download mysql connector directly instead of using distro version
* fix incorrect openssl command failing on Debian
* keep mysql connector version in sync with pom version
---
.travis.yml | 18 +++++++
integration-tests/docker-base/README.md | 31 -----------
integration-tests/docker-base/jdk11/Dockerfile | 62 ----------------------
integration-tests/docker-base/jdk8/Dockerfile | 27 ----------
integration-tests/docker/Dockerfile | 23 ++++----
.../{docker-base/setup.sh => docker/base-setup.sh} | 23 ++++----
integration-tests/docker/docker-compose.base.yml | 2 +
.../docker/service-supervisords/zookeeper.conf | 2 +-
.../tls/generate-server-certs-and-keystores.sh | 2 +-
integration-tests/pom.xml | 1 +
.../script/docker_build_containers.sh | 8 +--
11 files changed, 54 insertions(+), 145 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index bfdd495..d5c9391 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -26,6 +26,7 @@ cache:
env:
global:
+ - ZK_VERSION=3.5 # default ZK version for integration tests
- DOCKER_IP=127.0.0.1 # for integration tests
- MVN="mvn -B"
- > # Various options to make execution of maven goals faster (e.g., mvn install)
@@ -513,6 +514,16 @@ jobs:
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager'
+ # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
+ - <<: *integration_tests
+ name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) leadership and high availability integration tests"
+ jdk: openjdk8
+ env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
+ - <<: *integration_kafka_format_tests
+ name: "(Compile=openjdk8, Run=openjdk8, ZK=3.4) Kafka index integration test with various formats"
+ env: TESTNG_GROUPS='-Dgroups=kafka-data-format' JVM_RUNTIME='-Djvm.runtime=8' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
# END - Integration tests for Compile with Java 8 and Run with Java 8
# START - Integration tests for Compile with Java 8 and Run with Java 11
@@ -575,6 +586,13 @@ jobs:
name: "(Compile=openjdk8, Run=openjdk11) leadership and high availability integration tests"
jdk: openjdk8
env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager'
+
+ # Subset of integration tests to run with ZooKeeper 3.4.x for backwards compatibility
+ - <<: *integration_tests
+ name: "(Compile=openjdk8, Run=openjdk11, ZK=3.4) leadership and high availability integration tests"
+ jdk: openjdk8
+ env: TESTNG_GROUPS='-Dgroups=high-availability' JVM_RUNTIME='-Djvm.runtime=11' USE_INDEXER='middleManager' ZK_VERSION=3.4
+
# END - Integration tests for Compile with Java 8 and Run with Java 11
- &integration_batch_index_k8s
diff --git a/integration-tests/docker-base/README.md b/integration-tests/docker-base/README.md
deleted file mode 100644
index 00923f9..0000000
--- a/integration-tests/docker-base/README.md
+++ /dev/null
@@ -1,31 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-## Base Docker Image for Integration Tests
-
-### Building Docker Image for Integration Tests
-
-Run the following commands from <DRUIDS_REPO_DIR>/integration-tests/docker-base
-
-To build docker image for Java JDK8:
-- docker build -t druidbase:<NEW_TAG> -f jdk8/Dockerfile .
-
-To build docker image for Java JDK11:
-- docker build -t druidbase:<NEW_TAG> -f jdk11/Dockerfile .
-
diff --git a/integration-tests/docker-base/jdk11/Dockerfile b/integration-tests/docker-base/jdk11/Dockerfile
deleted file mode 100644
index c3ea66d..0000000
--- a/integration-tests/docker-base/jdk11/Dockerfile
+++ /dev/null
@@ -1,62 +0,0 @@
-# Based on the following projects/files:
-# - SequenceIQ hadoop-docker project hosted at https://github.com/sequenceiq/hadoop-docker
-# - AdoptOpenJDK openjdk-docker project hosted at https://github.com/AdoptOpenJDK/openjdk-docker
-# and modified at the Apache Software Foundation (ASF).
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:16.04
-
-# Install Java JDK 11 (OpenJDK 11.0.5)
-# Sourced from AdoptOpenJDK openjdk-docker project (https://github.com/AdoptOpenJDK/openjdk-docker)
-RUN apt-get update \
- && apt-get install -y --no-install-recommends curl ca-certificates fontconfig locales \
- && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
- && locale-gen en_US.UTF-8 \
- && rm -rf /var/lib/apt/lists/*
-
-RUN set -eux; \
- ARCH="$(dpkg --print-architecture)"; \
- case "${ARCH}" in \
- armhf) \
- ESUM='c6b1fda3f8807028cbfcc34a4ded2e8a5a6b6239d2bcc1f06673ea6b1530df94'; \
- BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_arm_linux_hotspot_11.0.5_10.tar.gz'; \
- ;; \
- ppc64el|ppc64le) \
- ESUM='d763481ddc29ac0bdefb24216b3a0bf9afbb058552682567a075f9c0f7da5814'; \
- BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_ppc64le_linux_hotspot_11.0.5_10.tar.gz'; \
- ;; \
- amd64|x86_64) \
- ESUM='6dd0c9c8a740e6c19149e98034fba8e368fd9aa16ab417aa636854d40db1a161'; \
- BINARY_URL='https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.5_10.tar.gz'; \
- ;; \
- *) \
- echo "Unsupported arch: ${ARCH}"; \
- exit 1; \
- ;; \
- esac; \
- curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \
- echo "${ESUM} */tmp/openjdk.tar.gz" | sha256sum -c -; \
- mkdir -p /opt/java/openjdk; \
- cd /opt/java/openjdk; \
- tar -xf /tmp/openjdk.tar.gz --strip-components=1; \
- rm -rf /tmp/openjdk.tar.gz;
-
-ENV JAVA_HOME=/opt/java/openjdk \
- PATH="/opt/java/openjdk/bin:$PATH"
-
-# Bundle everything into one script so cleanup can reduce image size.
-# Otherwise docker's layered images mean that things are not actually deleted.
-
-COPY setup.sh /root/setup.sh
-RUN chmod 0755 /root/setup.sh && /root/setup.sh
diff --git a/integration-tests/docker-base/jdk8/Dockerfile b/integration-tests/docker-base/jdk8/Dockerfile
deleted file mode 100644
index 7784bf5..0000000
--- a/integration-tests/docker-base/jdk8/Dockerfile
+++ /dev/null
@@ -1,27 +0,0 @@
-# Based on the SequenceIQ hadoop-docker project hosted at
-# https://github.com/sequenceiq/hadoop-docker, and modified at
-# the Apache Software Foundation (ASF).
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:16.04
-
-# Install Java JDK 8
-RUN apt-get update \
- && apt-get install -y openjdk-8-jdk
-
-# Bundle everything into one script so cleanup can reduce image size.
-# Otherwise docker's layered images mean that things are not actually deleted.
-
-COPY setup.sh /root/setup.sh
-RUN chmod 0755 /root/setup.sh && /root/setup.sh
\ No newline at end of file
diff --git a/integration-tests/docker/Dockerfile b/integration-tests/docker/Dockerfile
index 54fab63..10e82b9 100644
--- a/integration-tests/docker/Dockerfile
+++ b/integration-tests/docker/Dockerfile
@@ -13,16 +13,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+ARG JDK_VERSION=8-slim
+FROM openjdk:$JDK_VERSION as druidbase
-# This is default value for base image in case DOCKER_IMAGE is not given when building
-ARG DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1
-# Base image is built from integration-tests/docker-base in the Druid repo
-FROM $DOCKER_IMAGE
+# Bundle everything into one script so cleanup can reduce image size.
+# Otherwise docker's layered images mean that things are not actually deleted.
+
+COPY base-setup.sh /root/base-setup.sh
+RUN /root/base-setup.sh && rm -f /root/base-setup.sh
+
+FROM druidbase
+ARG MYSQL_VERSION
# Verify Java version
-ARG DOCKER_IMAGE
-ENV DOCKER_IMAGE_USED=$DOCKER_IMAGE
-RUN echo "Built using base docker image DOCKER_IMAGE_USED=$DOCKER_IMAGE_USED"
RUN java -version
RUN echo "[mysqld]\ncharacter-set-server=utf8\ncollation-server=utf8_bin\n" >> /etc/mysql/my.cnf
@@ -37,9 +40,9 @@ RUN find /var/lib/mysql -type f -exec touch {} \; && /etc/init.d/mysql start \
ADD lib/* /usr/local/druid/lib/
# Download the MySQL Java connector
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends apt-utils libmysql-java
-RUN ln -sf /usr/share/java/mysql-connector-java.jar /usr/local/druid/lib/mysql-connector-java.jar
+# target path must match the exact path referenced in environment-configs/common
+RUN wget -q "https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_VERSION/mysql-connector-java-$MYSQL_VERSION.jar" \
+ -O /usr/local/druid/lib/mysql-connector-java.jar
# Add sample data
# touch is needed because OverlayFS's copy-up operation breaks POSIX standards. See https://github.com/docker/for-linux/issues/72.
diff --git a/integration-tests/docker-base/setup.sh b/integration-tests/docker/base-setup.sh
old mode 100644
new mode 100755
similarity index 76%
rename from integration-tests/docker-base/setup.sh
rename to integration-tests/docker/base-setup.sh
index 3b9d27d..186874a
--- a/integration-tests/docker-base/setup.sh
+++ b/integration-tests/docker/base-setup.sh
@@ -25,24 +25,29 @@ apt-get update
apt-get install -y wget
# MySQL (Metadata store)
-apt-get install -y mysql-server
+apt-get install -y default-mysql-server
# Supervisor
apt-get install -y supervisor
# Zookeeper
-#ZK_VERSION=3.5.8
-#ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
+install_zk() {
+ wget -q -O /tmp/$ZK_TAR.tar.gz "https://archive.apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
+ tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
+ cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
+ rm /tmp/$ZK_TAR.tar.gz
+}
-ZK_VERISON=3.4.14
+ZK_VERSION=3.4.14
ZK_TAR=zookeeper-$ZK_VERSION
+install_zk
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.4
-wget -q -O /tmp/$ZK_TAR.tar.gz "https://apache.org/dist/zookeeper/zookeeper-$ZK_VERSION/$ZK_TAR.tar.gz"
-tar -xzf /tmp/$ZK_TAR.tar.gz -C /usr/local
-cp /usr/local/$ZK_TAR/conf/zoo_sample.cfg /usr/local/$ZK_TAR/conf/zoo.cfg
-ln -s /usr/local/$ZK_TAR /usr/local/zookeeper
-rm /tmp/$ZK_TAR.tar.gz
+ZK_VERSION=3.5.9
+ZK_TAR=apache-zookeeper-$ZK_VERSION-bin
+install_zk
+ln -s /usr/local/$ZK_TAR /usr/local/zookeeper-3.5
# Kafka
# Match the version to the Kafka client used by KafkaSupervisor
diff --git a/integration-tests/docker/docker-compose.base.yml b/integration-tests/docker/docker-compose.base.yml
index a8f63be..dfe6b89 100644
--- a/integration-tests/docker/docker-compose.base.yml
+++ b/integration-tests/docker/docker-compose.base.yml
@@ -53,6 +53,8 @@ services:
- ./service-supervisords/kafka.conf:/usr/lib/druid/conf/kafka.conf
env_file:
- ./environment-configs/common
+ environment:
+ - ZK_VERSION
druid-metadata-storage:
image: druid/cluster
diff --git a/integration-tests/docker/service-supervisords/zookeeper.conf b/integration-tests/docker/service-supervisords/zookeeper.conf
index 1ee5247..6039e26 100644
--- a/integration-tests/docker/service-supervisords/zookeeper.conf
+++ b/integration-tests/docker/service-supervisords/zookeeper.conf
@@ -1,5 +1,5 @@
[program:zookeeper]
-command=/usr/local/zookeeper/bin/zkServer.sh start-foreground
+command=/usr/local/zookeeper-%(ENV_ZK_VERSION)s/bin/zkServer.sh start-foreground
user=daemon
priority=0
stdout_logfile=/shared/logs/zookeeper.log
diff --git a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh b/integration-tests/docker/tls/generate-server-certs-and-keystores.sh
index 28bf09a..931b6e4 100755
--- a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh
+++ b/integration-tests/docker/tls/generate-server-certs-and-keystores.sh
@@ -61,7 +61,7 @@ DNS.2 = localhost
EOT
# Generate a server certificate for this machine
-openssl genrsa -out server.key 1024 -sha256
+openssl genrsa -out server.key 1024
openssl req -new -out server.csr -key server.key -reqexts req_ext -config csr.conf
openssl x509 -req -days 3650 -in server.csr -CA root.pem -CAkey root.key -set_serial 0x22222222 -out server.pem -sha256 -extfile csr.conf -extensions req_ext
diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml
index 53c3b37..ddcdaaa 100644
--- a/integration-tests/pom.xml
+++ b/integration-tests/pom.xml
@@ -416,6 +416,7 @@
<DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>${docker.build.skip}</DRUID_INTEGRATION_TEST_SKIP_BUILD_DOCKER>
<DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>${docker.run.skip}</DRUID_INTEGRATION_TEST_SKIP_RUN_DOCKER>
<DRUID_INTEGRATION_TEST_INDEXER>${it.indexer}</DRUID_INTEGRATION_TEST_INDEXER>
+ <MYSQL_VERSION>${mysql.version}</MYSQL_VERSION>
</environmentVariables>
<executable>${project.basedir}/build_run_cluster.sh</executable>
</configuration>
diff --git a/integration-tests/script/docker_build_containers.sh b/integration-tests/script/docker_build_containers.sh
index b578ca7..6bff502 100755
--- a/integration-tests/script/docker_build_containers.sh
+++ b/integration-tests/script/docker_build_containers.sh
@@ -21,18 +21,18 @@ set -e
if [ -z "$DRUID_INTEGRATION_TEST_JVM_RUNTIME" ]
then
- echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Build druid-cluster with Java 8"
- docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+ echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is not set. Building druid-cluster with default Java version"
+ docker build -t druid/cluster --build-arg MYSQL_VERSION $SHARED_DIR/docker
else
echo "\$DRUID_INTEGRATION_TEST_JVM_RUNTIME is set with value ${DRUID_INTEGRATION_TEST_JVM_RUNTIME}"
case "${DRUID_INTEGRATION_TEST_JVM_RUNTIME}" in
8)
echo "Build druid-cluster with Java 8"
- docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-1.8.0_191-1 $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=8-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
;;
11)
echo "Build druid-cluster with Java 11"
- docker build -t druid/cluster --build-arg DOCKER_IMAGE=imply/druiditbase:openjdk-11.0.5-1 $SHARED_DIR/docker
+ docker build -t druid/cluster --build-arg JDK_VERSION=11-slim --build-arg MYSQL_VERSION $SHARED_DIR/docker
;;
*)
echo "Invalid JVM Runtime given. Stopping"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org