You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/04/13 02:55:17 UTC
[impala] branch master updated: IMPALA-7995: part 2: Jenkins script
to automate e2e tests
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 3895828 IMPALA-7995: part 2: Jenkins script to automate e2e tests
3895828 is described below
commit 3895828c4e973d641e1a0d60a3fec348cbe287a5
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Mon Feb 11 17:07:16 2019 -0800
IMPALA-7995: part 2: Jenkins script to automate e2e tests
Testing:
Ran on https://jenkins.impala.io/job/ubuntu-16.04-dockerised-tests/
Change-Id: I67a3562904c959b51f4bde52107193c4002cb1ce
Reviewed-on: http://gerrit.cloudera.org:8080/12937
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../dockerized-impala-bootstrap-and-test.sh | 49 +++++++++++++
bin/jenkins/dockerized-impala-run-tests.sh | 80 ++++++++++++++++++++++
docker/configure_test_network.sh | 2 +
3 files changed, 131 insertions(+)
diff --git a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
new file mode 100755
index 0000000..2454894
--- /dev/null
+++ b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script sets up docker on Ubuntu 16.04 and then run's
+# Impala's tests with a dockerised minicluster.
+
+set -eu -o pipefail
+
+ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
+cd "$ROOT_DIR"
+
+source ./bin/bootstrap_system.sh
+
+# Following install instructions from
+# https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-docker-ce-1
+sudo apt-get install -y apt-transport-https ca-certificates curl \
+ gnupg-agent software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+# Bail if the fingerprint isn't what we expected.
+sudo apt-key fingerprint 0EBFCD88 | \
+ grep '9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88'
+sudo add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) stable"
+sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+sudo service docker restart
+sudo groupadd -f docker
+sudo usermod -aG docker $USER
+
+# Execute the tests using su to re-login so that group change made above
+# setup_docker takes effect.
+sudo su $USER -c "./bin/jenkins/dockerized-impala-run-tests.sh"
diff --git a/bin/jenkins/dockerized-impala-run-tests.sh b/bin/jenkins/dockerized-impala-run-tests.sh
new file mode 100755
index 0000000..92898a1
--- /dev/null
+++ b/bin/jenkins/dockerized-impala-run-tests.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# This script runs Impala's tests with a dockerised minicluster.
+# It has been tested on Ubuntu 16.04.
+set -x
+set -eu -o pipefail
+# Helper to source impala-config.sh, which may have unbound variables
+source_impala_config() {
+ set +u
+ . ./bin/impala-config.sh
+ set -u
+}
+source_impala_config
+
+# Check that docker is running and that our user can interact with it.
+docker run hello-world
+
+# Set up the test network.
+DOCKER_NETWORK="test-impala-cluster"
+./docker/configure_test_network.sh $DOCKER_NETWORK
+
+# Pick up the new variables.
+source_impala_config
+
+# Dump diagnostics for networks and check connectivity.
+ifconfig
+ping -c 1 $INTERNAL_LISTEN_HOST
+
+# Check that ssh to localhost via Docker gateway works.
+if ! ssh -n $INTERNAL_LISTEN_HOST "echo 'SSH success!'"; then
+ echo "Failed to ssh, will try to add docker network gateway to known hosts"
+ ssh-keyscan $INTERNAL_LISTEN_HOST >> ~/.ssh/known_hosts
+ ssh -n $INTERNAL_LISTEN_HOST "echo 'SSH success!'"
+fi
+
+# Build Impala and load data with the non-dockerised path.
+# TODO: this is a bit awkward. It would be better to have a single invocation
+# to run everything with docker.
+# Skip building backend tests, which aren't used.
+time -p ./buildall.sh -format -testdata -notests < /dev/null
+# Kill non-dockerised cluster.
+start-impala-cluster.py --kill
+
+# Build the docker images required to start the cluster.
+# parquet-reader is needed for e2e tests but not built for non-test build
+# Build serially to work around IMPALA-8392 TODO: do parallel build when fixed
+make docker_images parquet-reader
+
+source_impala_config
+
+export TEST_START_CLUSTER_ARGS="--docker_network=${DOCKER_NETWORK}"
+export MAX_PYTEST_FAILURES=0
+export NUM_CONCURRENT_TESTS=$(nproc)
+# Frontend tests fail because of localhost hardcoded everywhere
+export FE_TEST=false
+# No need to run backend tests - they are identical with non-docker build.
+export BE_TEST=false
+# TODO: custom cluster tests may provide some useful coverage but require work
+# to make them start up dockerised clusters and will probably make more assumptions
+# about the cluster being tested.
+export CLUSTER_TEST=false
+./bin/run-all-tests.sh
diff --git a/docker/configure_test_network.sh b/docker/configure_test_network.sh
index df5567b..5d6b349 100755
--- a/docker/configure_test_network.sh
+++ b/docker/configure_test_network.sh
@@ -51,3 +51,5 @@ echo "# Configuration to use docker network ${NETWORK_NAME}" \
echo "export INTERNAL_LISTEN_HOST=${GATEWAY}" >> "$IMPALA_HOME"/bin/impala-config-local.sh
echo "export DEFAULT_FS=hdfs://\${INTERNAL_LISTEN_HOST}:20500" \
>> "$IMPALA_HOME"/bin/impala-config-local.sh
+echo "export KUDU_MASTER_HOSTS=\${INTERNAL_LISTEN_HOST}" \
+ >> "$IMPALA_HOME"/bin/impala-config-local.sh