You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/10/18 20:45:55 UTC
[airflow] branch master updated: Improves stability of K8S tests by
caching binaries and repeats (#11634)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 4655409 Improves stability of K8S tests by caching binaries and repeats (#11634)
4655409 is described below
commit 4655409982c7286d1c2cafb99d614baeacdeeec8
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sun Oct 18 22:45:00 2020 +0200
Improves stability of K8S tests by caching binaries and repeats (#11634)
* Improves stability of K8S tests by caching binaries and repeats
The K8S tests on CI are controlled from the host, not from
inside of the CI container image. Therefore it needs virtualenv
to run the tests as well as some tools such as helm, kubectl
and kind. While those tools can bee downloaded and installed
on demand, from time to time the download fails intermittently.
This change introduces the following improvements:
* the commands to download and setup kind, helm, kubectl are
repeated up to 4 times in case they fail
* the "bin" directory where those binaries are downloaded is
cached between runs. Only the same combination of
versions of the tools are sharing the same cache.
This way both cases - regular re-runs of the same jobs and
upgrade of tools will be much more stable.
---
.github/workflows/ci.yml | 10 +++++++++
scripts/ci/libraries/_all_libs.sh | 2 ++
scripts/ci/libraries/_kind.sh | 10 +++++----
scripts/ci/libraries/_repeats.sh | 45 +++++++++++++++++++++++++++++++++++++++
4 files changed, 63 insertions(+), 4 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 31ef78c..5de2d3f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -746,6 +746,16 @@ jobs:
with:
path: .build/.kubernetes_venv
key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}"
+ - name: "Cache bin folder with tools for kubernetes testing"
+ uses: actions/cache@v2
+ env:
+ cache-name: cache-kubernetes-tests-bin-v6
+ with:
+ path: .build/bin
+ key: "${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('setup.py') }}\
+-${{ needs.build-info.outputs.defaultKindVersion }}\
+-${{ needs.build-info.outputs.defaultHelmVersion }}\
+-$${{ matrix.kubernetes-version }}"
- name: "Kubernetes Tests"
run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
- name: "Upload KinD logs"
diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
index 3688c03..11c0fef 100755
--- a/scripts/ci/libraries/_all_libs.sh
+++ b/scripts/ci/libraries/_all_libs.sh
@@ -28,6 +28,8 @@ readonly SCRIPTS_CI_DIR
. "${LIBRARIES_DIR}"/_traps.sh
# shellcheck source=scripts/ci/libraries/_initialization.sh
. "${LIBRARIES_DIR}"/_initialization.sh
+# shellcheck source=scripts/ci/libraries/_repeats.sh
+. "${LIBRARIES_DIR}"/_repeats.sh
# shellcheck source=scripts/ci/libraries/_sanity_checks.sh
. "${LIBRARIES_DIR}"/_sanity_checks.sh
# shellcheck source=scripts/ci/libraries/_build_images.sh
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index a5eab5f..ef1ce54 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -51,7 +51,8 @@ function kind::make_sure_kubernetes_tools_are_installed() {
if [[ ! -f "${KIND_BINARY_PATH}" || ${DOWNLOADED_KIND_VERSION} != "${KIND_VERSION}" ]]; then
echo
echo "Downloading Kind version ${KIND_VERSION}"
- curl --fail --location "${KIND_URL}" --output "${KIND_BINARY_PATH}"
+ repeats::run_with_retry 4 \
+ "curl --fail --location '${KIND_URL}' --output '${KIND_BINARY_PATH}'"
chmod a+x "${KIND_BINARY_PATH}"
else
echo "Kind version ok"
@@ -66,7 +67,8 @@ function kind::make_sure_kubernetes_tools_are_installed() {
if [[ ! -f "${KUBECTL_BINARY_PATH}" || ${DOWNLOADED_KUBECTL_VERSION} != "${KUBECTL_VERSION}" ]]; then
echo
echo "Downloading Kubectl version ${KUBECTL_VERSION}"
- curl --fail --location "${KUBECTL_URL}" --output "${KUBECTL_BINARY_PATH}"
+ repeats::run_with_retry 4 \
+ "curl --fail --location '${KUBECTL_URL}' --output '${KUBECTL_BINARY_PATH}'"
chmod a+x "${KUBECTL_BINARY_PATH}"
else
echo "Kubectl version ok"
@@ -81,8 +83,8 @@ function kind::make_sure_kubernetes_tools_are_installed() {
if [[ ! -f "${HELM_BINARY_PATH}" || ${DOWNLOADED_HELM_VERSION} != "${HELM_VERSION}" ]]; then
echo
echo "Downloading Helm version ${HELM_VERSION}"
- curl --location "${HELM_URL}" |
- tar -xvz -O "${SYSTEM}-amd64/helm" >"${HELM_BINARY_PATH}"
+ repeats::run_with_retry 4 \
+ "curl --location '${HELM_URL}' | tar -xvz -O '${SYSTEM}-amd64/helm' >'${HELM_BINARY_PATH}'"
chmod a+x "${HELM_BINARY_PATH}"
else
echo "Helm version ok"
diff --git a/scripts/ci/libraries/_repeats.sh b/scripts/ci/libraries/_repeats.sh
new file mode 100644
index 0000000..96e4582
--- /dev/null
+++ b/scripts/ci/libraries/_repeats.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Repeat the command up to n times in case of failure
+# Parameters:
+# $1 - how many times to repeat
+# $2 - command to repeat (run through bash -c)
+function repeats::run_with_retry() {
+ local num_repeats="${1}"
+ local command="${2}"
+ for ((n=0;n<num_repeats;n++));
+ do
+ local res
+ echo "Attempt no. ${n} to execute ${command}"
+ set +e
+ bash -c "${command}"
+ res=$?
+ set -e
+ if [[ ${res} == "0" ]]; then
+ return 0
+ fi
+ >&2 echo
+ >&2 echo "Unsuccessful attempt no. ${n}. Result: ${res}"
+ >&2 echo
+ done
+ >&2 echo
+ >&2 echo "Giving up after ${num_repeats} attempts!"
+ >&2 echo
+ return ${res}
+}