You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/04/07 18:08:25 UTC
[airflow] branch master updated: Run kubernetes tests in parallel
(#15222)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new ea0710e Run kubernetes tests in parallel (#15222)
ea0710e is described below
commit ea0710edc106a9091d666a3be629201ad7cbbcad
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Apr 7 20:08:00 2021 +0200
Run kubernetes tests in parallel (#15222)
---
.github/workflows/ci.yml | 61 ++++++------
.gitignore | 2 +-
.rat-excludes | 2 +-
TESTING.rst | 6 ++
breeze | 2 +-
.../ci/images/ci_wait_for_and_verify_ci_image.sh | 3 +-
.../ci/images/ci_wait_for_and_verify_prod_image.sh | 17 ++--
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 10 +-
...tup_cluster_and_deploy_airflow_to_kubernetes.sh | 6 +-
...cluster_and_run_kubernetes_tests_single_job.sh} | 51 +++++-----
...lusters_and_run_kubernetes_tests_in_parallel.sh | 106 +++++++++++++++++++++
scripts/ci/kubernetes/kind-cluster-conf.yaml | 4 +-
scripts/ci/libraries/_build_images.sh | 23 +----
scripts/ci/libraries/_docker_engine_resources.sh | 10 +-
scripts/ci/libraries/_initialization.sh | 10 +-
scripts/ci/libraries/_kind.sh | 57 +++++------
scripts/ci/libraries/_parallel.sh | 15 ++-
scripts/ci/libraries/_testing.sh | 2 +-
scripts/ci/libraries/_verbosity.sh | 2 +-
scripts/ci/selective_ci_checks.sh | 2 +
scripts/ci/testing/ci_run_quarantined_tests.sh | 3 +
scripts/in_container/entrypoint_ci.sh | 4 +-
22 files changed, 248 insertions(+), 150 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 589ac50..9719928 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -132,6 +132,7 @@ jobs:
pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
+ kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
@@ -147,7 +148,6 @@ jobs:
postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
- kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
@@ -647,7 +647,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
name: >
coverage-helm
- path: "./files/coverage.xml"
+ path: "./files/coverage*.xml"
retention-days: 7
tests-postgres:
@@ -705,7 +705,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
name: >
coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
- path: "./files/coverage.xml"
+ path: "./files/coverage*.xml"
retention-days: 7
tests-mysql:
@@ -761,7 +761,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
uses: actions/upload-artifact@v2
with:
name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
- path: "./files/coverage.xml"
+ path: "./files/coverage*.xml"
retention-days: 7
tests-sqlite:
@@ -815,7 +815,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
uses: actions/upload-artifact@v2
with:
name: coverage-sqlite-${{matrix.python-version}}
- path: ./files/coverage.xml
+ path: ./files/coverage*.xml
retention-days: 7
tests-quarantined:
@@ -886,7 +886,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
uses: actions/upload-artifact@v2
with:
name: coverage-quarantined-${{ matrix.backend }}
- path: "./files/coverage.xml"
+ path: "./files/coverage*.xml"
retention-days: 7
upload-coverage:
@@ -966,27 +966,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-kubernetes:
timeout-minutes: 50
- name: K8s ${{matrix.python-version}} ${{matrix.kubernetes-version}} ${{matrix.kubernetes-mode}}
+ name: K8s tests
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, prod-images]
- strategy:
- matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- kubernetes-version: ${{ fromJson(needs.build-info.outputs.kubernetesVersions) }}
- kubernetes-mode: ${{ fromJson(needs.build-info.outputs.kubernetesModes) }}
- exclude: ${{ fromJson(needs.build-info.outputs.kubernetesExclude) }}
- fail-fast: false
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
- PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}"
- KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}"
- KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
+ KUBERNETES_MODE: "image"
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
+ CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
+ ${{needs.build-info.outputs.pythonVersionsListAsString}}
+ CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
+ ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
if: needs.build-info.outputs.run-kubernetes-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -999,45 +994,43 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- - name: "Prepare PROD Image"
- run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- - name: "Setup cluster and deploy Airflow"
- id: setp-cluster-deploy-app
- run: ./scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
- env:
- # We have the right image pulled already by the previous step
- SKIP_BUILDING_PROD_IMAGE: "true"
+ - name: "Get all PROD images"
+ run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v2
with:
- path: ".build/.kubernetes_venv_ ${{ needs.build-info.outputs.defaultPythonVersion }}"
+ path: ".build/.kubernetes_venv"
key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\
+-${{needs.build-info.outputs.kubernetesVersionsListAsString}}
+-${{needs.build-info.outputs.pythonVersionsListAsString}}
-${{ hashFiles('setup.py','setup.cfg') }}"
- restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-"
+ restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\
+-${{needs.build-info.outputs.kubernetesVersionsListAsString}}
+-${{needs.build-info.outputs.pythonVersionsListAsString}}"
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v2
with:
- path: ".build/bin"
- key: "bin-${{ matrix.kubernetes-version }}\
+ path: ".build/kubernetes-bin"
+ key: "kubernetes-binaries
-${{ needs.build-info.outputs.defaultKindVersion }}\
-${{ needs.build-info.outputs.defaultHelmVersion }}"
- restore-keys: "bin-${{ matrix.kubernetes-version }}"
+ restore-keys: "kubernetes-binaries"
- name: "Kubernetes Tests"
- run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+ run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
- name: "Upload KinD logs"
uses: actions/upload-artifact@v2
if: failure()
with:
name: >
- kind-logs-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}
+ kind-logs-
path: /tmp/kind_logs_*
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v2
with:
name: >
- coverage-k8s-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}}
- path: "./files/coverage.xml"
+ coverage-k8s-
+ path: "./files/coverage*.xml"
retention-days: 7
push-prod-images-to-github-registry:
diff --git a/.gitignore b/.gitignore
index cd07d5f..f9ad568 100644
--- a/.gitignore
+++ b/.gitignore
@@ -62,7 +62,7 @@ htmlcov/
.coverage.*
.cache
nosetests.xml
-coverage.xml
+coverage*.xml
*,cover
.hypothesis/
.pytest_cache
diff --git a/.rat-excludes b/.rat-excludes
index 8ec47aa..f6ee041 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -72,7 +72,7 @@ node_modules/*
coverage/*
git_version
flake8_diff.sh
-coverage.xml
+coverage*.xml
_sources/*
rat-results.txt
diff --git a/TESTING.rst b/TESTING.rst
index 07a3e73..766be56 100644
--- a/TESTING.rst
+++ b/TESTING.rst
@@ -629,6 +629,12 @@ Entering shell with Kubernetes Cluster
This shell is prepared to run Kubernetes tests interactively. It has ``kubectl`` and ``kind`` cli tools
available in the path, it has also activated virtualenv environment that allows you to run tests via pytest.
+The binaries are available in ./.build/kubernetes-bin/``KUBERNETES_VERSION`` path.
+The virtualenv is available in ./.build/.kubernetes_venv/``KIND_CLUSTER_NAME``_host_python_``HOST_PYTHON_VERSION``
+
+Where ``KIND_CLUSTER_NAME`` is the name of the cluster and ``HOST_PYTHON_VERSION`` is the version of python
+in the host.
+
You can enter the shell via those scripts
./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in
diff --git a/breeze b/breeze
index 302cebb..df24b14 100755
--- a/breeze
+++ b/breeze
@@ -3418,7 +3418,7 @@ function breeze::run_breeze_command() {
set +u
local dc_run_file
local run_command="breeze::run_command"
- if [[ ${DRY_RUN_DOCKER} != "false" ]]; then
+ if [[ ${DRY_RUN_DOCKER=} != "false" ]]; then
run_command="breeze::print_command"
fi
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
index e83b0d6..29daca7 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
+++ b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
@@ -31,7 +31,6 @@ shift
function pull_ci_image() {
local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
start_end::group_start "Pulling ${image_name_with_tag} image"
-
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}"
start_end::group_end
@@ -39,7 +38,9 @@ function pull_ci_image() {
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
+start_end::group_start "Configure Docker Registry"
build_image::configure_docker_registry
+start_end::group_end
export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
diff --git a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
index b4a482a..84c310e 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
+++ b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
@@ -28,17 +28,11 @@ shift
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-function pull_prod_image() {
- local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}"
-
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
- start_end::group_end
-}
-
push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
+start_end::group_start "Configure Docker Registry"
build_image::configure_docker_registry
+start_end::group_end
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
@@ -48,8 +42,11 @@ push_pull_remove_images::wait_for_github_registry_image \
"${AIRFLOW_PROD_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
start_end::group_end
+start_end::group_start "Pulling the PROD Image"
build_images::prepare_prod_build
-
-pull_prod_image
+image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+verbosity::print_info "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}"
+push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
+start_end::group_end
verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index 65f885c..fbcf66b 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -62,7 +62,7 @@ function parse_tests_to_run() {
"--durations=100"
"--cov=airflow/"
"--cov-config=.coveragerc"
- "--cov-report=xml:files/coverage.xml"
+ "--cov-report=xml:files/coverage=${KIND_CLUSTER_NAME}.xml"
"--color=yes"
"--maxfail=50"
"--pythonwarnings=ignore::DeprecationWarning"
@@ -73,12 +73,12 @@ function parse_tests_to_run() {
}
function create_virtualenv() {
- start_end::group_start "Creating virtualenv"
HOST_PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")')
readonly HOST_PYTHON_VERSION
- local virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv_${HOST_PYTHON_VERSION}"
+ local virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv/${KIND_CLUSTER_NAME}_host_python_${HOST_PYTHON_VERSION}"
+ mkdir -pv "${BUILD_CACHE_DIR}/.kubernetes_venv/"
if [[ ! -d ${virtualenv_path} ]]; then
echo
echo "Creating virtualenv at ${virtualenv_path}"
@@ -95,14 +95,10 @@ function create_virtualenv() {
pip install -e ".[kubernetes]" \
--constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${HOST_PYTHON_VERSION}.txt"
-
- start_end::group_end
}
function run_tests() {
- start_end::group_start "Running K8S tests"
pytest "${pytest_args[@]}" "${tests_to_run[@]}"
- start_end::group_end
}
cd "${AIRFLOW_SOURCES}" || exit 1
diff --git a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
index ec493f8..1e0fa36 100755
--- a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
+++ b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh
@@ -15,9 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+export SKIP_BUILDING_PROD_IMAGE="true"
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-set -euo pipefail
traps::add_trap "kind::dump_kind_logs" EXIT HUP INT TERM
@@ -25,7 +27,7 @@ kind::make_sure_kubernetes_tools_are_installed
kind::get_kind_cluster_name
kind::perform_kind_cluster_operation "start"
build_images::prepare_prod_build
-build_images::build_prod_images_with_group
+build_images::build_prod_images
kind::build_image_for_kubernetes_tests
kind::load_image_to_kind_cluster
kind::deploy_airflow_with_helm
diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
similarity index 52%
copy from scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
copy to scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
index e83b0d6..9b0d86f 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
+++ b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh
@@ -15,43 +15,40 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
if [[ $1 == "" ]]; then
- >&2 echo "Requires python MAJOR/MINOR version as first parameter"
+ >&2 echo "Requires Kubernetes_version as first parameter"
exit 1
fi
-
-export PYTHON_MAJOR_MINOR_VERSION=$1
+export KUBERNETES_VERSION=$1
shift
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-function pull_ci_image() {
- local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- start_end::group_start "Pulling ${image_name_with_tag} image"
-
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}"
- start_end::group_end
-
-}
-
-push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
-
-build_image::configure_docker_registry
+if [[ $1 == "" ]]; then
+ >&2 echo "Requires Python Major/Minor version as second parameter"
+ exit 1
+fi
+export PYTHON_MAJOR_MINOR_VERSION=$1
+shift
-export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
+# Requires PARALLEL_JOB_STATUS
-start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear"
+if [[ -z "${PARALLEL_JOB_STATUS=}" ]]; then
+ echo "Needs PARALLEL_JOB_STATUS to be set"
+ exit 1
+fi
-push_pull_remove_images::wait_for_github_registry_image \
- "${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+echo
+echo "KUBERNETES_VERSION: ${KUBERNETES_VERSION}"
+echo "PYTHON_MAJOR_MINOR_VERSION: ${PYTHON_MAJOR_MINOR_VERSION}"
+echo
-build_images::prepare_ci_build
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-pull_ci_image
+kind::get_kind_cluster_name
+trap 'echo $? > "${PARALLEL_JOB_STATUS}"; kind::perform_kind_cluster_operation "stop"' EXIT HUP INT TERM
-verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
+"$( dirname "${BASH_SOURCE[0]}" )/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh"
-start_end::group_end
+export CLUSTER_FORWARDED_PORT="${FORWARDED_PORT_NUMBER}"
+"$( dirname "${BASH_SOURCE[0]}" )/ci_run_kubernetes_tests.sh"
diff --git a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
new file mode 100755
index 0000000..88aa2cd
--- /dev/null
+++ b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -euo pipefail
+
+# We cannot perform full initialization because it will be done later in the "single run" scripts
+# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed
+LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd)
+# shellcheck source=scripts/ci/libraries/_all_libs.sh
+source "${LIBRARIES_DIR}/_all_libs.sh"
+export SEMAPHORE_NAME="kubernetes-tests"
+
+function get_maximum_parallel_k8s_jobs() {
+ docker_engine_resources::get_available_cpus_in_docker
+ if [[ -n ${RUNS_ON=} && ${RUNS_ON} != *"self-hosted"* ]]; then
+ echo
+ echo "${COLOR_YELLOW}This is a Github Public runner - for now we are forcing max parallel K8S tests jobs to 1 for those${COLOR_RESET}"
+ echo
+ export MAX_PARALLEL_K8S_JOBS="1"
+ else
+ if [[ ${MAX_PARALLEL_K8S_JOBS=} != "" ]]; then
+ echo
+ echo "${COLOR_YELLOW}Maximum parallel k8s jobs forced vi MAX_PARALLEL_K8S_JOBS = ${MAX_PARALLEL_K8S_JOBS}${COLOR_RESET}"
+ echo
+ else
+ MAX_PARALLEL_K8S_JOBS=${CPUS_AVAILABLE_FOR_DOCKER}
+ echo
+ echo "${COLOR_YELLOW}Maximum parallel k8s jobs set to number of CPUs available for Docker = ${MAX_PARALLEL_K8S_JOBS}${COLOR_RESET}"
+ echo
+ fi
+ fi
+ export MAX_PARALLEL_K8S_JOBS
+}
+
+# Launches parallel building of images. Redirects output to log set the right directories
+# $1 - test_specification
+# $2 - bash file to execute in parallel
+function run_kubernetes_test() {
+ local kubernetes_version=$1
+ local python_version=$2
+ local job="Cluster-${kubernetes_version}-python-${python_version}"
+
+ mkdir -p "${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}"
+ export JOB_LOG="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}/stdout"
+ export PARALLEL_JOB_STATUS="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}/status"
+ echo "Starting K8S tests for kubernetes version ${kubernetes_version}, python version: ${python_version}"
+ parallel --ungroup --bg --semaphore --semaphorename "${SEMAPHORE_NAME}" \
+ --jobs "${MAX_PARALLEL_K8S_JOBS}" \
+ "$(dirname "${BASH_SOURCE[0]}")/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh" \
+ "${kubernetes_version}" "${python_version}" >"${JOB_LOG}" 2>&1
+}
+
+function run_k8s_tests_in_parallel() {
+ parallel::cleanup_runner
+ start_end::group_start "Monitoring k8s tests"
+ parallel::initialize_monitoring
+ parallel::monitor_progress
+
+ # In case there are more kubernetes versions than strings, we can reuse python versions so we add it twice here
+ local repeated_python_versions
+ # shellcheck disable=SC2206
+ repeated_python_versions=(${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING})
+ local index=0
+ for kubernetes_version in ${CURRENT_KUBERNETES_VERSIONS_AS_STRING}
+ do
+ index=$((index + 1))
+ python_version=${repeated_python_versions[${index}]}
+ FORWARDED_PORT_NUMBER=$((38080 + index))
+ export FORWARDED_PORT_NUMBER
+ API_SERVER_PORT=$((19090 + index))
+ export API_SERVER_PORT
+ run_kubernetes_test "${kubernetes_version}" "${python_version}" "${@}"
+ done
+ set +e
+ parallel --semaphore --semaphorename "${SEMAPHORE_NAME}" --wait
+ parallel::kill_monitor
+ set -e
+ start_end::group_end
+}
+
+initialization::set_output_color_variables
+
+parallel::make_sure_gnu_parallel_is_installed
+parallel::make_sure_python_versions_are_specified
+parallel::make_sure_kubernetes_versions_are_specified
+
+get_maximum_parallel_k8s_jobs
+
+run_k8s_tests_in_parallel "${@}"
+
+# this will exit with error code in case some of the tests failed
+parallel::print_job_summary_and_return_status_code
diff --git a/scripts/ci/kubernetes/kind-cluster-conf.yaml b/scripts/ci/kubernetes/kind-cluster-conf.yaml
index f03c1b7..4e891f8 100644
--- a/scripts/ci/kubernetes/kind-cluster-conf.yaml
+++ b/scripts/ci/kubernetes/kind-cluster-conf.yaml
@@ -20,12 +20,12 @@ apiVersion: kind.x-k8s.io/v1alpha4
networking:
ipFamily: ipv4
apiServerAddress: "127.0.0.1"
- apiServerPort: 19090
+ apiServerPort: {{API_SERVER_PORT}}
nodes:
- role: control-plane
- role: worker
extraPortMappings:
- containerPort: 30007
- hostPort: 8080
+ hostPort: {{FORWARDED_PORT_NUMBER}}
listenAddress: "127.0.0.1"
protocol: TCP
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 771a06d..edf9e29 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -462,7 +462,6 @@ function build_images::get_docker_image_names() {
# Also enable experimental features of docker (we need `docker manifest` command)
function build_image::configure_docker_registry() {
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- start_end::group_start "Determine GitHub Registry token"
local token=""
if [[ "${GITHUB_REGISTRY}" == "ghcr.io" ]]; then
# For now ghcr.io can only authenticate using Personal Access Token with package access scope.
@@ -482,8 +481,6 @@ function build_image::configure_docker_registry() {
echo
exit 1
fi
- start_end::group_end
- start_end::group_start "Logging in to GitHub Registry"
if [[ -z "${token}" ]] ; then
verbosity::print_info
verbosity::print_info "Skip logging in to GitHub Registry. No Token available!"
@@ -497,14 +494,9 @@ function build_image::configure_docker_registry() {
else
verbosity::print_info "Skip Login to GitHub Registry ${GITHUB_REGISTRY} as token is missing"
fi
- start_end::group_end
-
- start_end::group_start "Make sure experimental docker features are enabled"
local new_config
new_config=$(jq '.experimental = "enabled"' "${HOME}/.docker/config.json")
echo "${new_config}" > "${HOME}/.docker/config.json"
- start_end::group_end
-
fi
}
@@ -861,9 +853,10 @@ function build_images::build_prod_images() {
build_images::print_build_info
if [[ ${SKIP_BUILDING_PROD_IMAGE} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Skip building production image. Assume the one we have is good!"
- verbosity::print_info
+ echo
+ echo "${COLOR_YELLOW}Skip building production image. Assume the one we have is good!${COLOR_RESET}"
+ echo "${COLOR_YELLOW}You must run './breeze build-image --production-image before for all python versions!${COLOR_RESET}"
+ echo
return
fi
@@ -978,12 +971,6 @@ function build_images::build_prod_images() {
fi
}
-function build_images::build_prod_images_with_group() {
- start_end::group_start "Build PROD images ${AIRFLOW_PROD_BUILD_IMAGE}"
- build_images::build_prod_images
- start_end::group_end
-}
-
# Waits for image tag to appear in GitHub Registry, pulls it and tags with the target tag
# Parameters:
# $1 - image name to wait for
@@ -1084,7 +1071,7 @@ function build_images::build_prod_images_from_locally_built_airflow_packages() {
build_airflow_packages::build_airflow_packages
mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
- build_images::build_prod_images_with_group
+ build_images::build_prod_images
}
# Useful information for people who stumble upon a pip check failure
diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh
index f5ed3e6..0433359 100644
--- a/scripts/ci/libraries/_docker_engine_resources.sh
+++ b/scripts/ci/libraries/_docker_engine_resources.sh
@@ -19,10 +19,16 @@
function docker_engine_resources::print_overall_stats() {
echo
- echo "Overall resource statistics"
+ echo "Docker statistics"
echo
docker stats --all --no-stream --no-trunc
- docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c "cat /proc/meminfo"
+ echo
+ echo "Memory statistics"
+ echo
+ docker run --rm --entrypoint /bin/sh "alpine:latest" -c "free -m"
+ echo
+ echo "Disk statistics"
+ echo
df -h || true
}
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index bbdf116..de9a2dd 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -510,14 +510,18 @@ function initialization::initialize_kubernetes_variables() {
# Kubectl version
export KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}}
# Local Kind path
- export KIND_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kind"
+ export KIND_BINARY_PATH="${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}/kind"
readonly KIND_BINARY_PATH
# Local Helm path
- export HELM_BINARY_PATH="${BUILD_CACHE_DIR}/bin/helm"
+ export HELM_BINARY_PATH="${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}/helm"
readonly HELM_BINARY_PATH
# local Kubectl path
- export KUBECTL_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kubectl"
+ export KUBECTL_BINARY_PATH="${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}/kubectl"
readonly KUBECTL_BINARY_PATH
+ FORWARDED_PORT_NUMBER="${FORWARDED_PORT_NUMBER:="8080"}"
+ readonly FORWARDED_PORT_NUMBER
+ API_SERVER_PORT="${API_SERVER_PORT:="19090"}"
+ readonly API_SERVER_PORT
}
function initialization::initialize_git_variables() {
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index a8deaac..bb883ec 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -21,26 +21,24 @@ function kind::get_kind_cluster_name() {
export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
# Name of the KinD cluster to connect to when referred to via kubectl
export KUBECTL_CLUSTER_NAME=kind-${KIND_CLUSTER_NAME}
- export KUBECONFIG="${BUILD_CACHE_DIR}/.kube/config"
- mkdir -pv "${BUILD_CACHE_DIR}/.kube/"
+ export KUBECONFIG="${BUILD_CACHE_DIR}/${KIND_CLUSTER_NAME}/.kube/config"
+ mkdir -pv "${BUILD_CACHE_DIR}/${KIND_CLUSTER_NAME}/.kube/"
touch "${KUBECONFIG}"
}
function kind::dump_kind_logs() {
- start_end::group_start "Dumping logs from KinD"
+ verbosity::print_info "Dumping logs from KinD"
local DUMP_DIR_NAME DUMP_DIR
DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}
DUMP_DIR="/tmp/${DUMP_DIR_NAME}"
kind --name "${KIND_CLUSTER_NAME}" export logs "${DUMP_DIR}"
- start_end::group_end
}
function kind::make_sure_kubernetes_tools_are_installed() {
- start_end::group_start "Make sure Kubernetes tools are installed"
SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${SYSTEM}-amd64"
- mkdir -pv "${BUILD_CACHE_DIR}/bin"
+ mkdir -pv "${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}"
if [[ -f "${KIND_BINARY_PATH}" ]]; then
DOWNLOADED_KIND_VERSION=v"$(${KIND_BINARY_PATH} --version | awk '{ print $3 }')"
echo "Currently downloaded kind version = ${DOWNLOADED_KIND_VERSION}"
@@ -87,15 +85,17 @@ function kind::make_sure_kubernetes_tools_are_installed() {
echo "Helm version ok"
echo
fi
- PATH=${PATH}:${BUILD_CACHE_DIR}/bin
- start_end::group_end
+ PATH=${PATH}:${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}
}
function kind::create_cluster() {
- kind create cluster \
- --name "${KIND_CLUSTER_NAME}" \
- --config "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/kind-cluster-conf.yaml" \
- --image "kindest/node:${KUBERNETES_VERSION}"
+ sed "s/{{FORWARDED_PORT_NUMBER}}/${FORWARDED_PORT_NUMBER}/" < \
+ "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/kind-cluster-conf.yaml" | \
+ sed "s/{{API_SERVER_PORT}}/${API_SERVER_PORT}/" | \
+ kind create cluster \
+ --name "${KIND_CLUSTER_NAME}" \
+ --config - \
+ --image "kindest/node:${KUBERNETES_VERSION}"
echo
echo "Created cluster ${KIND_CLUSTER_NAME}"
echo
@@ -106,7 +106,7 @@ function kind::delete_cluster() {
echo
echo "Deleted cluster ${KIND_CLUSTER_NAME}"
echo
- rm -rf "${HOME}/.kube/*"
+ rm -rf "${BUILD_CACHE_DIR}/${KIND_CLUSTER_NAME}/.kube/"
}
function kind::set_current_context() {
@@ -122,7 +122,6 @@ function kind::perform_kind_cluster_operation() {
echo
exit 1
fi
- start_end::group_start "Perform KinD cluster operation: ${1}"
set -u
OPERATION="${1}"
@@ -229,7 +228,6 @@ function kind::perform_kind_cluster_operation() {
exit 1
fi
fi
- start_end::group_end
}
function kind::check_cluster_ready_for_airflow() {
@@ -250,7 +248,6 @@ function kind::check_cluster_ready_for_airflow() {
}
function kind::build_image_for_kubernetes_tests() {
- start_end::group_start "Build image for kubernetes tests ${AIRFLOW_PROD_IMAGE_KUBERNETES}"
cd "${AIRFLOW_SOURCES}" || exit 1
docker_v build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF
FROM ${AIRFLOW_PROD_IMAGE}
@@ -261,13 +258,10 @@ COPY airflow/kubernetes_executor_templates/ \${AIRFLOW_HOME}/pod_templates/
EOF
echo "The ${AIRFLOW_PROD_IMAGE_KUBERNETES} is prepared for test kubernetes deployment."
- start_end::group_end
}
function kind::load_image_to_kind_cluster() {
- start_end::group_start "Loading ${AIRFLOW_PROD_IMAGE_KUBERNETES} to ${KIND_CLUSTER_NAME}"
kind load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE_KUBERNETES}"
- start_end::group_end
}
MAX_NUM_TRIES_FOR_HEALTH_CHECK=12
@@ -276,12 +270,7 @@ readonly MAX_NUM_TRIES_FOR_HEALTH_CHECK
SLEEP_TIME_FOR_HEALTH_CHECK=10
readonly SLEEP_TIME_FOR_HEALTH_CHECK
-FORWARDED_PORT_NUMBER=8080
-readonly FORWARDED_PORT_NUMBER
-
-
function kind::wait_for_webserver_healthy() {
- start_end::group_start "Waiting for webserver being healthy"
num_tries=0
set +e
sleep "${SLEEP_TIME_FOR_HEALTH_CHECK}"
@@ -300,14 +289,10 @@ function kind::wait_for_webserver_healthy() {
echo
echo "Connection to 'airflow webserver' established on port ${FORWARDED_PORT_NUMBER}"
echo
- initialization::ga_env CLUSTER_FORWARDED_PORT "${FORWARDED_PORT_NUMBER}"
- export CLUSTER_FORWARDED_PORT="${FORWARDED_PORT_NUMBER}"
set -e
- start_end::group_end
}
function kind::deploy_airflow_with_helm() {
- start_end::group_start "Deploying Airflow with Helm"
echo "Deleting namespace ${HELM_AIRFLOW_NAMESPACE}"
kubectl delete namespace "${HELM_AIRFLOW_NAMESPACE}" >/dev/null 2>&1 || true
kubectl delete namespace "test-namespace" >/dev/null 2>&1 || true
@@ -331,7 +316,14 @@ function kind::deploy_airflow_with_helm() {
kubectl -n "${HELM_AIRFLOW_NAMESPACE}" patch serviceaccount default -p '{"imagePullSecrets": [{"name": "regcred"}]}'
fi
- pushd "${AIRFLOW_SOURCES}/chart" >/dev/null 2>&1 || exit 1
+ local chartdir
+ chartdir=$(mktemp -d)
+ traps::add_trap "rm -rf ${chartdir}" EXIT INT HUP TERM
+ # Copy chart to temporary directory to allow chart deployment in parallel
+ # Otherwise helm deployment will fail on renaming charts to tmpcharts
+ cp -r "${AIRFLOW_SOURCES}/chart" "${chartdir}"
+
+ pushd "${chartdir}/chart" >/dev/null 2>&1 || exit 1
helm repo add stable https://charts.helm.sh/stable/
helm dep update
helm install airflow . --namespace "${HELM_AIRFLOW_NAMESPACE}" \
@@ -343,15 +335,10 @@ function kind::deploy_airflow_with_helm() {
--set "config.api.enable_experimental_api=true"
echo
popd > /dev/null 2>&1|| exit 1
- start_end::group_end
}
function kind::deploy_test_kubernetes_resources() {
- start_end::group_start "Deploying Airflow with Helm"
- echo
- echo "Deploying Custom kubernetes resources"
- echo
+ verbosity::print_info "Deploying Custom kubernetes resources"
kubectl apply -f "scripts/ci/kubernetes/volumes.yaml" --namespace default
kubectl apply -f "scripts/ci/kubernetes/nodeport.yaml" --namespace airflow
- start_end::group_end
}
diff --git a/scripts/ci/libraries/_parallel.sh b/scripts/ci/libraries/_parallel.sh
index f38d06f..03e910c 100644
--- a/scripts/ci/libraries/_parallel.sh
+++ b/scripts/ci/libraries/_parallel.sh
@@ -194,7 +194,6 @@ function parallel::cleanup_runner() {
start_end::group_end
}
-
function parallel::make_sure_python_versions_are_specified() {
if [[ -z "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING=}" ]]; then
echo
@@ -203,6 +202,18 @@ function parallel::make_sure_python_versions_are_specified() {
exit 1
fi
echo
- echo "${COLOR_BLUE}Running parallel builds for those Python versions: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}!${COLOR_RESET}"
+ echo "${COLOR_BLUE}Running parallel builds for those Python versions: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}${COLOR_RESET}"
+ echo
+}
+
+function parallel::make_sure_kubernetes_versions_are_specified() {
+ if [[ -z "${CURRENT_KUBERNETES_VERSIONS_AS_STRING=}" ]]; then
+ echo
+ echo "${COLOR_RED}The CURRENT_KUBERNETES_VERSIONS_AS_STRING variable must be set and list K8S versions to use!${COLOR_RESET}"
+ echo
+ exit 1
+ fi
+ echo
+ echo "${COLOR_BLUE}Running parallel builds for those Kubernetes versions: ${CURRENT_KUBERNETES_VERSIONS_AS_STRING}${COLOR_RESET}"
echo
}
diff --git a/scripts/ci/libraries/_testing.sh b/scripts/ci/libraries/_testing.sh
index 28d1fc6..638daf5 100644
--- a/scripts/ci/libraries/_testing.sh
+++ b/scripts/ci/libraries/_testing.sh
@@ -52,7 +52,7 @@ function testing::get_docker_compose_local() {
function testing::get_maximum_parallel_test_jobs() {
docker_engine_resources::get_available_cpus_in_docker
- if [[ ${RUNS_ON} != *"self-hosted"* ]]; then
+ if [[ -n ${RUNS_ON=} && ${RUNS_ON} != *"self-hosted"* ]]; then
echo
echo "${COLOR_YELLOW}This is a Github Public runner - for now we are forcing max parallel Quarantined tests jobs to 1 for those${COLOR_RESET}"
echo
diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh
index dc3ca5a..68b356d 100644
--- a/scripts/ci/libraries/_verbosity.sh
+++ b/scripts/ci/libraries/_verbosity.sh
@@ -40,7 +40,7 @@ function verbosity::restore_exit_on_error_status() {
# printed before execution. In case of DRY_RUN_DOCKER flag set to "true"
# show the command to execute instead of executing them
function docker_v {
- if [[ ${DRY_RUN_DOCKER} != "false" ]]; then
+ if [[ ${DRY_RUN_DOCKER=} != "false" ]]; then
echo
echo "${COLOR_CYAN}docker" "${@}" "${COLOR_RESET}"
echo
diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh
index 5712ee8..8f9c74d 100755
--- a/scripts/ci/selective_ci_checks.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -61,6 +61,7 @@ function output_all_basic_variables() {
initialization::ga_output all-python-versions \
"$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
initialization::ga_output python-versions-list-as-string "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[*]}"
+ initialization::ga_output kubernetes-versions-list-as-string "${CURRENT_KUBERNETES_VERSIONS[*]}"
else
initialization::ga_output python-versions \
"$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")"
@@ -69,6 +70,7 @@ function output_all_basic_variables() {
initialization::ga_output all-python-versions \
"$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")"
initialization::ga_output python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
+ initialization::ga_output kubernetes-versions-list-as-string "${DEFAULT_KUBERNETES_VERSION}"
fi
initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
diff --git a/scripts/ci/testing/ci_run_quarantined_tests.sh b/scripts/ci/testing/ci_run_quarantined_tests.sh
index 0c1108e..57e4aca 100755
--- a/scripts/ci/testing/ci_run_quarantined_tests.sh
+++ b/scripts/ci/testing/ci_run_quarantined_tests.sh
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -euo pipefail
# Enable automated tests execution
RUN_TESTS="true"
@@ -29,6 +30,8 @@ export SEMAPHORE_NAME
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+initialization::set_output_color_variables
+
BACKEND_TEST_TYPES=(mysql postgres sqlite)
# Starts test types in parallel
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index 5cb8d99..16aabbb 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -210,7 +210,7 @@ if [[ "${RUN_TESTS}" != "true" ]]; then
fi
set -u
-export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}.xml"
+export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}-${BACKEND}.xml"
EXTRA_PYTEST_ARGS=(
"--verbosity=0"
@@ -218,7 +218,7 @@ EXTRA_PYTEST_ARGS=(
"--durations=100"
"--cov=airflow/"
"--cov-config=.coveragerc"
- "--cov-report=xml:/files/coverage.xml"
+ "--cov-report=xml:/files/coverage-${TEST_TYPE}-${BACKEND}.xml"
"--color=yes"
"--maxfail=50"
"--pythonwarnings=ignore::DeprecationWarning"