You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/04/05 21:48:04 UTC
[airflow] 05/16: Better handling of docker command (#15080)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f840d1674d3cc3634764fe2ee911332d9872df3a
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Apr 2 07:31:00 2021 +0200
Better handling of docker command (#15080)
Not all docker commands are replaced with functions now.
Earlier wer replaced all docker commands with a function to be able
to capture docker commands used and display it with -v for breeze.
This has proven to be harmful as
this is an unexpected behaviour for a docker command.
This change introduces docker_v command which outputs the command
when needed.
(cherry picked from commit 535e1a8e692ba28ad8ce9474a66b941af1df4875)
---
breeze | 2 +-
.../enter_breeze_provider_package_tests.sh | 2 +-
scripts/ci/libraries/_build_images.sh | 42 +++++++++++-----------
scripts/ci/libraries/_docker_engine_resources.sh | 12 +++----
scripts/ci/libraries/_initialization.sh | 2 ++
scripts/ci/libraries/_kind.sh | 2 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 28 +++++++--------
scripts/ci/libraries/_runs.sh | 10 +++---
scripts/ci/libraries/_start_end.sh | 2 +-
scripts/ci/libraries/_verbosity.sh | 6 ++--
scripts/ci/libraries/_verify_image.sh | 8 ++---
.../ci_install_and_test_provider_packages.sh | 2 +-
scripts/ci/static_checks/bats_tests.sh | 2 +-
scripts/ci/static_checks/check_license.sh | 2 +-
scripts/ci/static_checks/flake8.sh | 4 +--
.../ci/static_checks/in_container_bats_tests.sh | 4 +--
scripts/ci/static_checks/lint_dockerfile.sh | 4 +--
scripts/ci/static_checks/mypy.sh | 2 +-
scripts/ci/static_checks/pylint.sh | 4 +--
scripts/ci/static_checks/refresh_pylint_todo.sh | 2 +-
scripts/ci/testing/ci_run_airflow_testing.sh | 2 +-
scripts/ci/tools/ci_clear_tmp.sh | 2 +-
scripts/ci/tools/ci_fix_ownership.sh | 2 +-
scripts/ci/tools/ci_free_space_on_ci.sh | 2 +-
24 files changed, 75 insertions(+), 75 deletions(-)
diff --git a/breeze b/breeze
index 4df4937..c85a5ac 100755
--- a/breeze
+++ b/breeze
@@ -609,7 +609,7 @@ if [[ \${VERBOSE} == "true" ]]; then
echo
echo "Executing script:"
echo
- echo "${file} \${@}"
+ echo "${COLOR_CYAN}${file} \${@}${COLOR_RESET}"
echo
set -x
fi
diff --git a/dev/provider_packages/enter_breeze_provider_package_tests.sh b/dev/provider_packages/enter_breeze_provider_package_tests.sh
index 0e9467f..b67b33f 100755
--- a/dev/provider_packages/enter_breeze_provider_package_tests.sh
+++ b/dev/provider_packages/enter_breeze_provider_package_tests.sh
@@ -21,7 +21,7 @@ export MOUNT_SELECTED_LOCAL_SOURCES="false"
. "$(dirname "${BASH_SOURCE[0]}")/../../scripts/ci/libraries/_script_init.sh"
function enter_breeze_with_mapped_sources() {
- docker run -it "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run -it "${EXTRA_DOCKER_FLAGS[@]}" \
-v "${AIRFLOW_SOURCES}/setup.py:/airflow_sources/setup.py:cached" \
-v "${AIRFLOW_SOURCES}/setup.cfg:/airflow_sources/setup.cfg:cached" \
-v "${AIRFLOW_SOURCES}/airflow/__init__.py:/airflow_sources/airflow/__init__.py:cached" \
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 7d7d180..771a06d 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -89,7 +89,7 @@ function build_images::add_build_args_for_remote_install() {
# Retrieves version of airflow stored in the production image (used to display the actual
# Version we use if it was build from PyPI or GitHub
function build_images::get_airflow_version_from_production_image() {
- VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
+ docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
}
# Removes the "Forced answer" (yes/no/quit) given previously, unless you specifically want to remember it.
@@ -252,7 +252,7 @@ function build_images::confirm_non-empty-docker-context-files() {
# We cannot use docker registry APIs as they are available only with authorisation
# But this image can be pulled without authentication
function build_images::build_ci_image_manifest() {
- docker build \
+ docker_v build \
--tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \
-f- . <<EOF
FROM scratch
@@ -270,8 +270,8 @@ function build_images::get_local_build_cache_hash() {
set +e
# Remove the container just in case
- docker rm --force "local-airflow-ci-container" 2>/dev/null >/dev/null
- if ! docker inspect "${AIRFLOW_CI_IMAGE}" 2>/dev/null >/dev/null; then
+ docker_v rm --force "local-airflow-ci-container" 2>/dev/null >/dev/null
+ if ! docker_v inspect "${AIRFLOW_CI_IMAGE}" 2>/dev/null >/dev/null; then
verbosity::print_info
verbosity::print_info "Local airflow CI image not available"
verbosity::print_info
@@ -281,8 +281,8 @@ function build_images::get_local_build_cache_hash() {
return
fi
- docker create --name "local-airflow-ci-container" "${AIRFLOW_CI_IMAGE}" 2>/dev/null
- docker cp "local-airflow-ci-container:/build-cache-hash" \
+ docker_v create --name "local-airflow-ci-container" "${AIRFLOW_CI_IMAGE}" 2>/dev/null
+ docker_v cp "local-airflow-ci-container:/build-cache-hash" \
"${LOCAL_IMAGE_BUILD_CACHE_HASH_FILE}" 2>/dev/null ||
touch "${LOCAL_IMAGE_BUILD_CACHE_HASH_FILE}"
set -e
@@ -305,7 +305,7 @@ function build_images::get_local_build_cache_hash() {
function build_images::get_remote_image_build_cache_hash() {
set +e
# Pull remote manifest image
- if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" 2>/dev/null >/dev/null; then
+ if ! docker_v pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}" 2>/dev/null >/dev/null; then
verbosity::print_info
verbosity::print_info "Remote docker registry unreachable"
verbosity::print_info
@@ -317,11 +317,11 @@ function build_images::get_remote_image_build_cache_hash() {
set -e
rm -f "${REMOTE_IMAGE_CONTAINER_ID_FILE}"
# Create container dump out of the manifest image without actually running it
- docker create --cidfile "${REMOTE_IMAGE_CONTAINER_ID_FILE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
+ docker_v create --cidfile "${REMOTE_IMAGE_CONTAINER_ID_FILE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
# Extract manifest and store it in local file
- docker cp "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}"):/build-cache-hash" \
+ docker_v cp "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}"):/build-cache-hash" \
"${REMOTE_IMAGE_BUILD_CACHE_HASH_FILE}"
- docker rm --force "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}")"
+ docker_v rm --force "$(cat "${REMOTE_IMAGE_CONTAINER_ID_FILE}")"
rm -f "${REMOTE_IMAGE_CONTAINER_ID_FILE}"
verbosity::print_info
verbosity::print_info "Remote build cache hash: '$(cat "${REMOTE_IMAGE_BUILD_CACHE_HASH_FILE}")'"
@@ -490,7 +490,7 @@ function build_image::configure_docker_registry() {
verbosity::print_info
fi
if [[ -n "${token}" ]]; then
- echo "${token}" | docker login \
+ echo "${token}" | docker_v login \
--username "${GITHUB_USERNAME:-apache}" \
--password-stdin \
"${GITHUB_REGISTRY}"
@@ -745,7 +745,7 @@ Docker building ${AIRFLOW_CI_IMAGE}.
if [[ -n "${RUNTIME_APT_COMMAND}" ]]; then
additional_runtime_args+=("--build-arg" "RUNTIME_APT_COMMAND=\"${RUNTIME_APT_COMMAND}\"")
fi
- docker build \
+ docker_v build \
"${EXTRA_DOCKER_CI_BUILD_FLAGS[@]}" \
--build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
@@ -781,11 +781,11 @@ Docker building ${AIRFLOW_CI_IMAGE}.
set -u
if [[ -n "${DEFAULT_CI_IMAGE=}" ]]; then
echo "Tagging additionally image ${AIRFLOW_CI_IMAGE} with ${DEFAULT_CI_IMAGE}"
- docker tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}"
+ docker_v tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}"
fi
if [[ -n "${IMAGE_TAG=}" ]]; then
echo "Tagging additionally image ${AIRFLOW_CI_IMAGE} with ${IMAGE_TAG}"
- docker tag "${AIRFLOW_CI_IMAGE}" "${IMAGE_TAG}"
+ docker_v tag "${AIRFLOW_CI_IMAGE}" "${IMAGE_TAG}"
fi
if [[ -n ${SPIN_PID=} ]]; then
kill -HUP "${SPIN_PID}" || true
@@ -898,7 +898,7 @@ function build_images::build_prod_images() {
if [[ -n "${DEV_APT_COMMAND}" ]]; then
additional_dev_args+=("--build-arg" "DEV_APT_COMMAND=\"${DEV_APT_COMMAND}\"")
fi
- docker build \
+ docker_v build \
"${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
--build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \
--build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
@@ -934,7 +934,7 @@ function build_images::build_prod_images() {
if [[ -n "${RUNTIME_APT_COMMAND}" ]]; then
additional_runtime_args+=("--build-arg" "RUNTIME_APT_COMMAND=\"${RUNTIME_APT_COMMAND}\"")
fi
- docker build \
+ docker_v build \
"${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
--build-arg PYTHON_BASE_IMAGE="${AIRFLOW_PYTHON_BASE_IMAGE}" \
--build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \
@@ -970,11 +970,11 @@ function build_images::build_prod_images() {
set -u
if [[ -n "${DEFAULT_PROD_IMAGE:=}" ]]; then
echo "Tagging additionally image ${AIRFLOW_PROD_IMAGE} with ${DEFAULT_PROD_IMAGE}"
- docker tag "${AIRFLOW_PROD_IMAGE}" "${DEFAULT_PROD_IMAGE}"
+ docker_v tag "${AIRFLOW_PROD_IMAGE}" "${DEFAULT_PROD_IMAGE}"
fi
if [[ -n "${IMAGE_TAG=}" ]]; then
echo "Tagging additionally image ${AIRFLOW_PROD_IMAGE} with ${IMAGE_TAG}"
- docker tag "${AIRFLOW_PROD_IMAGE}" "${IMAGE_TAG}"
+ docker_v tag "${AIRFLOW_PROD_IMAGE}" "${IMAGE_TAG}"
fi
}
@@ -1000,7 +1000,7 @@ function build_images::wait_for_image_tag() {
while true; do
set +e
echo "${COLOR_BLUE}Docker pull ${IMAGE_TO_WAIT_FOR} ${COLOR_RESET}" >"${OUTPUT_LOG}"
- docker pull "${IMAGE_TO_WAIT_FOR}" >>"${OUTPUT_LOG}" 2>&1
+ docker_v pull "${IMAGE_TO_WAIT_FOR}" >>"${OUTPUT_LOG}" 2>&1
set -e
local image_hash
echo "${COLOR_BLUE} Docker images -q ${IMAGE_TO_WAIT_FOR}${COLOR_RESET}" >>"${OUTPUT_LOG}"
@@ -1020,12 +1020,12 @@ function build_images::wait_for_image_tag() {
echo
echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${IMAGE_NAME}."
echo
- docker tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}"
+ docker_v tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}"
for TARGET_TAG in "${@}"; do
echo
echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${TARGET_TAG}."
echo
- docker tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}"
+ docker_v tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}"
done
break
fi
diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh
index b5283b3..f5ed3e6 100644
--- a/scripts/ci/libraries/_docker_engine_resources.sh
+++ b/scripts/ci/libraries/_docker_engine_resources.sh
@@ -22,27 +22,25 @@ function docker_engine_resources::print_overall_stats() {
echo "Overall resource statistics"
echo
docker stats --all --no-stream --no-trunc
- docker run --rm --entrypoint /bin/bash "${AIRFLOW_CI_IMAGE}" -c "free -h"
- df --human || true
+ docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c "cat /proc/meminfo"
+ df -h || true
}
function docker_engine_resources::get_available_memory_in_docker() {
- MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash debian:buster-slim -c \
- 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))')
+ MEMORY_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c 'echo $(($(getconf _PHYS_PAGES) * $(getconf PAGE_SIZE) / (1024 * 1024)))')
echo "${COLOR_BLUE}Memory available for Docker${COLOR_RESET}: $(numfmt --to iec $((MEMORY_AVAILABLE_FOR_DOCKER * 1024 * 1024)))"
export MEMORY_AVAILABLE_FOR_DOCKER
}
function docker_engine_resources::get_available_cpus_in_docker() {
- CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash debian:buster-slim -c \
- 'grep -cE "cpu[0-9]+" </proc/stat')
+ CPUS_AVAILABLE_FOR_DOCKER=$(docker run --rm "debian:buster-slim" grep -cE 'cpu[0-9]+' /proc/stat)
echo "${COLOR_BLUE}CPUS available for Docker${COLOR_RESET}: ${CPUS_AVAILABLE_FOR_DOCKER}"
export CPUS_AVAILABLE_FOR_DOCKER
}
function docker_engine_resources::get_available_disk_space_in_docker() {
- DISK_SPACE_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash debian:buster-slim -c \
+ DISK_SPACE_AVAILABLE_FOR_DOCKER=$(docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c \
'df / | tail -1 | awk '\''{print $4}'\')
echo "${COLOR_BLUE}Disk space available for Docker${COLOR_RESET}: $(numfmt --to iec $((DISK_SPACE_AVAILABLE_FOR_DOCKER * 1024)))"
export DISK_SPACE_AVAILABLE_FOR_DOCKER
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 93dabea..cb42693 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -566,11 +566,13 @@ function initialization::set_output_color_variables() {
COLOR_RED=$'\e[31m'
COLOR_RESET=$'\e[0m'
COLOR_YELLOW=$'\e[33m'
+ COLOR_CYAN=$'\e[36m'
export COLOR_BLUE
export COLOR_GREEN
export COLOR_RED
export COLOR_RESET
export COLOR_YELLOW
+ export COLOR_CYAN
}
# Common environment that is initialized by both Breeze and CI scripts
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index 4fbfee1..a8deaac 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -252,7 +252,7 @@ function kind::check_cluster_ready_for_airflow() {
function kind::build_image_for_kubernetes_tests() {
start_end::group_start "Build image for kubernetes tests ${AIRFLOW_PROD_IMAGE_KUBERNETES}"
cd "${AIRFLOW_SOURCES}" || exit 1
- docker build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF
+ docker_v build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF
FROM ${AIRFLOW_PROD_IMAGE}
COPY airflow/example_dags/ \${AIRFLOW_HOME}/dags/
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 3624a9a..b0723e4 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -25,7 +25,7 @@ function push_pull_remove_images::push_image_with_retries() {
set +e
echo
echo "Trying to push the image ${1}. Number of try: ${try_num}"
- docker push "${1}"
+ docker_v push "${1}"
local res=$?
set -e
if [[ ${res} != "0" ]]; then
@@ -61,7 +61,7 @@ function push_pull_remove_images::pull_image_if_not_present_or_forced() {
echo
echo "Pulling the image ${IMAGE_TO_PULL}"
echo
- docker pull "${IMAGE_TO_PULL}"
+ docker_v pull "${IMAGE_TO_PULL}"
EXIT_VALUE="$?"
if [[ ${EXIT_VALUE} != "0" && ${FAIL_ON_GITHUB_DOCKER_PULL_ERROR} == "true" ]]; then
echo
@@ -97,7 +97,7 @@ function push_pull_remove_images::pull_image_github_dockerhub() {
set +e
if push_pull_remove_images::pull_image_if_not_present_or_forced "${GITHUB_IMAGE}"; then
# Tag the image to be the DockerHub one
- docker tag "${GITHUB_IMAGE}" "${DOCKERHUB_IMAGE}"
+ docker_v tag "${GITHUB_IMAGE}" "${DOCKERHUB_IMAGE}"
else
push_pull_remove_images::pull_image_if_not_present_or_forced "${DOCKERHUB_IMAGE}"
fi
@@ -109,9 +109,9 @@ function push_pull_remove_images::rebuild_python_base_image() {
echo
echo "Rebuilding ${AIRFLOW_PYTHON_BASE_IMAGE} from latest ${PYTHON_BASE_IMAGE}"
echo
- docker pull "${PYTHON_BASE_IMAGE}"
+ docker_v pull "${PYTHON_BASE_IMAGE}"
echo "FROM ${PYTHON_BASE_IMAGE}" | \
- docker build \
+ docker_v build \
--label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
-t "${AIRFLOW_PYTHON_BASE_IMAGE}" -
}
@@ -144,7 +144,7 @@ function push_pull_remove_images::pull_base_python_image() {
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PYTHON_BASE_IMAGE}" \
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
else
- docker pull "${AIRFLOW_PYTHON_BASE_IMAGE}"
+ docker_v pull "${AIRFLOW_PYTHON_BASE_IMAGE}"
fi
}
@@ -194,7 +194,7 @@ function push_pull_remove_images::pull_prod_images_if_needed() {
function push_pull_remove_images::push_ci_images_to_dockerhub() {
push_pull_remove_images::push_image_with_retries "${AIRFLOW_PYTHON_BASE_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_IMAGE}"
- docker tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
+ docker_v tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
if [[ -n ${DEFAULT_CI_IMAGE=} ]]; then
# Only push default image to DockerHub registry if it is defined
@@ -214,7 +214,7 @@ function push_pull_remove_images::push_python_image_to_github() {
if [[ ${GITHUB_REGISTRY_PUSH_IMAGE_TAG} != "latest" ]]; then
PYTHON_TAG_SUFFIX="-${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
fi
- docker tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \
+ docker_v tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
push_pull_remove_images::push_image_with_retries \
"${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
@@ -224,12 +224,12 @@ function push_pull_remove_images::push_python_image_to_github() {
function push_pull_remove_images::push_ci_images_to_github() {
push_pull_remove_images::push_python_image_to_github
AIRFLOW_CI_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
- docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}"
+ docker_v tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_TAGGED_IMAGE}"
if [[ -n ${GITHUB_SHA=} ]]; then
# Also push image to GitHub registry with commit SHA
AIRFLOW_CI_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${COMMIT_SHA}"
- docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_SHA_IMAGE}"
+ docker_v tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_SHA_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_SHA_IMAGE}"
fi
}
@@ -265,17 +265,17 @@ function push_pull_remove_images::push_prod_images_to_dockerhub () {
function push_pull_remove_images::push_prod_images_to_github () {
push_pull_remove_images::push_python_image_to_github
AIRFLOW_PROD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
- docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}"
+ docker_v tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}"
push_pull_remove_images::push_image_with_retries "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
if [[ -n ${COMMIT_SHA=} ]]; then
# Also push image to GitHub registry with commit SHA
AIRFLOW_PROD_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}"
- docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_SHA_IMAGE}"
+ docker_v tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_SHA_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_PROD_SHA_IMAGE}"
fi
# Also push prod build image
AIRFLOW_PROD_BUILD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
- docker tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}"
+ docker_v tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_PROD_BUILD_TAGGED_IMAGE}"
}
@@ -327,7 +327,7 @@ function push_pull_remove_images::check_for_image_in_github_container_registry()
local image_to_wait_for="ghcr.io/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}"
echo "GitHub Container Registry: checking for ${image_to_wait_for} via docker manifest inspect!"
- docker manifest inspect "${image_to_wait_for}"
+ docker_v manifest inspect "${image_to_wait_for}"
local res=$?
if [[ ${res} == "0" ]]; then
echo "Image: ${image_to_wait_for} found in Container Registry: ${COLOR_GREEN}OK.${COLOR_RESET}"
diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh
index bd80108..45c4c2b 100644
--- a/scripts/ci/libraries/_runs.sh
+++ b/scripts/ci/libraries/_runs.sh
@@ -19,7 +19,7 @@
# Docker command to build documentation
function runs::run_docs() {
start_end::group_start "Run build docs"
- docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" -t \
-e "GITHUB_ACTIONS=${GITHUB_ACTIONS="false"}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
@@ -30,7 +30,7 @@ function runs::run_docs() {
# Docker command to generate constraint files.
function runs::run_generate_constraints() {
start_end::group_start "Run generate constraints"
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/in_container/run_generate_constraints.sh"
@@ -40,7 +40,7 @@ function runs::run_generate_constraints() {
# Docker command to prepare airflow packages
function runs::run_prepare_airflow_packages() {
start_end::group_start "Run prepare airflow packages"
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
-t \
-v "${AIRFLOW_SOURCES}:/opt/airflow" \
@@ -53,7 +53,7 @@ function runs::run_prepare_airflow_packages() {
# Docker command to prepare provider packages
function runs::run_prepare_provider_packages() {
# No group here - groups are added internally
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
-t \
-v "${AIRFLOW_SOURCES}:/opt/airflow" \
@@ -64,7 +64,7 @@ function runs::run_prepare_provider_packages() {
# Docker command to generate release notes for provider packages
function runs::run_prepare_provider_documentation() {
# No group here - groups are added internally
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
-t \
-v "${AIRFLOW_SOURCES}:/opt/airflow" \
diff --git a/scripts/ci/libraries/_start_end.sh b/scripts/ci/libraries/_start_end.sh
index b7fa74f..d2d6a61 100644
--- a/scripts/ci/libraries/_start_end.sh
+++ b/scripts/ci/libraries/_start_end.sh
@@ -76,7 +76,7 @@ function start_end::dump_container_logs() {
echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}"
echo " Dumping logs from ${container} container"
echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}"
- docker logs "${container}" > "${dump_file}"
+ docker_v logs "${container}" > "${dump_file}"
echo " Container ${container} logs dumped to ${dump_file}"
echo "${COLOR_BLUE}###########################################################################################${COLOR_RESET}"
start_end::group_end
diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh
index 26a077d..dc3ca5a 100644
--- a/scripts/ci/libraries/_verbosity.sh
+++ b/scripts/ci/libraries/_verbosity.sh
@@ -39,10 +39,10 @@ function verbosity::restore_exit_on_error_status() {
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be
# printed before execution. In case of DRY_RUN_DOCKER flag set to "true"
# show the command to execute instead of executing them
-function docker {
+function docker_v {
if [[ ${DRY_RUN_DOCKER} != "false" ]]; then
echo
- echo "${COLOR_YELLOW}docker" "${@}" "${COLOR_RESET}"
+ echo "${COLOR_CYAN}docker" "${@}" "${COLOR_RESET}"
echo
return
fi
@@ -52,7 +52,7 @@ function docker {
${VERBOSE_COMMANDS:=} != "true" && \
# And when generally printing info is disabled
${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then
- >&2 echo "docker" "${@}"
+ >&2 echo "${COLOR_CYAN}docker ${*} ${COLOR_RESET}"
fi
if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then
${DOCKER_BINARY_PATH} "${@}" >>"${OUTPUT_LOG}" 2>&1
diff --git a/scripts/ci/libraries/_verify_image.sh b/scripts/ci/libraries/_verify_image.sh
index f9d2bb6..b0060ac 100644
--- a/scripts/ci/libraries/_verify_image.sh
+++ b/scripts/ci/libraries/_verify_image.sh
@@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.
function verify_image::run_command_in_image() {
- docker run --rm \
+ docker_v run --rm \
-e COLUMNS=180 \
--entrypoint /bin/bash "${DOCKER_IMAGE}" \
-c "${@}"
@@ -83,7 +83,7 @@ function verify_image::verify_prod_image_has_airflow_and_providers() {
function verify_image::verify_ci_image_dependencies() {
start_end::group_start "Checking if Airflow dependencies are non-conflicting in ${DOCKER_IMAGE} image."
set +e
- docker run --rm --entrypoint /bin/bash "${DOCKER_IMAGE}" -c 'pip check'
+ docker_v run --rm --entrypoint /bin/bash "${DOCKER_IMAGE}" -c 'pip check'
local res=$?
if [[ ${res} != "0" ]]; then
echo "${COLOR_RED}ERROR: ^^^ Some dependencies are conflicting. See instructions below on how to deal with it. ${COLOR_RESET}"
@@ -212,7 +212,7 @@ function verify_image::verify_prod_image_as_root() {
echo "Checking airflow as root"
local output
local res
- output=$(docker run --rm --user 0 "${DOCKER_IMAGE}" "airflow" "info" 2>&1)
+ output=$(docker_v run --rm --user 0 "${DOCKER_IMAGE}" "airflow" "info" 2>&1)
res=$?
if [[ ${res} == "0" ]]; then
echo "${COLOR_GREEN}OK${COLOR_RESET}"
@@ -229,7 +229,7 @@ function verify_image::verify_prod_image_as_root() {
tmp_dir="$(mktemp -d)"
touch "${tmp_dir}/__init__.py"
echo 'print("Awesome")' >> "${tmp_dir}/awesome.py"
- output=$(docker run \
+ output=$(docker_v run \
--rm \
-e "PYTHONPATH=${tmp_dir}" \
-v "${tmp_dir}:${tmp_dir}" \
diff --git a/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh b/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
index 5b29c7e..51575eb 100755
--- a/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+++ b/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
@@ -29,7 +29,7 @@ fi
function run_test_package_import_all_classes() {
# Groups are added internally
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
-t \
-v "${AIRFLOW_SOURCES}/setup.py:/airflow_sources/setup.py:cached" \
diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/bats_tests.sh
index e54e5b1..eaf9171 100755
--- a/scripts/ci/static_checks/bats_tests.sh
+++ b/scripts/ci/static_checks/bats_tests.sh
@@ -53,7 +53,7 @@ function run_bats_tests() {
# deduplicate
FS=" " read -r -a bats_arguments <<< "$(tr ' ' '\n' <<< "${bats_arguments[@]}" | sort -u | tr '\n' ' ' )"
if [[ ${#@} == "0" ]]; then
- # Run all tests
+ # Run al tests
docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
apache/airflow:bats-2020.09.05-1.2.1 --tap /airflow/tests/bats/
elif [[ ${#bats_arguments} == "0" ]]; then
diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh
index 8698bc9..d3a8be7 100755
--- a/scripts/ci/static_checks/check_license.sh
+++ b/scripts/ci/static_checks/check_license.sh
@@ -31,7 +31,7 @@ function run_check_license() {
echo "Running license checks. This can take a while."
# We mount ALL airflow files for the licence check. We want to check them all!
- if ! docker run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \
+ if ! docker_v run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \
--user "$(id -ur):$(id -gr)" \
--rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
apache/airflow:apache-rat-2020.07.10-0.13 \
diff --git a/scripts/ci/static_checks/flake8.sh b/scripts/ci/static_checks/flake8.sh
index 322ab9e..1c5440c 100755
--- a/scripts/ci/static_checks/flake8.sh
+++ b/scripts/ci/static_checks/flake8.sh
@@ -20,12 +20,12 @@
function run_flake8() {
if [[ "${#@}" == "0" ]]; then
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/in_container/run_flake8.sh"
else
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/in_container/run_flake8.sh" "${@}"
diff --git a/scripts/ci/static_checks/in_container_bats_tests.sh b/scripts/ci/static_checks/in_container_bats_tests.sh
index a7c0121..fa4eacd 100644
--- a/scripts/ci/static_checks/in_container_bats_tests.sh
+++ b/scripts/ci/static_checks/in_container_bats_tests.sh
@@ -20,13 +20,13 @@
function run_in_container_bats_tests() {
if [[ "${#@}" == "0" ]]; then
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/opt/bats/bin/bats" \
"-v" "$(pwd):/airflow" \
"${AIRFLOW_CI_IMAGE}" \
--tap "tests/bats/in_container/"
else
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/opt/bats/bin/bats" \
"-v" "$(pwd):/airflow" \
"${AIRFLOW_CI_IMAGE}" \
diff --git a/scripts/ci/static_checks/lint_dockerfile.sh b/scripts/ci/static_checks/lint_dockerfile.sh
index 491662d..38327f7 100755
--- a/scripts/ci/static_checks/lint_dockerfile.sh
+++ b/scripts/ci/static_checks/lint_dockerfile.sh
@@ -25,7 +25,7 @@ function run_docker_lint() {
echo "Running docker lint for all Dockerfiles"
echo
# shellcheck disable=SC2046
- docker run \
+ docker_v run \
-v "$(pwd):/root" \
-w "/root" \
--rm \
@@ -37,7 +37,7 @@ function run_docker_lint() {
echo
echo "Running docker lint for $*"
echo
- docker run \
+ docker_v run \
-v "$(pwd):/root" \
-w "/root" \
--rm \
diff --git a/scripts/ci/static_checks/mypy.sh b/scripts/ci/static_checks/mypy.sh
index a7257a9..7ebbd63 100755
--- a/scripts/ci/static_checks/mypy.sh
+++ b/scripts/ci/static_checks/mypy.sh
@@ -26,7 +26,7 @@ function run_mypy() {
files=("$@")
fi
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \
"${AIRFLOW_CI_IMAGE}" \
diff --git a/scripts/ci/static_checks/pylint.sh b/scripts/ci/static_checks/pylint.sh
index edadd94..c69498e 100755
--- a/scripts/ci/static_checks/pylint.sh
+++ b/scripts/ci/static_checks/pylint.sh
@@ -20,12 +20,12 @@
function run_pylint() {
if [[ "${#@}" == "0" ]]; then
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/in_container/run_pylint.sh"
else
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/in_container/run_pylint.sh" "${@}"
diff --git a/scripts/ci/static_checks/refresh_pylint_todo.sh b/scripts/ci/static_checks/refresh_pylint_todo.sh
index 05dce88..52474b7 100755
--- a/scripts/ci/static_checks/refresh_pylint_todo.sh
+++ b/scripts/ci/static_checks/refresh_pylint_todo.sh
@@ -21,7 +21,7 @@ export FORCE_ANSWER_TO_QUESTIONS="quit"
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function refresh_pylint_todo() {
- docker run "${EXTRA_DOCKER_FLAGS[@]}" \
+ docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
"${AIRFLOW_CI_IMAGE}" \
"/opt/airflow/scripts/in_container/refresh_pylint_todo.sh"
}
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index 0867e3c..af147ad 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -110,7 +110,7 @@ function system_prune_docker() {
echo
echo "${COLOR_BLUE}System-prune docker${COLOR_RESET}"
echo
- docker system prune --force --volumes
+ docker_v system prune --force --volumes
echo
}
diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh
index d367967..bef3fa5 100755
--- a/scripts/ci/tools/ci_clear_tmp.sh
+++ b/scripts/ci/tools/ci_clear_tmp.sh
@@ -27,7 +27,7 @@ sanity_checks::sanitize_mounted_files
read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)"
-docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
+docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
--rm \
--env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
"${AIRFLOW_CI_IMAGE}" \
diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh
index 2d57d65..56463d2 100755
--- a/scripts/ci/tools/ci_fix_ownership.sh
+++ b/scripts/ci/tools/ci_fix_ownership.sh
@@ -33,7 +33,7 @@ sanity_checks::sanitize_mounted_files
read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)"
-docker run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
+docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
--rm \
--env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
"${AIRFLOW_CI_IMAGE}" \
diff --git a/scripts/ci/tools/ci_free_space_on_ci.sh b/scripts/ci/tools/ci_free_space_on_ci.sh
index 4747848..a337545 100755
--- a/scripts/ci/tools/ci_free_space_on_ci.sh
+++ b/scripts/ci/tools/ci_free_space_on_ci.sh
@@ -26,7 +26,7 @@ echo "${COLOR_BLUE}Cleaning apt${COLOR_RESET}"
sudo apt clean
echo "${COLOR_BLUE}Pruning docker${COLOR_RESET}"
-docker system prune --all --force --volumes
+docker_v system prune --all --force --volumes
echo "${COLOR_BLUE}Free disk space ${COLOR_RESET}"
df -h