You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/08/19 13:25:01 UTC
[airflow] branch master updated: Replaced aliases for common tools
with functions. (#10402)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new db446f2 Replaced aliases for common tools with functions. (#10402)
db446f2 is described below
commit db446f267748c1c44229f03990b3aeb519c112f8
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Wed Aug 19 15:23:57 2020 +0200
Replaced aliases for common tools with functions. (#10402)
This allows for all the kinds of verbosity we want, including
writing outputs to output files, and it also works out-of-the-box
in git-commit non-interactive shell scripts. Also as a side effect
we have mocked tools in bats tests, which will allow us to write
more comprehensive unit tests for the bash scripts of ours
(this is a long overdue task).
Part of #10368
---
scripts/ci/libraries/_build_images.sh | 8 ++---
scripts/ci/libraries/_push_pull_remove_images.sh | 12 ++------
scripts/ci/libraries/_verbosity.sh | 35 +++++++++++++---------
.../ci/pre_commit/pre_commit_breeze_cmd_line.sh | 7 -----
.../pre_commit_build_providers_dependencies.sh | 5 ----
.../ci/pre_commit/pre_commit_check_integrations.sh | 5 ----
tests/bats/bats_utils.bash | 6 ++++
tests/bats/{bats_utils.bash => mocks/docker.sh} | 7 ++---
tests/bats/{bats_utils.bash => mocks/helm.sh} | 7 ++---
tests/bats/{bats_utils.bash => mocks/kind.sh} | 7 ++---
tests/bats/{bats_utils.bash => mocks/kubectl.sh} | 7 ++---
11 files changed, 40 insertions(+), 66 deletions(-)
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index fdfbc10..bd710cf 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -332,9 +332,7 @@ function prepare_ci_build() {
export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
if [[ ${USE_GITHUB_REGISTRY="false"} == "true" ]]; then
if [[ ${CACHE_REGISTRY_PASSWORD:=} != "" ]]; then
- # need eto be explicitly verbose in order to have pre-commit spinner working
- # No aliases in pre-commit non-interactive shells :(
- echo "${CACHE_REGISTRY_PASSWORD}" | verbose_docker login \
+ echo "${CACHE_REGISTRY_PASSWORD}" | docker login \
--username "${CACHE_REGISTRY_USERNAME}" \
--password-stdin \
"${CACHE_REGISTRY}"
@@ -577,9 +575,7 @@ Docker building ${AIRFLOW_CI_IMAGE}.
" > "${DETECTED_TERMINAL}"
fi
set +u
- # need eto be explicitly verbose in order to have pre-commit spinner working
- # No aliases in pre-commit non-interactive shells :(
- verbose_docker build \
+ docker build \
--build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
--build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index cd769dd..3a8d154 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -33,9 +33,7 @@ function pull_image_if_needed() {
echo
echo "Pulling the image ${IMAGE_TO_PULL}"
echo
- # need eto be explicitly verbose in order to have pre-commit spinner working
- # No aliases in pre-commit non-interactive shells :(
- verbose_docker pull "${IMAGE_TO_PULL}"
+ docker pull "${IMAGE_TO_PULL}"
EXIT_VALUE="$?"
echo
return ${EXIT_VALUE}
@@ -86,9 +84,7 @@ Docker pulling ${PYTHON_BASE_IMAGE}.
if [[ ${PULL_PYTHON_BASE_IMAGES_FROM_CACHE:="true"} == "true" ]]; then
pull_image_possibly_from_cache "${PYTHON_BASE_IMAGE}" "${CACHED_PYTHON_BASE_IMAGE}"
else
- # need eto be explicitly verbose in order to have pre-commit spinner working
- # No aliases in pre-commit non-interactive shells :(
- verbose_docker pull "${PYTHON_BASE_IMAGE}"
+ docker pull "${PYTHON_BASE_IMAGE}"
fi
echo
fi
@@ -110,9 +106,7 @@ function pull_prod_images_if_needed() {
if [[ ${PULL_PYTHON_BASE_IMAGES_FROM_CACHE:="true"} == "true" ]]; then
pull_image_possibly_from_cache "${PYTHON_BASE_IMAGE}" "${CACHED_PYTHON_BASE_IMAGE}"
else
- # need eto be explicitly verbose in order to have pre-commit spinner working
- # No aliases in pre-commit non-interactive shells :(
- verbose_docker pull "${PYTHON_BASE_IMAGE}"
+ docker pull "${PYTHON_BASE_IMAGE}"
fi
echo
fi
diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh
index c01f776..4230d96 100644
--- a/scripts/ci/libraries/_verbosity.sh
+++ b/scripts/ci/libraries/_verbosity.sh
@@ -24,9 +24,12 @@ function check_verbose_setup {
fi
}
+DOCKER_BINARY="${DOCKER_BINARY:=$(command -v docker)}"
+export DOCKER_BINARY
+
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be
# printed before execution
-function verbose_docker {
+function docker {
if [[ ${VERBOSE:="false"} == "true" && \
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
${VERBOSE_COMMANDS:=} != "true" && \
@@ -35,9 +38,9 @@ function verbose_docker {
>&2 echo "docker" "${@}"
fi
if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then
- docker "${@}" >>"${OUTPUT_LOG}" 2>&1
+ ${DOCKER_BINARY} "${@}" >>"${OUTPUT_LOG}" 2>&1
else
- docker "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
+ ${DOCKER_BINARY} "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
fi
EXIT_CODE="$?"
if [[ ${EXIT_CODE} == "0" ]]; then
@@ -48,43 +51,52 @@ function verbose_docker {
return "${EXIT_CODE}"
}
+HELM_BINARY="${HELM_BINARY:=$(command -v helm)}"
+export HELM_BINARY
+
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all helm commands run will be
# printed before execution
-function verbose_helm {
+function helm {
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "helm" "${@}"
fi
- helm "${@}" | tee -a "${OUTPUT_LOG}"
+ ${HELM_BINARY} "${@}" | tee -a "${OUTPUT_LOG}"
if [[ ${EXIT_CODE} == "0" ]]; then
# No matter if "set -e" is used the log will be removed on success.
rm -f "${OUTPUT_LOG}"
fi
}
+KUBECTL_BINARY=${KUBECTL_BINARY:=$(command -v kubectl)}
+export KUBECTL_BINARY
+
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kubectl commands run will be
# printed before execution
-function verbose_kubectl {
+function kubectl {
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "kubectl" "${@}"
fi
- kubectl "${@}" | tee -a "${OUTPUT_LOG}"
+ ${KUBECTL_BINARY} "${@}" | tee -a "${OUTPUT_LOG}"
if [[ ${EXIT_CODE} == "0" ]]; then
# No matter if "set -e" is used the log will be removed on success.
rm -f "${OUTPUT_LOG}"
fi
}
+KIND_BINARY="${KIND_BINARY:=$(command -v kind)}"
+export KIND_BINARY
+
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kind commands run will be
# printed before execution
-function verbose_kind {
+function kind {
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "kind" "${@}"
fi
# kind outputs nice output on terminal.
- kind "${@}"
+ ${KIND_BINARY} "${@}"
}
# Prints verbose information in case VERBOSE variable is set
@@ -106,8 +118,3 @@ function set_verbosity() {
}
set_verbosity
-
-alias docker=verbose_docker
-alias kubectl=verbose_kubectl
-alias helm=verbose_helm
-alias kind=verbose_kind
diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
index 68346de..b476cf2 100755
--- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
@@ -23,13 +23,6 @@ cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
-
-# For Pre-commits run in non-interactive shell so aliases do not work for them we need to add
-# local echo script to path so that it can be silenced.
-PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ):${PATH}"
-export PATH
-
-
TMP_FILE=$(mktemp)
TMP_OUTPUT=$(mktemp)
diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
index 21c1ea3..400b524 100755
--- a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
+++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.sh
@@ -24,11 +24,6 @@ cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
-# For Pre-commits run in non-interactive shell so aliases do not work for them we need to add
-# local echo script to path so that it can be silenced.
-PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ):${PATH}"
-export PATH
-
PYTHONPATH="$(pwd)"
export PYTHONPATH
find airflow/providers -name '*.py' -print0 | \
diff --git a/scripts/ci/pre_commit/pre_commit_check_integrations.sh b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
index e219685..0e9e85b 100755
--- a/scripts/ci/pre_commit/pre_commit_check_integrations.sh
+++ b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
@@ -23,11 +23,6 @@ cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
-# For Pre-commits run in non-interactive shell so aliases do not work for them we need to add
-# local echo script to path so that it can be silenced.
-PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ):${PATH}"
-export PATH
-
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash
index c06b182..e2f174e 100644
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/bats_utils.bash
@@ -18,5 +18,11 @@
AIRFLOW_SOURCES=$(pwd)
export AIRFLOW_SOURCES
export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
+
+export DOCKER_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/docker.sh
+export KUBECTL_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/kubectl.sh
+export KIND_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/kind.sh
+export HELM_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/helm.sh
+
# shellcheck source=scripts/ci/libraries/_all_libs.sh
source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
diff --git a/tests/bats/bats_utils.bash b/tests/bats/mocks/docker.sh
old mode 100644
new mode 100755
similarity index 79%
copy from tests/bats/bats_utils.bash
copy to tests/bats/mocks/docker.sh
index c06b182..af71146
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/mocks/docker.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,8 +16,4 @@
# specific language governing permissions and limitations
# under the License.
-AIRFLOW_SOURCES=$(pwd)
-export AIRFLOW_SOURCES
-export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
+# Noop for now, but we can use it in the future to mock more tests of bash scripts
diff --git a/tests/bats/bats_utils.bash b/tests/bats/mocks/helm.sh
old mode 100644
new mode 100755
similarity index 79%
copy from tests/bats/bats_utils.bash
copy to tests/bats/mocks/helm.sh
index c06b182..af71146
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/mocks/helm.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,8 +16,4 @@
# specific language governing permissions and limitations
# under the License.
-AIRFLOW_SOURCES=$(pwd)
-export AIRFLOW_SOURCES
-export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
+# Noop for now, but we can use it in the future to mock more tests of bash scripts
diff --git a/tests/bats/bats_utils.bash b/tests/bats/mocks/kind.sh
old mode 100644
new mode 100755
similarity index 79%
copy from tests/bats/bats_utils.bash
copy to tests/bats/mocks/kind.sh
index c06b182..af71146
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/mocks/kind.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,8 +16,4 @@
# specific language governing permissions and limitations
# under the License.
-AIRFLOW_SOURCES=$(pwd)
-export AIRFLOW_SOURCES
-export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
+# Noop for now, but we can use it in the future to mock more tests of bash scripts
diff --git a/tests/bats/bats_utils.bash b/tests/bats/mocks/kubectl.sh
old mode 100644
new mode 100755
similarity index 79%
copy from tests/bats/bats_utils.bash
copy to tests/bats/mocks/kubectl.sh
index c06b182..af71146
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/mocks/kubectl.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,8 +16,4 @@
# specific language governing permissions and limitations
# under the License.
-AIRFLOW_SOURCES=$(pwd)
-export AIRFLOW_SOURCES
-export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
+# Noop for now, but we can use it in the future to mock more tests of bash scripts