You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/08/15 19:15:03 UTC
[airflow] branch master updated: Breeze: More fancy environment
checking (#10329)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 88e5c35 Breeze: More fancy environment checking (#10329)
88e5c35 is described below
commit 88e5c35c819e00ba36423a4e2d8659acc6ec8938
Author: Kamil BreguĊa <mi...@users.noreply.github.com>
AuthorDate: Sat Aug 15 21:14:30 2020 +0200
Breeze: More fancy environment checking (#10329)
* More fancy environment checking
* fixup! More fancy environment checking
---
scripts/ci/in_container/check_environment.sh | 117 ++++++++-------------------
1 file changed, 35 insertions(+), 82 deletions(-)
diff --git a/scripts/ci/in_container/check_environment.sh b/scripts/ci/in_container/check_environment.sh
index d2ad571..9b2f0d7 100755
--- a/scripts/ci/in_container/check_environment.sh
+++ b/scripts/ci/in_container/check_environment.sh
@@ -23,20 +23,12 @@ EXIT_CODE=0
DISABLED_INTEGRATIONS=""
-function check_integration {
+function check_service {
INTEGRATION_NAME=$1
CALL=$2
MAX_CHECK=${3:=1}
- ENV_VAR_NAME=INTEGRATION_${INTEGRATION_NAME^^}
- if [[ ${!ENV_VAR_NAME:=} != "true" ]]; then
- DISABLED_INTEGRATIONS="${DISABLED_INTEGRATIONS} ${INTEGRATION_NAME}"
- return
- fi
-
- echo "-----------------------------------------------------------------------------------------------"
- echo " Checking integration ${INTEGRATION_NAME}"
- echo "-----------------------------------------------------------------------------------------------"
+ echo -n "${INTEGRATION_NAME}: "
while true
do
set +e
@@ -44,121 +36,82 @@ function check_integration {
RES=$?
set -e
if [[ ${RES} == 0 ]]; then
- echo
- echo " Integration ${INTEGRATION_NAME} OK!"
- echo
+ echo -e " \e[32mOK.\e[0m"
break
else
echo -n "."
MAX_CHECK=$((MAX_CHECK-1))
fi
if [[ ${MAX_CHECK} == 0 ]]; then
- echo
- echo "ERROR! Maximum number of retries while checking ${INTEGRATION_NAME} integration. Exiting"
- echo
+ echo -e " \e[31mERROR!\e[30m"
+ echo "Maximum number of retries while checking service. Exiting"
break
else
sleep 1
fi
done
if [[ ${RES} != 0 ]]; then
- echo " ERROR: Integration ${INTEGRATION_NAME} could not be started!"
+ echo "Service could not be started!"
echo
echo "${LAST_CHECK_RESULT}"
echo
EXIT_CODE=${RES}
fi
- echo "-----------------------------------------------------------------------------------------------"
}
-function check_db_connection {
- MAX_CHECK=${1:=3}
+function check_integration {
+ INTEGRATION_NAME=$1
+
+ ENV_VAR_NAME=INTEGRATION_${INTEGRATION_NAME^^}
+ if [[ ${!ENV_VAR_NAME:=} != "true" ]]; then
+ DISABLED_INTEGRATIONS="${DISABLED_INTEGRATIONS} ${INTEGRATION_NAME}"
+ return
+ fi
+ check_service "${@}"
+}
+
+function check_db_backend {
+ MAX_CHECK=${1:=1}
if [[ ${BACKEND} == "postgres" ]]; then
- HOSTNAME=postgres
- PORT=5432
+ check_service "postgres" "nc -zvv postgres 5432" "${MAX_CHECK}"
elif [[ ${BACKEND} == "mysql" ]]; then
- HOSTNAME=mysql
- PORT=3306
- else
+ check_service "mysql" "nc -zvv mysql 3306" "${MAX_CHECK}"
+ elif [[ ${BACKEND} == "sqlite" ]]; then
return
+ else
+ echo "Unknown backend. Supported values: [postgres,mysql,sqlite]. Current value: [${BACKEND}]"
+ exit 1
fi
- echo "-----------------------------------------------------------------------------------------------"
- echo " Checking DB ${BACKEND}"
- echo "-----------------------------------------------------------------------------------------------"
- while true
- do
- set +e
- LAST_CHECK_RESULT=$(nc -zvv ${HOSTNAME} ${PORT} 2>&1)
- RES=$?
- set -e
- if [[ ${RES} == 0 ]]; then
- echo
- echo " Backend ${BACKEND} OK!"
- echo
- break
- else
- echo -n "."
- MAX_CHECK=$((MAX_CHECK-1))
- fi
- if [[ ${MAX_CHECK} == 0 ]]; then
- echo
- echo "ERROR! Maximum number of retries while checking ${BACKEND} db. Exiting"
- echo
- break
- else
- sleep 1
- fi
- done
- if [[ ${RES} != 0 ]]; then
- echo " ERROR: ${BACKEND} db could not be reached!"
- echo
- echo "${LAST_CHECK_RESULT}"
- echo
- EXIT_CODE=${RES}
- fi
- echo "-----------------------------------------------------------------------------------------------"
}
function resetdb_if_requested() {
if [[ ${DB_RESET:="false"} == "true" ]]; then
if [[ ${RUN_AIRFLOW_1_10} == "true" ]]; then
- airflow resetdb -y
+ airflow resetdb -y
else
- airflow db reset -y
+ airflow db reset -y
fi
fi
return $?
}
-if [[ -n ${BACKEND:=} ]]; then
- echo "==============================================================================================="
- echo " Checking backend: ${BACKEND}"
- echo "==============================================================================================="
-
- set +e
- check_db_connection 20
- set -e
- if [[ ${EXIT_CODE} == 0 ]]; then
- echo "==============================================================================================="
- echo " Backend database is sane"
- echo "==============================================================================================="
- echo
- fi
-else
- echo "==============================================================================================="
- echo " Skip checking backend - BACKEND not set"
- echo "==============================================================================================="
- echo
+echo "==============================================================================================="
+echo " Checking integrations and backends"
+echo "==============================================================================================="
+if [[ -n ${BACKEND:=} ]]; then
+ check_db_backend 20
+ echo "-----------------------------------------------------------------------------------------------"
fi
-
check_integration kerberos "nc -zvv kerberos 88" 30
check_integration mongo "nc -zvv mongo 27017" 20
check_integration redis "nc -zvv redis 6379" 20
check_integration rabbitmq "nc -zvv rabbitmq 5672" 20
check_integration cassandra "nc -zvv cassandra 9042" 20
check_integration openldap "nc -zvv openldap 389" 20
+check_integration presto "nc -zvv presto 8080" 40
+echo "-----------------------------------------------------------------------------------------------"
if [[ ${EXIT_CODE} != 0 ]]; then
echo