You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/12/06 12:46:18 UTC
[airflow] branch master updated: Build normal/backport providers
based on correct list (#12848)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 82dfa5f Build normal/backport providers based on correct list (#12848)
82dfa5f is described below
commit 82dfa5f74bfa39bedf127763dde4d08678202445
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Sun Dec 6 12:45:07 2020 +0000
Build normal/backport providers based on correct list (#12848)
This was using the backport list when trying to build the normal list,
leading to papermill package not getting built.
I have re-introduced the exit removed in #12841, and also done a little
bit of drive-by-tidying
---
.../run_install_and_test_provider_packages.sh | 18 +++++---
.../in_container/run_prepare_provider_packages.sh | 51 +++++++++++++++-------
2 files changed, 49 insertions(+), 20 deletions(-)
diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh
index 3894f08..faf1ad5 100755
--- a/scripts/in_container/run_install_and_test_provider_packages.sh
+++ b/scripts/in_container/run_install_and_test_provider_packages.sh
@@ -79,18 +79,24 @@ function discover_all_provider_packages() {
echo Listing available providers via 'airflow providers list'
echo
- airflow providers list
+ # Columns is to force it wider, so it doesn't wrap at 80 characters
+ COLUMNS=180 airflow providers list
- local expected_number_of_providers=60
+ local expected_number_of_providers=61
local actual_number_of_providers
- actual_number_of_providers=$(airflow providers list --output table | grep -c apache-airflow-providers | xargs)
- if [[ ${actual_number_of_providers} != "${expected_number_of_providers}" ]]; then
+ actual_providers=$(airflow providers list --output yaml | grep package_name)
+ actual_number_of_providers=$(wc -l <<<"$actual_providers")
+ if [[ "${actual_number_of_providers}" != "${expected_number_of_providers}" ]]; then
echo
- echo "${COLOR_RED_ERROR} Number of providers installed is wrong ${COLOR_RESET}"
+ echo "${COLOR_RED_ERROR}Number of providers installed is wrong${COLOR_RESET}"
echo "Expected number was '${expected_number_of_providers}' and got '${actual_number_of_providers}'"
echo
echo "Either increase the number of providers if you added one or diagnose and fix the problem."
echo
+ echo "Providers were:"
+ echo
+ echo "$actual_providers"
+ exit 1
fi
}
@@ -111,6 +117,7 @@ function discover_all_hooks() {
echo
echo "Either increase the number of hooks if you added one or diagnose and fix the problem."
echo
+ exit 1
fi
}
@@ -131,6 +138,7 @@ function discover_all_extra_links() {
echo
echo "Either increase the number of links if you added one or diagnose and fix the problem."
echo
+ exit 1
fi
}
diff --git a/scripts/in_container/run_prepare_provider_packages.sh b/scripts/in_container/run_prepare_provider_packages.sh
index 75c1d09..c40a70b 100755
--- a/scripts/in_container/run_prepare_provider_packages.sh
+++ b/scripts/in_container/run_prepare_provider_packages.sh
@@ -20,12 +20,7 @@
setup_provider_packages
-LIST_OF_DIRS_FILE=$(mktemp)
-cd "${AIRFLOW_SOURCES}/airflow/providers" || exit 1
-
-find . -type d | sed 's/.\///; s/\//\./g' | grep -E 'hooks|operators|sensors|secrets|utils' \
- > "${LIST_OF_DIRS_FILE}"
cd "${AIRFLOW_SOURCES}/provider_packages" || exit 1
@@ -34,12 +29,25 @@ REFACTOR_PROVIDER_PACKAGES_PY="${AIRFLOW_SOURCES}/dev/provider_packages/refactor
verify_suffix_versions_for_package_preparation
-if [[ -z "$*" ]]; then
- PROVIDERS_PACKAGES=$(python3 "${PREPARE_PROVIDER_PACKAGES_PY}" list-providers-packages)
+if [[ ${BACKPORT_PACKAGES} == "true" ]]; then
+ list_subcmd="list-backportable-packages"
+else
+ list_subcmd="list-providers-packages"
+fi
+function check_missing_providers() {
PACKAGE_ERROR="false"
+
+ pushd "${AIRFLOW_SOURCES}/airflow/providers" >/dev/null || exit 1
+
+ LIST_OF_DIRS_FILE=$(mktemp)
+ find . -type d | sed 's!./!!; s!/!.!g' | grep -E 'hooks|operators|sensors|secrets|utils' \
+ > "${LIST_OF_DIRS_FILE}"
+
+ popd >/dev/null
+
# Check if all providers are included
- for PACKAGE in ${PROVIDERS_PACKAGES}
+ for PACKAGE in "${PROVIDER_PACKAGES[@]}"
do
if ! grep -E "^${PACKAGE}" <"${LIST_OF_DIRS_FILE}" >/dev/null; then
echo "The package ${PACKAGE} is not available in providers dir"
@@ -54,16 +62,30 @@ if [[ -z "$*" ]]; then
exit 1
fi
- NUM_LINES=$(wc -l "${LIST_OF_DIRS_FILE}" | awk '{ print $1 }')
- if [[ ${NUM_LINES} != "0" ]]; then
+ if [[ $(wc -l < "${LIST_OF_DIRS_FILE}") != "0" ]]; then
echo "ERROR! Some folders from providers package are not defined"
echo " Please add them to dev/provider_packages/prepare_provider_packages.py:"
echo
cat "${LIST_OF_DIRS_FILE}"
echo
+
+ rm "$LIST_OF_DIRS_FILE"
exit 1
fi
- PROVIDER_PACKAGES=$(python3 "${PREPARE_PROVIDER_PACKAGES_PY}" list-backportable-packages)
+ rm "$LIST_OF_DIRS_FILE"
+}
+
+if [[ -z "$*" ]]; then
+ PROVIDER_PACKAGES=()
+ while IFS='' read -r line; do PROVIDER_PACKAGES+=("$line"); done < <(
+ python3 "${PREPARE_PROVIDER_PACKAGES_PY}" "$list_subcmd"
+ )
+
+ if [[ "$BACKPORT_PACKAGES" != "true" ]]; then
+ # Don't check for missing packages when we are building backports -- we have filtered someout, and the
+ # non-backport build will check for any missing.
+ check_missing_providers
+ fi
else
if [[ "$1" == "--help" ]]; then
echo
@@ -76,7 +98,7 @@ else
echo
exit
fi
- PROVIDER_PACKAGES="$*"
+ PROVIDER_PACKAGES=("$@")
fi
if [[ ${BACKPORT_PACKAGES} == "true" ]]; then
@@ -93,10 +115,9 @@ python3 "${REFACTOR_PROVIDER_PACKAGES_PY}"
rm -rf dist/*
-for PROVIDER_PACKAGE in ${PROVIDER_PACKAGES}
+for PROVIDER_PACKAGE in "${PROVIDER_PACKAGES[@]}"
do
- rm -rf -- *.egg-info
- rm -rf build/
+ rm -rf -- *.egg-info build/
LOG_FILE=$(mktemp)
python3 "${PREPARE_PROVIDER_PACKAGES_PY}" generate-setup-files "${PROVIDER_PACKAGE}"
echo "==================================================================================="