You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/09/02 16:02:40 UTC
[airflow] branch revert-breeze created (now beceab5)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a change to branch revert-breeze
in repository https://gitbox.apache.org/repos/asf/airflow.git.
at beceab5 Revert "Implement Google Shell Conventions for breeze script … (#10651)"
This branch includes the following new commits:
new d9c4263 Revert "Add packages to function names in bash (#10670)"
new beceab5 Revert "Implement Google Shell Conventions for breeze script … (#10651)"
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airflow] 02/02: Revert "Implement Google Shell Conventions for breeze script … (#10651)"
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch revert-breeze
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit beceab5ac0dcc92be7a0b2dd6604b6ca53b52391
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Wed Sep 2 17:01:36 2020 +0100
Revert "Implement Google Shell Conventions for breeze script … (#10651)"
This reverts commit 46c8d6714c981746cc114b8b1af5cb27aa0018e2.
---
BREEZE.rst | 8 +-
breeze | 2571 ++++++++------------
breeze-complete | 20 +-
confirm | 2 +-
.../ci_prepare_and_test_backport_packages.sh | 2 +
.../ci_prepare_backport_packages.sh | 4 +
.../ci_prepare_backport_readme.sh | 4 +
...ci_test_backport_packages_import_all_classes.sh | 4 +
...ci_test_backport_packages_install_separately.sh | 4 +
scripts/ci/constraints/ci_branch_constraints.sh | 4 +
scripts/ci/constraints/ci_commit_constraints.sh | 4 +
scripts/ci/constraints/ci_generate_constraints.sh | 4 +
scripts/ci/docs/ci_docs.sh | 4 +
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 1 +
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 1 +
scripts/ci/images/ci_push_ci_images.sh | 2 +
scripts/ci/images/ci_push_production_images.sh | 2 +
scripts/ci/images/ci_wait_for_all_ci_images.sh | 26 +-
scripts/ci/images/ci_wait_for_all_prod_images.sh | 29 +-
.../ci/kubernetes/ci_deploy_app_to_kubernetes.sh | 12 +-
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 11 +-
scripts/ci/libraries/_all_libs.sh | 10 +-
scripts/ci/libraries/_build_images.sh | 210 +-
scripts/ci/libraries/_initialization.sh | 251 +-
scripts/ci/libraries/_kind.sh | 56 +-
scripts/ci/libraries/_md5sum.sh | 8 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 37 +-
scripts/ci/libraries/_sanity_checks.sh | 25 +-
scripts/ci/libraries/_script_init.sh | 23 +-
scripts/ci/libraries/_start_end.sh | 1 +
scripts/ci/libraries/_traps.sh | 38 -
scripts/ci/libraries/_verbosity.sh | 48 +-
scripts/ci/openapi/client_codegen_diff.sh | 1 +
.../ci/pre_commit/pre_commit_breeze_cmd_line.sh | 16 -
.../ci/pre_commit/pre_commit_check_integrations.sh | 5 +-
scripts/ci/pre_commit/pre_commit_ci_build.sh | 1 +
.../ci/pre_commit/pre_commit_local_yml_mounts.sh | 4 +-
scripts/ci/pre_commit/pre_commit_mermaid.sh | 9 +-
scripts/ci/static_checks/bat_tests.sh | 4 +
scripts/ci/static_checks/flake8.sh | 4 +
scripts/ci/static_checks/mypy.sh | 4 +
scripts/ci/static_checks/pylint.sh | 4 +
scripts/ci/static_checks/refresh_pylint_todo.sh | 4 +-
scripts/ci/static_checks/run_static_checks.sh | 4 +
scripts/ci/testing/ci_run_airflow_testing.sh | 7 +-
scripts/ci/tools/ci_clear_tmp.sh | 16 +
scripts/ci/tools/ci_count_changed_files.sh | 3 +
scripts/ci/tools/ci_fix_ownership.sh | 16 +
scripts/in_container/_in_container_script_init.sh | 2 +-
scripts/in_container/_in_container_utils.sh | 25 +-
scripts/in_container/check_environment.sh | 4 +-
scripts/in_container/entrypoint_ci.sh | 6 +-
scripts/in_container/prod/entrypoint_prod.sh | 4 +-
scripts/in_container/run_cli_tool.sh | 10 +-
scripts/in_container/run_docs_build.sh | 5 +-
scripts/in_container/run_generate_constraints.sh | 5 +-
.../in_container/run_prepare_backport_packages.sh | 8 +-
.../in_container/run_prepare_backport_readme.sh | 5 +-
tests/bats/bats_utils.bash | 17 +-
tests/bats/test_breeze_params.bats | 26 +
tests/bats/test_empty_test.bats | 2 +
tests/bats/test_local_mounts.bats | 2 +
62 files changed, 1467 insertions(+), 2182 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 5d87345..2ffa186 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1048,10 +1048,8 @@ This is the current syntax for `./breeze <./breeze>`_:
Usage: breeze [FLAGS] [COMMAND] -- <EXTRA_ARGS>
- By default the script enters the CI container and drops you to bash shell, but you can choose
- one of the commands to run specific actions instead.
-
- Add --help after each command to see details:
+ By default the script enters IT environment and drops you to bash shell, but you can choose one
+ of the commands to run specific actions instead. Add --help after each command to see details:
Commands without arguments:
@@ -1801,7 +1799,7 @@ This is the current syntax for `./breeze <./breeze>`_:
Detailed usage for command: static-check
- breeze static-check [FLAGS] static_check [-- <EXTRA_ARGS>]
+ breeze static-check [FLAGS] STATIC_CHECK [-- <EXTRA_ARGS>]
Run selected static checks for currently changed files. You should specify static check that
you would like to run or 'all' to run all checks. One of:
diff --git a/breeze b/breeze
index d03e556..1e4a939 100755
--- a/breeze
+++ b/breeze
@@ -18,81 +18,83 @@
# under the License.
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-export AIRFLOW_SOURCES
-readonly AIRFLOW_SOURCES
+AIRFLOW_SOURCES="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Bash arrays need to be defined outside of functions unfortunately :(
-# Because on Mac OS Bash 3.4 defining arrays inside functions does not work
# Array with extra options for Docker compose
declare -a EXTRA_DC_OPTIONS
-export EXTRA_DC_OPTIONS
-
# Array with selected integrations
declare -a INTEGRATIONS
-export INTEGRATIONS
-
# This is where remaining args are passed
declare -a REMAINING_ARGS
-export REMAINING_ARGS
-
# This is where static check options are defined
declare -a EXTRA_STATIC_CHECK_OPTIONS
-export EXTRA_STATIC_CHECK_OPTIONS
-#######################################################################################################
-# Sets up all the default variables for Breeze. They are needed by all other functions
-# All those variables are exported. They are not set to read-only because those
-# defaults can be modified later on when command line arguments are parsed
-# and variables stored in .build directory (stored in the previous run) are read
-#
-# Used globals:
-# FORCE_SCREEN_WIDTH
-#
-# Modified globals (constants or candidates for constants after we override them via appropriate flags):
-#
-# BREEZE
-# SUPPRESS_CHEATSHEET_FILE
-# SUPPRESS_ASCIIART_FILE
-# MAX_SCREEN_WIDTH
-# SCREEN_WIDTH
-# MOUNT_LOCAL_SOURCES
-# MOUNT_FILES
-# FORCE_PULL_IMAGES
-# ENABLE_KIND_CLUSTER
-# FORWARD_CREDENTIALS
-# DB_RESET
-# INSTALL_AIRFLOW_VERSION
-# INSTALL_AIRFLOW_REFERENCE
-# FORCE_BUILD_IMAGES
-# PRODUCTION_IMAGE
-# PYTHON_MAJOR_MINOR_VERSION
-#
-# Global variables:
-#
-# command_to_run
-# second_command_to_run
-# docker_compose_command
-#
-# Also it sets the variables and globals set by common initialization functions from
-# scripts/ci/libraries/_initialization.sh and breeze-complete script (which sets-up auto-complete).
-#
-#######################################################################################################
-function setup_default_breeze_constants() {
+# Sets up all the default variables for Breeze
+# They are needed by all other functions
+function setup_default_breeze_variables() {
+ # Default command to run - entering breeze environment
+ COMMAND_TO_RUN="enter_breeze"
+ # In some cases we also want to run two commands in a row (for example when we restart the environment)
+ SECOND_COMMAND_TO_RUN=""
# Indicates that we are inside Breeze environment
export BREEZE=true
- readonly BREEZE
+ # Maximum screen width to print the lines spanning the whole terminal width
+ export MAX_SCREEN_WIDTH=100
+ # By default we build CI images but when we specify `--production-image` we switch to production image
+ export PRODUCTION_IMAGE="false"
+ # Directory where all CI scripts are located
+ export SCRIPTS_CI_DIR="${AIRFLOW_SOURCES}/scripts/ci"
+
+ # Directory where all the build cache is stored - we keep there status of all the docker images
+ # As well as hashes of the important files, but also we generate build scripts there that are
+ # Used to execute the commands for breeze
+ export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
+ # This folder is mounted to inside the container in /files folder. This is the way how
+ # We can exchange DAGs, scripts, packages etc with the container environment
+ export FILES_DIR="${AIRFLOW_SOURCES}/files"
+ # Temporary dir used well ... temporarily
+ export TMP_DIR="${AIRFLOW_SOURCES}/tmp"
+
+ # Create those folders above in case they do not exist
+ mkdir -p "${BUILD_CACHE_DIR}" >/dev/null
+ mkdir -p "${TMP_DIR}" >/dev/null
+ mkdir -p "${FILES_DIR}" >/dev/null
- # If those files are present, the ASCII-art/cheat-sheet are suppressed
- SUPPRESS_CHEATSHEET_FILE="${AIRFLOW_SOURCES}/.build/.suppress_cheatsheet"
- readonly SUPPRESS_CHEATSHEET_FILE
+ # load all the common functions here - those are the functions that are shared between Breeze
+ # and CI scripts (CI scripts do not use Breeze as execution environment)
+ # shellcheck source=scripts/ci/libraries/_all_libs.sh
+ . "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
- SUPPRESS_ASCIIART_FILE="${AIRFLOW_SOURCES}/.build/.suppress_asciiart"
- readonly SUPPRESS_ASCIIART_FILE
+ # We have different versions of images depending on the python version used. We keep up with the
+ # Latest patch-level changes in Python (this is done automatically during CI builds) so we have
+ # To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor
+ # version of python in X.Y format (3.6, 3.7, 3.8 etc).
+ export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
- # Maximum screen indented_screen_width to print the lines spanning the whole terminal indented_screen_width
- export MAX_SCREEN_WIDTH=100
- readonly MAX_SCREEN_WIDTH
+ # When we generate documentation for README files, we want to force the width of terminal so that
+ # No matter who is running the documentation generation gets the same output
+ if [[ ${FORCE_SCREEN_WIDTH:="false"} != "true" ]]; then
+ # Sets width of the screen from terminal
+ SCREEN_WIDTH="$(tput cols)"
+ if [[ -z ${SCREEN_WIDTH} ]]; then
+ SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
+ fi
+ if (( SCREEN_WIDTH > MAX_SCREEN_WIDTH )); then
+ SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
+ fi
+ else
+ SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
+ fi
+
+ # Name of the script is kept in this variable
+ CMDNAME="$(basename -- "$0")"
+ export CMDNAME
+
+ # Update short and long options in the breeze-complete script
+ # This way autocomplete will work automatically with all options available
+ # shellcheck source=breeze-complete
+ . "${AIRFLOW_SOURCES}/breeze-complete"
# By default we mount local Airflow sources
export MOUNT_LOCAL_SOURCES="true"
@@ -100,6 +102,19 @@ function setup_default_breeze_constants() {
# By default we mount files folder
export MOUNT_FILES="true"
+ # Holds last sub-command used - this is used by --help flag to print help for the command entered
+ export LAST_SUBCOMMAND=""
+
+ # Determines if help should be run (set to true by --help flag)
+ export RUN_HELP="false"
+
+ # Holds docker compose command if the `docker-compose` command is used.
+ export DOCKER_COMPOSE_COMMAND=""
+
+ # If set to true, the docker images are rebuilt locally. By default we assume we do not need to
+ # rebuild the image but if we find we do, this variable will be set to "true"
+ export NEEDS_DOCKER_BUILD="false"
+
# By default we only pull images if we do not have them locally.
# This can be overridden by '--force-pull-images' flag
export FORCE_PULL_IMAGES="false"
@@ -115,89 +130,55 @@ function setup_default_breeze_constants() {
# If it set is set to specified version, then the source version of Airflow
# is removed and the specified version of Airflow is installed from PyPi
- export INSTALL_AIRFLOW_VERSION=""
+ export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""}
# If it is set to specified reference (tag/branch), then the source version
# of Airflow is removed and the specified version of Airflow is installed from GitHub
- export INSTALL_AIRFLOW_REFERENCE=""
+ export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""}
# Determines whether to force build without checking if it is needed
# Can be overridden by '--force-build-images' flag.
- export FORCE_BUILD_IMAGES="false"
+ export FORCE_BUILD_IMAGES=${FORCE_BUILD_IMAGES:="false"}
- # load all the common functions here - those are the functions that are shared between Breeze
- # and CI scripts. The CI scripts do not use Breeze as driving script - they read all configuration
- # from the environment variables. That's why we keep all the common initialization in those libs
- # shellcheck source=scripts/ci/libraries/_all_libs.sh
- . "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
- # When we generate documentation for README files, we want to force the indented_screen_width of terminal so that
- # No matter who is running the documentation generation gets the same output
- if [[ ${FORCE_SCREEN_WIDTH:="false"} != "true" ]]; then
- # Sets indented_screen_width of the screen from terminal
- SCREEN_WIDTH="$(tput cols)"
- if [[ -z ${SCREEN_WIDTH=} ]]; then
- SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
- fi
- if ((SCREEN_WIDTH > MAX_SCREEN_WIDTH)); then
- SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
- fi
- else
- SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
- fi
-
- export SCREEN_WIDTH
- readonly SCREEN_WIDTH
-
- # Update short and long options in the breeze-complete script
- # This way autocomplete will work automatically with all options available
- # shellcheck source=breeze-complete
- . "${AIRFLOW_SOURCES}/breeze-complete"
-
- # Default command to run - entering breeze environment
- command_to_run="enter_breeze"
-
- # In some cases we also want to run two commands in a row (for example when we restart the environment)
- second_command_to_run=""
-
- # Determines if help should be run (set to true by --help flag)
- run_help="false"
-
- # Holds docker compose command if the `docker-compose` command is used.
- docker_compose_command=""
+ # If those files are present, the ASCII-art/cheat-sheet are suppressed
+ SUPPRESS_CHEATSHEET_FILE="${AIRFLOW_SOURCES}/.suppress_cheatsheet"
+ SUPPRESS_ASCIIART_FILE="${AIRFLOW_SOURCES}/.suppress_asciiart"
+
+ # Default values for the flags used
+
+ _BREEZE_DEFAULT_BACKEND="sqlite"
+ _BREEZE_DEFAULT_KUBERNETES_MODE="image"
+ _BREEZE_DEFAULT_KUBERNETES_VERSION="v1.18.6"
+ _BREEZE_DEFAULT_KIND_VERSION="v0.8.0"
+ _BREEZE_DEFAULT_HELM_VERSION="v3.2.4"
+ _BREEZE_DEFAULT_POSTGRES_VERSION="9.6"
+ _BREEZE_DEFAULT_POSTGRES_VERSION="9.6"
+ _BREEZE_DEFAULT_MYSQL_VERSION="5.7"
+ STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION=3.6
}
-#######################################################################################################
-#
# Initializes development-friendly virtualenv if you are already in such env. It installs all the necessary
# packages from PyPI and it case of problems it provides useful hints on what prerequisites should be
# installed. It also removes and resets the existing AIRFLOW_HOME installation to make sure that you
# have it synchronized with the version of airflow installed. It resets the airflow's sqlite database to
# a clean state. You can use this function if your virtualenv is broken, to clean it up
-#
-# Used globals:
-# PYTHON_MAJOR_MINOR_VERSION
-# AIRFLOW_HOME_DIR
-# AIRFLOW_SOURCES
-# DEFAULT_CONSTRAINTS_BRANCH
-# OSTYPE
-#
-#######################################################################################################
function initialize_virtualenv() {
- # Check if we are inside virtualenv
- set +e
- echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" |
- python"${PYTHON_MAJOR_MINOR_VERSION}"
- local res=$?
- set -e
- if [[ ${res} != "0" ]]; then
+ # Check if we are inside virtualenv
+ set +e
+ echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" | "python${PYTHON_MAJOR_MINOR_VERSION}"
+ RES=$?
+ set -e
+ if [[ ${RES} != "0" ]]; then
echo >&2
echo >&2 "ERROR: Initializing local virtualenv only works when you have virtualenv activated"
echo >&2
echo >&2 "Please enter your local virtualenv before (for example using 'pyenv activate' or 'workon') "
echo >&2
exit 1
- else
+ else
+ # If no Airflow Home defined - fallback to ${HOME}/airflow
+ AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow}
+ export CASS_DRIVER_NO_CYTHON="1"
echo
echo "Initializing the virtualenv: $(command -v python)!"
echo
@@ -207,18 +188,18 @@ function initialize_virtualenv() {
echo
pushd "${AIRFLOW_SOURCES}"
set +e
- # We need to export this one to speed up Cassandra driver installation in virtualenv
- CASS_DRIVER_NO_CYTHON="1" pip install -e ".[devel]" \
+ pip install -e ".[devel]" \
--constraint "https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
- res=$?
+ RES=$?
set -e
popd
- if [[ ${res} != "0" ]]; then
+ if [[ ${RES} != "0" ]]; then
echo "#######################################################################"
echo " You had some troubles installing the venv !!!!!"
echo " Try running the command below and rerun virtualenv installation"
echo
- if [[ ${OSTYPE} == "darwin"* ]]; then
+ SYSTEM=$(uname -s)
+ if [[ ${SYSTEM} == "Darwin" ]]; then
echo " brew install sqlite mysql postgresql openssl"
else
echo " sudo apt install build-essentials python3.6-dev python3.7-dev python3.8-dev python-dev openssl \\"
@@ -226,7 +207,7 @@ function initialize_virtualenv() {
fi
echo
echo "#######################################################################"
- exit ${res}
+ exit ${RES}
fi
echo
echo "Wiping and recreating ${AIRFLOW_HOME_DIR}"
@@ -236,26 +217,18 @@ function initialize_virtualenv() {
echo
echo "Resetting AIRFLOW sqlite database"
echo
- AIRFLOW__CORE__UNIT_TEST_MODE="False" airflow db reset -y
+ unset AIRFLOW__CORE__UNIT_TEST_MODE
+ airflow db reset -y
echo
echo "Resetting AIRFLOW sqlite unit test database"
echo
- AIRFLOW__CORE__UNIT_TEST_MODE="True" airflow db reset -y
+ AIRFLOW__CORE__UNIT_TEST_MODE=True airflow db reset -y
exit 0
- fi
+ fi
}
-#######################################################################################################
-#
+
# Sets up autocomplete for Breeze for both - bash and zsh
-#
-# Used globals:
-#
-# AIRFLOW_SOURCES
-# HOME
-# OSTYPE
-#
-#######################################################################################################
function setup_autocomplete() {
echo "Installing bash/zsh completion for local user"
echo
@@ -267,17 +240,17 @@ function setup_autocomplete() {
echo
echo "Breeze Bash completion is now linked to: ${AIRFLOW_SOURCES}/breeze-complete"
echo
- local breeze_comment="Added by Airflow Breeze autocomplete setup"
- if ! grep "${breeze_comment}" "${HOME}/.bashrc" >/dev/null 2>&1; then
+ local BREEZE_COMMENT="Added by Airflow Breeze autocomplete setup"
+ if ! grep "${BREEZE_COMMENT}" "${HOME}/.bashrc" >/dev/null 2>&1; then
touch ~/.bashrc
# shellcheck disable=SC2129
- echo "# START: ${breeze_comment}" >>~/.bashrc
+ echo "# START: ${BREEZE_COMMENT}" >>~/.bashrc
cat <<"EOF" >>~/.bashrc
-for bcfile in ~/.bash_completion.d/* ; do
- . ${bcfile}
+for BCFILE in ~/.bash_completion.d/* ; do
+ . ${BCFILE}
done
EOF
- echo "# END: ${breeze_comment}" >>~/.bashrc
+ echo "# END: ${BREEZE_COMMENT}" >>~/.bashrc
echo
echo "The ${HOME}/.bashrc has been modified"
echo
@@ -286,15 +259,15 @@ EOF
echo "The ${HOME}/.bashrc was already modified before. Not changing it."
echo
fi
- if ! grep "${breeze_comment}" "${HOME}/.zshrc" >/dev/null 2>&1; then
+ if ! grep "${BREEZE_COMMENT}" "${HOME}/.zshrc" >/dev/null 2>&1; then
# shellcheck disable=SC2129
- echo "# START: ${breeze_comment}" >>~/.zshrc
+ echo "# START: ${BREEZE_COMMENT}" >>~/.zshrc
cat <<"EOF" >>~/.zshrc
autoload compinit && compinit
autoload bashcompinit && bashcompinit
source ~/.bash_completion.d/breeze-complete
EOF
- echo "# END: ${breeze_comment}" >>~/.zshrc
+ echo "# END: ${BREEZE_COMMENT}" >>~/.zshrc
echo
echo "The ${HOME}/.zshrc has been modified"
echo
@@ -307,15 +280,15 @@ EOF
# For MacOS we have to handle the special case where terminal app DOES NOT run .bashrc by default
# But re-runs .bash_profile :(
# See https://scriptingosx.com/2017/04/about-bash_profile-and-bashrc-on-macos/
- if ! grep "${breeze_comment}" "${HOME}/.bash_profile"; then
+ if ! grep "${BREEZE_COMMENT}" "${HOME}/.bash_profile"; then
# shellcheck disable=SC2129
- echo "# START: ${breeze_comment}" >>~/.bash_profile
+ echo "# START: ${BREEZE_COMMENT}" >>~/.bash_profile
cat <<"EOF" >>~/.bash_profile
if [ -r ~/.bashrc ]; then
source ~/.bashrc
fi
EOF
- echo "# END: ${breeze_comment}" >>~/.bash_profile
+ echo "# END: ${BREEZE_COMMENT}" >>~/.bash_profile
echo
echo "The ${HOME}/.bash_profile has been modified"
echo
@@ -336,41 +309,17 @@ EOF
exit 0
}
-#######################################################################################################
-#
# Prints information about the current configuration of Breeze - if you enter breeze interactively
-# and you did not suppress cheatsheet or asciiart, it also prints those. It also prints values
-# of constants set by read_saved_environment_variables() function and other initialization functions.
-#
-# Used globals:
-#
-# BACKEND
-# POSTGRES_VERSION
-# MYSQL_VERSION
-# SUPPRESS_CHEATSHEET_FILE
-# SUPPRESS_ASCIIART_FILE
-# PRODUCTION_IMAGE
-# BRANCH_NAME
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-# INSTALL_AIRFLOW_VERSION
-# INSTALL_AIRFLOW_REFERENCE
-#
-# Outputs:
-# Prints the information about the build to stdout.
-#
-#######################################################################################################
-function print_badge() {
- local backend_version=""
+# and you did not suppress cheatsheet or asciiart, it also prints those
+function print_badge {
if [[ ${BACKEND} == "postgres" ]]; then
- backend_version="${POSTGRES_VERSION}"
+ BACKEND_VERSION="${POSTGRES_VERSION}"
elif [[ ${BACKEND} == "mysql" ]]; then
- backend_version="${MYSQL_VERSION}"
+ BACKEND_VERSION="${MYSQL_VERSION}"
+ else
+ BACKEND_VERSION=""
fi
- if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" && ${command_to_run} == "enter_breeze" ]]; then
+ if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" && ${COMMAND_TO_RUN} == "enter_breeze" ]]; then
cat <<EOF
@@ -437,12 +386,12 @@ EOF
Python version: ${PYTHON_MAJOR_MINOR_VERSION}
DockerHub user: ${DOCKERHUB_USER}
DockerHub repo: ${DOCKERHUB_REPO}
- Backend: ${BACKEND} ${backend_version}
+ Backend: ${BACKEND} ${BACKEND_VERSION}
EOF
- if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE=} ]]; then
+ if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
cat <<EOF
- Airflow installed: ${INSTALL_AIRFLOW_VERSION=}${INSTALL_AIRFLOW_REFERENCE=}
+ Airflow installed from: ${INSTALL_AIRFLOW_VERSION}${INSTALL_AIRFLOW_REFERENCE}
EOF
fi
@@ -470,9 +419,9 @@ EOF
Python version: ${PYTHON_MAJOR_MINOR_VERSION}
DockerHub user: ${DOCKERHUB_USER}
DockerHub repo: ${DOCKERHUB_REPO}
- Backend: ${BACKEND} ${backend_version}
+ Backend: ${BACKEND} ${BACKEND_VERSION}
EOF
- if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE=} ]]; then
+ if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
cat <<EOF
Airflow installed from: ${INSTALL_AIRFLOW_VERSION}${INSTALL_AIRFLOW_REFERENCE}
@@ -482,72 +431,36 @@ EOF
fi
}
-#######################################################################################################
-#
-# Prepares command file that can be used to easily run the docker commands outside of Breeze.
-#
+# Prepares command file that can be used to easily run the commands without the need of using Breeze
# The command file generated in cache ./build directory is a standalone script that contains
-# All the environment variables and docker-compose configuration to run the command.
-# This is because depending on configuration of Breeze we might have different compose files
-# used and different env variables set.
-#
-# Those are a convenience scripts that you might use to debug command execution although
-# In most cases they are used internally by Breeze.
-#
-# Used Globals:
-# BRANCH_NAME
-# PYTHON_MAJOR_MINOR_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-# HOST_AIRFLOW_SOURCES
-# BACKEND
-# AIRFLOW_VERSION
-# INSTALL_AIRFLOW_VERSION
-# WEBSERVER_HOST_PORT
-# POSTGRES_HOST_PORT
-# POSTGRES_VERSION
-# MYSQL_HOST_PORT
-# MYSQL_VERSION
-# AIRFLOW_SOURCES
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_PROD_IMAGE_KUBERNETES
-# AIRFLOW_PROD_BASE_TAG
-# SQLITE_URL
-#
-# Arguments:
-#
-# file to prepare
-# command to run
-# compose_file to use
-# airflow_image to use
-#
-# Outputs:
-# Creates the convenience command file that can be run to use the docker command.
-#
-#######################################################################################################
+# All the environment variables and docker-compose configuration to run the command. This is because
+# depending on configuration of Breeze we might have different compose files used and different variables
+# set. Those are a convenience scripts that you might use to debug command execution although
+# In most cases they are used internally by Breeze
function prepare_command_file() {
- local file="${1}"
- local command="${2}"
- local compose_file="${3}"
- local airflow_image="${4}"
- cat <<EOF >"${file}"
+ local FILE="${1}"
+ local CMD="${2}"
+ local COMPOSE_FILE="${3}"
+ local AIRFLOW_IMAGE="${4}"
+ cat <<EOF > "${FILE}"
#!/usr/bin/env bash
if [[ \${VERBOSE} == "true" ]]; then
echo
echo "Executing script:"
echo
- echo "${file} \${@}"
+ echo "\${BASH_SOURCE[0]} \${@}"
echo
set -x
fi
cd "\$( dirname "\${BASH_SOURCE[0]}" )" || exit
export DOCKERHUB_USER=${DOCKERHUB_USER}
export DOCKERHUB_REPO=${DOCKERHUB_REPO}
-export HOST_USER_ID=${HOST_USER_ID}
-export HOST_GROUP_ID=${HOST_GROUP_ID}
+HOST_USER_ID=\$(id -ur)
+export HOST_USER_ID
+HOST_GROUP_ID=\$(id -gr)
+export HOST_GROUP_ID
export HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}"
-export COMPOSE_FILE="${compose_file}"
+export COMPOSE_FILE="${COMPOSE_FILE}"
export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}"
export BACKEND="${BACKEND}"
export AIRFLOW_VERSION="${AIRFLOW_VERSION}"
@@ -561,103 +474,92 @@ export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}"
export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}"
export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}"
export AIRFLOW_PROD_IMAGE_KUBERNETES="${AIRFLOW_PROD_IMAGE_KUBERNETES}"
-export AIRFLOW_PROD_BASE_TAG="${AIRFLOW_PROD_BASE_TAG}"
-export AIRFLOW_IMAGE="${airflow_image}"
+export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+export AIRFLOW_IMAGE="${AIRFLOW_IMAGE}"
export SQLITE_URL="${SQLITE_URL}"
-docker-compose --log-level INFO ${command}
+docker-compose --log-level INFO ${CMD}
EOF
- chmod u+x "${file}"
+ chmod u+x "${FILE}"
}
-#######################################################################################################
-#
-# Prepare all command files that we are using. Depending on the command to execute we use two
+
+# Prepare all command files that we are using. Depending on the command to execute we use different
# convenience scripts:
-#
+# cmd_run_ci - to run CI image command (for example entering Breeze, or running a script in CI image)
+# cmd_run_prod - to run PROD image command (for example entering Prod image or running a script there)
+# test_run_ci - to run test target in CI image
# dc_ci - to run docker compose command for CI image
# dc_prod - to run docker compose command for PROD image
#
-# Global constants set:
-#
-# PYTHON_BASE_IMAGE_VERSION
-# PYTHON_BASE_IMAGE
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_BASE_TAG
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_PROD_IMAGE_KUBERNETES
-# BUILT_CI_IMAGE_FLAG_FILE
-#
-#######################################################################################################
function prepare_command_files() {
- local main_ci_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
- local main_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
- local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
- local local_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local.yml
- local files_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/files.yml
- local local_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local-prod.yml
- local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
- local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
-
- local compose_ci_file=${main_ci_docker_compose_file}:${backend_docker_compose_file}
- local compose_prod_file=${main_prod_docker_compose_file}:${backend_docker_compose_file}
+ MAIN_CI_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/base.yml
+ MAIN_PROD_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/base.yml
+ BACKEND_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
+ LOCAL_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/local.yml
+ FILES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/files.yml
+ LOCAL_PROD_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/local-prod.yml
+ REMOVE_SOURCES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
+ FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
+
+ COMPOSE_CI_FILE=${MAIN_CI_DOCKER_COMPOSE_FILE}:${BACKEND_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${MAIN_PROD_DOCKER_COMPOSE_FILE}:${BACKEND_DOCKER_COMPOSE_FILE}
if [[ "${MOUNT_LOCAL_SOURCES}" != "false" ]]; then
- compose_ci_file=${compose_ci_file}:${local_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${local_prod_docker_compose_file}
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${LOCAL_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${LOCAL_PROD_DOCKER_COMPOSE_FILE}
fi
if [[ "${MOUNT_FILES}" != "false" ]]; then
- compose_ci_file=${compose_ci_file}:${files_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${files_docker_compose_file}
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${FILES_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${FILES_DOCKER_COMPOSE_FILE}
fi
if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
- compose_ci_file=${compose_ci_file}:${forward_credentials_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${forward_credentials_docker_compose_file}
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE}
fi
- if [[ -n ${INSTALL_AIRFLOW_VERSION=} ]]; then
- compose_ci_file=${compose_ci_file}:${remove_sources_docker_compose_file}
+ if [[ ${INSTALL_AIRFLOW_VERSION} != "" ]]; then
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${REMOVE_SOURCES_DOCKER_COMPOSE_FILE}
fi
set +u
- local unique_integrations
# shellcheck disable=SC2207
- unique_integrations=($(echo "${INTEGRATIONS[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
+ UNIQUE_INTEGRATIONS=($(echo "${INTEGRATIONS[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
- local integration
- for integration in "${unique_integrations[@]}"; do
- compose_ci_file=${compose_ci_file}:${SCRIPTS_CI_DIR}/docker-compose/integration-${integration}.yml
+ for _INT in "${UNIQUE_INTEGRATIONS[@]}"
+ do
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml
done
set -u
- export DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI="dc_ci"
- readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI
+ export COMPOSE_CI_FILE
+ export COMPOSE_PROD_FILE
- export DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD="dc_prod"
- readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD
+ get_base_image_version
+ # Base python image for the build
+ export PYTHON_BASE_IMAGE=python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster
+ export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
+ export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
+
+ LAST_DC_CI_FILE="dc_ci"
+ LAST_DC_PROD_FILE="dc_prod"
# Prepare script for "run docker compose CI command"
- prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" \
- "\"\${@}\"" "${compose_ci_file}" "${AIRFLOW_CI_IMAGE}"
+ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" \
+ "\"\${@}\"" "${COMPOSE_CI_FILE}" "${AIRFLOW_CI_IMAGE}"
# Prepare script for "run docker compose PROD command"
- prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}" \
- "\"\${@}\"" "${compose_prod_file}" "${AIRFLOW_PROD_IMAGE}"
+ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}" \
+ "\"\${@}\"" "${COMPOSE_PROD_FILE}" "${AIRFLOW_PROD_IMAGE}"
}
-#######################################################################################################
-#
# Prints detailed help for all commands and flags. Used to generate documentation added to BREEZE.rst
# automatically.
-#
-# Used globals:
-# ALL_BREEZE_COMMANDS
-#
-# Outputs:
-# Prints detailed help for all commands to stdout.
-#
-#######################################################################################################
function do_help_all() {
echo
print_line
@@ -669,695 +571,548 @@ function do_help_all() {
echo
print_line
echo
- local subcommand
- for subcommand in ${ALL_BREEZE_COMMANDS}; do
- detailed_usage "${subcommand}"
- print_line
- echo
+ for SUBCOMMAND in ${ALL_BREEZE_COMMANDS}
+ do
+ detailed_usage "${SUBCOMMAND}"
+ print_line
+ echo
done
echo
flags
}
-#######################################################################################################
-#
# Parses all arguments that can be passed to Breeze command - that includes command to run and flags.
-#
-# Updated global constants:
-# By the end of this function, all the constants from `make_constants_read_only` function are set
-# and they are set as read-only.
-#
-#######################################################################################################
function parse_arguments() {
set -u
- local params
- if ! params=$(getopt \
+ if ! PARAMS=$(getopt \
-o "${_BREEZE_GETOPT_SHORT_OPTIONS:=}" \
-l "${_BREEZE_GETOPT_LONG_OPTIONS:=}" \
- --name "$CMDNAME" -- "$@"); then
+ --name "$CMDNAME" -- "$@")
+ then
flags
exit 1
fi
- eval set -- "${params}"
- unset params
+ eval set -- "${PARAMS}"
+ unset PARAMS
# Parse Flags.
# Please update short and long options in the breeze-complete script
# This way autocomplete will work out-of-the-box
- while true; do
- case "${1}" in
- -h | --help)
- run_help="true"
- shift
- ;;
- -p | --python)
- export PYTHON_MAJOR_MINOR_VERSION="${2}"
- echo "Python version: ${PYTHON_MAJOR_MINOR_VERSION}"
- echo
- shift 2
- ;;
- -b | --backend)
- export BACKEND="${2}"
- echo "Backend: ${BACKEND}"
- echo
- shift 2
- ;;
- -i | --integration)
- local integration=${2}
- check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
- echo "Integration: ${integration}"
- if [[ ${integration} == "all" ]]; then
- for integration in ${_BREEZE_ALLOWED_INTEGRATIONS}; do
- if [[ ${integration} != "all" ]]; then
- echo "${integration}"
- INTEGRATIONS+=("${integration}")
- fi
- done
- else
- INTEGRATIONS+=("${INTEGRATION}")
- fi
- echo
- shift 2
- ;;
- -K | --kubernetes-mode)
- export KUBERNETES_MODE="${2}"
- echo "Kubernetes mode: ${KUBERNETES_MODE}"
- echo
- shift 2
- ;;
- -V | --kubernetes-version)
- export KUBERNETES_VERSION="${2}"
- echo "Kubernetes version: ${KUBERNETES_VERSION}"
- echo
- shift 2
- ;;
+ while true
+ do
+ case "${1}" in
+ -h|--help)
+ RUN_HELP="true"
+ shift ;;
+ -p|--python)
+ export PYTHON_MAJOR_MINOR_VERSION="${2}";
+ echo "Python version: ${PYTHON_MAJOR_MINOR_VERSION}"
+ echo
+ shift 2 ;;
+ -b|--backend)
+ export BACKEND="${2}";
+ echo "Backend: ${BACKEND}"
+ echo
+ shift 2 ;;
+ -i|--integration)
+ INTEGRATION=${2}
+ check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
+ echo "Integration: ${INTEGRATION}"
+ if [[ ${INTEGRATION} == "all" ]]; then
+ for _INT in ${_BREEZE_ALLOWED_INTEGRATIONS}
+ do
+ if [[ ${_INT} != "all" ]]; then
+ echo "${_INT}"
+ INTEGRATIONS+=("${_INT}")
+ fi
+ done
+ else
+ INTEGRATIONS+=("${INTEGRATION}");
+ fi
+ echo
+ shift 2 ;;
+ -K|--kubernetes-mode)
+ export KUBERNETES_MODE="${2}";
+ echo "Kubernetes mode: ${KUBERNETES_MODE}"
+ echo
+ shift 2 ;;
+ -V|--kubernetes-version)
+ export KUBERNETES_VERSION="${2}";
+ echo "Kubernetes version: ${KUBERNETES_VERSION}"
+ echo
+ shift 2 ;;
--kind-version)
- export KIND_VERSION="${2}"
- echo "Kind version: ${KIND_VERSION}"
- echo
- shift 2
- ;;
+ export KIND_VERSION="${2}";
+ echo "Kind version: ${KIND_VERSION}"
+ echo
+ shift 2 ;;
--helm-version)
- export HELM_VERSION="${2}"
- echo "Helm version: ${HELM_VERSION}"
- echo
- shift 2
- ;;
+ export HELM_VERSION="${2}";
+ echo "Helm version: ${HELM_VERSION}"
+ echo
+ shift 2 ;;
--postgres-version)
- export POSTGRES_VERSION="${2}"
- echo "Postgres version: ${POSTGRES_VERSION}"
- echo
- shift 2
- ;;
+ export POSTGRES_VERSION="${2}";
+ echo "Postgres version: ${POSTGRES_VERSION}"
+ echo
+ shift 2 ;;
--mysql-version)
- export MYSQL_VERSION="${2}"
- echo "MySQL version: ${MYSQL_VERSION}"
- echo
- shift 2
- ;;
- -l | --skip-mounting-local-sources)
- MOUNT_LOCAL_SOURCES="false"
- echo "Mount local sources: ${MOUNT_LOCAL_SOURCES}"
- echo
- shift
- ;;
- -a | --install-airflow-version)
- INSTALL_AIRFLOW_VERSION="${2}"
- # Reference is mutually exclusive with version
- INSTALL_AIRFLOW_REFERENCE=""
- echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}"
- echo
- shift 2
- ;;
- -t | --install-airflow-reference)
- INSTALL_AIRFLOW_REFERENCE="${2}"
- # Reference is mutually exclusive with version
- INSTALL_AIRFLOW_VERSION=""
- echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}"
- echo
- shift 2
- ;;
- -d | --db-reset)
- echo "Resetting the DB!"
- echo
- export DB_RESET="true"
- shift 1
- ;;
- -v | --verbose)
- export VERBOSE="true"
- echo "Verbose output"
- echo
- shift
- ;;
- -y | --assume-yes)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- echo "Assuming 'yes' answer to all questions."
- echo
- shift
- ;;
- -n | --assume-no)
- export FORCE_ANSWER_TO_QUESTIONS="no"
- echo "Assuming 'no' answer to all questions."
- echo
- shift
- ;;
- -q | --assume-quit)
- export FORCE_ANSWER_TO_QUESTIONS="quit"
- echo "Assuming 'quit' answer to all questions."
- echo
- shift
- ;;
- -F | --force-build-images)
- echo "Force build images"
- echo
- export FORCE_BUILD_IMAGES="true"
- # if you want to force build an image - assume you want to build it :)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- shift
- ;;
- -C | --force-clean-images)
- echo "Clean build of images without cache"
- echo
- export DOCKER_CACHE="no-cache"
- # if not set here, docker cached is determined later, depending on type of image to be build
- readonly DOCKER_CACHE
- export FORCE_BUILD_IMAGES="true"
- shift
- ;;
- -r | --skip-rebuild-check)
- echo "Skips checking image for rebuilds"
- echo
- export CHECK_IMAGE_FOR_REBUILD="false"
- export SKIP_BUILDING_PROD_IMAGE="true"
- shift
- ;;
- -L | --build-cache-local)
- echo "Use local cache to build images"
- echo
- export DOCKER_CACHE="local"
- # if not set here, docker cached is determined later, depending on type of image to be build
- readonly DOCKER_CACHE
- shift
- ;;
- -U | --build-cache-pulled)
- echo "Use pulled cache to build images"
- echo
- export DOCKER_CACHE="pulled"
- # if not set here, docker cached is determined later, depending on type of image to be build
- readonly DOCKER_CACHE
- shift
- ;;
- -X | --build-cache-disabled)
- echo "Use disabled cache to build images"
- echo
- export DOCKER_CACHE="disabled"
- readonly DOCKER_CACHE
- # if not set here, docker cached is determined later, depending on type of image to be build
- shift
- ;;
- -P | --force-pull-images)
- echo "Force pulling images before build. Uses pulled images as cache."
- echo
- export FORCE_PULL_IMAGES="true"
- export FORCE_BUILD_IMAGES="true"
- # if you want to force build an image - assume you want to build it :)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- shift
- ;;
- -I | --production-image)
- export PRODUCTION_IMAGE="true"
- export SQLITE_URL=
- echo
- echo "*************** PRODUCTION IMAGE *************************"
- echo
- shift
- ;;
- -E | --extras)
- export AIRFLOW_EXTRAS="${2}"
- echo "Extras : ${AIRFLOW_EXTRAS}"
- shift 2
- ;;
+ export MYSQL_VERSION="${2}";
+ echo "MySQL version: ${MYSQL_VERSION}"
+ echo
+ shift 2 ;;
+ -l|--skip-mounting-local-sources)
+ MOUNT_LOCAL_SOURCES="false"
+ echo "Mount local sources: ${MOUNT_LOCAL_SOURCES}"
+ echo
+ shift ;;
+ -a|--install-airflow-version)
+ INSTALL_AIRFLOW_VERSION="${2}"
+ # Reference is exclusive with version
+ INSTALL_AIRFLOW_REFERENCE=""
+ echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}"
+ echo
+ shift 2 ;;
+ -t|--install-airflow-reference)
+ INSTALL_AIRFLOW_REFERENCE="${2}"
+ # Reference is exclusive with version
+ INSTALL_AIRFLOW_VERSION=""
+ echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}"
+ echo
+ shift 2 ;;
+ -d|--db-reset)
+ echo "Resetting the DB!"
+ echo
+ export DB_RESET="true"
+ shift 1 ;;
+ -v|--verbose)
+ export VERBOSE="true"
+ echo "Verbose output"
+ echo
+ shift ;;
+ -y|--assume-yes)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ echo "Assuming 'yes' answer to all questions."
+ echo
+ shift ;;
+ -n|--assume-no)
+ export FORCE_ANSWER_TO_QUESTIONS="no"
+ echo "Assuming 'no' answer to all questions."
+ echo
+ shift ;;
+ -q|--assume-quit)
+ export FORCE_ANSWER_TO_QUESTIONS="quit"
+ echo "Assuming 'quit' answer to all questions."
+ echo
+ shift ;;
+ -F|--force-build-images)
+ echo "Force build images"
+ echo
+ export FORCE_BUILD_IMAGES="true"
+ # if you want to force build an image - assume you want to build it :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ shift ;;
+ -C|--force-clean-images)
+ echo "Clean build of images without cache"
+ echo
+ export DOCKER_CACHE="no-cache"
+ export FORCE_BUILD_IMAGES="true"
+ shift ;;
+ -r|--skip-rebuild-check)
+ echo "Skips checking image for rebuilds"
+ echo
+ export CHECK_IMAGE_FOR_REBUILD="false"
+ export SKIP_BUILDING_PROD_IMAGE="true"
+ shift ;;
+ -L|--build-cache-local)
+ echo "Use local cache to build images"
+ echo
+ export DOCKER_CACHE="local"
+ shift ;;
+ -U|--build-cache-pulled)
+ echo "Use pulled cache to build images"
+ echo
+ export DOCKER_CACHE="pulled"
+ shift ;;
+ -X|--build-cache-disabled)
+ echo "Use disabled cache to build images"
+ echo
+ export DOCKER_CACHE="disabled"
+ shift ;;
+ -P|--force-pull-images)
+ echo "Force pulling images before build. Uses pulled images as cache."
+ echo
+ export FORCE_PULL_IMAGES="true"
+ export FORCE_BUILD_IMAGES="true"
+ # if you want to force build an image - assume you want to build it :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ shift ;;
+ -I|--production-image)
+ export PRODUCTION_IMAGE="true"
+ export SQLITE_URL=
+ echo
+ echo "*************** PRODUCTION IMAGE *************************"
+ echo
+ shift ;;
+ -E|--extras)
+ export AIRFLOW_EXTRAS="${2}"
+ echo "Extras : ${AIRFLOW_EXTRAS}"
+ shift 2 ;;
--additional-extras)
- export ADDITIONAL_AIRFLOW_EXTRAS="${2}"
- echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}"
- shift 2
- ;;
+ export ADDITIONAL_AIRFLOW_EXTRAS="${2}"
+ echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}"
+ shift 2 ;;
--additional-python-deps)
- export ADDITIONAL_PYTHON_DEPS="${2}"
- echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}"
- shift 2
- ;;
+ export ADDITIONAL_PYTHON_DEPS="${2}"
+ echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}"
+ shift 2 ;;
--additional-dev-deps)
- export ADDITIONAL_DEV_DEPS="${2}"
- echo "Additional apt dev dependencies: ${ADDITIONAL_DEV_DEPS}"
- shift 2
- ;;
+ export ADDITIONAL_DEV_DEPS="${2}"
+ echo "Additional apt dev dependencies: ${ADDITIONAL_DEV_DEPS}"
+ shift 2 ;;
--additional-runtime-deps)
- export ADDITIONAL_RUNTIME_DEPS="${2}"
- echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}"
- shift 2
- ;;
- -D | --dockerhub-user)
- export DOCKERHUB_USER="${2}"
- echo "Dockerhub user ${DOCKERHUB_USER}"
- echo
- shift 2
- ;;
- -R | --dockerhub-repo)
- export DOCKERHUB_REPO="${2}"
- echo "Dockerhub repo ${DOCKERHUB_REPO}"
- echo
- shift 2
- ;;
- -f | --forward-credentials)
- echo "Forwarding credentials. Be careful as your credentials ar available in the container!"
- echo
- export FORWARD_CREDENTIALS="true"
- shift 1
- ;;
- -c | --github-registry)
- echo
- echo "Use github registry"
- echo
- export USE_GITHUB_REGISTRY="true"
- shift
- ;;
- -g | --github-repository)
- echo
- echo "GitHub repository: ${2}"
- echo
- echo "Using github registry."
- echo
- export GITHUB_REPOSITORY="${2}"
- export USE_GITHUB_REGISTRY="true"
- shift 2
- ;;
- -s | --github-image-id)
- echo
- echo "GitHub image id: ${2}}"
- echo
- echo "Force pulling the image, using github registry and skip mounting local sources."
- echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!."
- echo
- export FORCE_PULL_IMAGES="true"
- export USE_GITHUB_REGISTRY="true"
- export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}"
- export GITHUB_REGISTRY_PUSH_IMAGE_TAG="${2}"
- export CHECK_IMAGE_FOR_REBUILD="false"
- export SKIP_BUILDING_PROD_IMAGE="true"
- export MOUNT_LOCAL_SOURCES="false"
- export SKIP_CHECK_REMOTE_IMAGE="true"
- shift 2
- ;;
- -S | --version-suffix-for-pypi)
- if [[ -n ${VERSION_SUFFIX_FOR_SVN=} ]]; then
- echo
- echo "You can only set one version suffix - either for PyPI or for SVN"
- echo
- exit 1
- fi
- export VERSION_SUFFIX_FOR_PYPI="${2}"
- echo "Version suffix for PyPI ${VERSION_SUFFIX_FOR_PYPI}"
- echo
- shift 2
- ;;
- -N | --version-suffix-for-svn)
- if [[ -n ${VERSION_SUFFIX_FOR_PYPI=} ]]; then
- echo
- echo "You can only set one version suffix - either for PyPI or for SVN"
- echo
- exit 1
- fi
- export VERSION_SUFFIX_FOR_SVN="${2}"
- echo "Version suffix for SVN ${VERSION_SUFFIX_FOR_SVN}"
- echo
- shift 2
- ;;
+ export ADDITIONAL_RUNTIME_DEPS="${2}"
+ echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}"
+ shift 2 ;;
+ -D|--dockerhub-user)
+ export DOCKERHUB_USER="${2}"
+ echo "Dockerhub user ${DOCKERHUB_USER}"
+ echo
+ shift 2 ;;
+ -R|--dockerhub-repo)
+ export DOCKERHUB_REPO="${2}"
+ echo "Dockerhub repo ${DOCKERHUB_REPO}"
+ echo
+ shift 2 ;;
+ -f|--forward-credentials)
+ echo "Forwarding credentials. Be careful as your credentials ar available in the container!"
+ echo
+ export FORWARD_CREDENTIALS="true"
+ shift 1 ;;
+ -c|--github-registry)
+ echo
+ echo "Use github registry"
+ echo
+ export USE_GITHUB_REGISTRY="true"
+ shift ;;
+ -g|--github-repository)
+ echo
+ echo "GitHub repository: ${2}"
+ echo
+ echo "Using github registry."
+ echo
+ export GITHUB_REPOSITORY="${2}"
+ export USE_GITHUB_REGISTRY="true"
+ shift 2;;
+ -s|--github-image-id)
+ echo
+ echo "GitHub image id: ${2}}"
+ echo
+ echo "Force pulling the image, using github registry and skip mounting local sources."
+ echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!."
+ echo
+ export FORCE_PULL_IMAGES="true"
+ export USE_GITHUB_REGISTRY="true"
+ export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}"
+ export GITHUB_REGISTRY_PUSH_IMAGE_TAG="${2}"
+ export CHECK_IMAGE_FOR_REBUILD="false"
+ export SKIP_BUILDING_PROD_IMAGE="true"
+ export MOUNT_LOCAL_SOURCES="false"
+ export SKIP_CHECK_REMOTE_IMAGE="true"
+ shift 2;;
+ -S|--version-suffix-for-pypi)
+ if [[ ${VERSION_SUFFIX_FOR_SVN} != "" ]]; then
+ echo
+ echo "You can only set one version suffix - either for PyPI or for SVN"
+ echo
+ exit 1
+ fi
+ export VERSION_SUFFIX_FOR_PYPI="${2}"
+ echo "Version suffix for PyPI ${VERSION_SUFFIX_FOR_PYPI}"
+ echo
+ shift 2 ;;
+ -N|--version-suffix-for-svn)
+ if [[ ${VERSION_SUFFIX_FOR_PYPI} != "" ]]; then
+ echo
+ echo "You can only set one version suffix - either for PyPI or for SVN"
+ echo
+ exit 1
+ fi
+ export VERSION_SUFFIX_FOR_SVN="${2}"
+ echo "Version suffix for SVN ${VERSION_SUFFIX_FOR_SVN}"
+ echo
+ shift 2 ;;
--)
- shift
- break
- ;;
+ shift ;
+ break ;;
*)
- flags
- echo >&2
- echo >&2 "ERROR: Unknown flag ${1}"
- echo >&2
- exit 1
- ;;
- esac
+ flags
+ echo >&2
+ echo >&2 "ERROR: Unknown flag ${1}"
+ echo >&2
+ exit 1
+ ;;
+ esac
done
- local last_subcommand=""
# Parse commands
if [[ "$#" -ne 0 ]]; then
- case "${1}" in
+ case "${1}" in
shell)
- last_subcommand="${1}"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ shift ;;
exec)
- last_subcommand="${1}"
- command_to_run="run_exec"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="run_exec"
+ shift ;;
build-docs)
- last_subcommand="${1}"
- command_to_run="build_docs"
- shift 1
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="build_docs"
+ shift 1 ;;
build-image)
- last_subcommand="${1}"
- command_to_run="build_image"
- # if you want to build an image - assume you want to build it :)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- # and assume you want to build it no matter if it is needed
- export FORCE_BUILD_IMAGES="true"
- echo "Build image"
- echo
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="build_image"
+ # if you want to build an image - assume you want to build it :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ # and assume you want to build it no matter if it is needed
+ export FORCE_BUILD_IMAGES="true"
+ echo "Build image"
+ echo
+ shift ;;
cleanup-image)
- last_subcommand="${1}"
- echo "Cleanup the image"
- echo
- command_to_run="cleanup_image"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ echo "Cleanup the image"
+ echo
+ COMMAND_TO_RUN="cleanup_image"
+ shift ;;
docker-compose)
- last_subcommand="${1}"
- if [[ $# -lt 2 ]]; then
- echo "You should specify docker compose command to run"
- shift
- run_help="true"
- else
- docker_compose_command="${2}"
- shift 2
- fi
- command_to_run="run_docker_compose"
- ;;
- generate-constraints)
- last_subcommand="${1}"
- command_to_run="perform_generate_constraints"
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- export FORCE_BUILD_IMAGES="true"
- export UPGRADE_TO_LATEST_CONSTRAINTS="true"
+ LAST_SUBCOMMAND="${1}"
+ if [[ $# -lt 2 ]]; then
+ echo "You should specify docker compose command to run"
shift
- ;;
+ RUN_HELP="true"
+ else
+ DOCKER_COMPOSE_COMMAND="${2}"
+ shift 2
+ fi
+ COMMAND_TO_RUN="run_docker_compose"
+ ;;
+ generate-constraints)
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_generate_constraints"
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ export FORCE_BUILD_IMAGES="true"
+ export UPGRADE_TO_LATEST_CONSTRAINTS="true"
+ shift ;;
prepare-backport-packages)
- last_subcommand="${1}"
- command_to_run="perform_prepare_backport_packages"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_prepare_backport_packages"
+ shift ;;
prepare-backport-readme)
- last_subcommand="${1}"
- command_to_run="perform_prepare_backport_readme"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_prepare_backport_readme"
+ shift ;;
push-image)
- last_subcommand="${1}"
- command_to_run="perform_push_image"
- export SKIP_CHECK_REMOTE_IMAGE="true"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_push_image"
+ export SKIP_CHECK_REMOTE_IMAGE="true"
+ shift ;;
initialize-local-virtualenv)
- last_subcommand="${1}"
- echo "Initializing local virtualenv"
- echo
- command_to_run="perform_initialize_local_virtualenv"
- shift
- ;;
- kind-cluster)
- last_subcommand="${1}"
- # Switch to production image for all kind operations
- export PRODUCTION_IMAGE="true"
- command_to_run="manage_kind_cluster"
- export KIND_CLUSTER_OPERATION="${2:-}"
- if [[ -n ${KIND_CLUSTER_OPERATION=} ]]; then
- shift 2
- else
- shift
- fi
- ;;
- setup-autocomplete)
- last_subcommand="${1}"
- echo "Setting up autocomplete"
- echo
- command_to_run="perform_setup_autocomplete"
- shift
- ;;
- static-check)
- last_subcommand="${1}"
- command_to_run="perform_static_checks"
- if [[ "$#" -lt 2 ]]; then
- if [[ ${run_help} != "true" ]]; then
- echo "You should specify static check that you would like to run or 'all' to run all checks."
- echo
- echo "One of :"
- echo
- echo "${_BREEZE_ALLOWED_STATIC_CHECKS:=}"
- echo
- echo "For example:"
- echo
- echo "${CMDNAME} static-check mypy"
- echo
- exit 1
- else
- shift
- fi
+ LAST_SUBCOMMAND="${1}"
+ echo "Initializing local virtualenv"
+ echo
+ COMMAND_TO_RUN="perform_initialize_local_virtualenv"
+ shift ;;
+ kind-cluster)
+ LAST_SUBCOMMAND="${1}"
+ # Force local cache strategy for all kind-cluster operations
+ # this helps to iterate with production images
+ DOCKER_CACHE="local"
+ COMMAND_TO_RUN="manage_kind_cluster"
+ export KIND_CLUSTER_OPERATION="${2:-}"
+ if [[ ${KIND_CLUSTER_OPERATION} != "" ]]; then
+ shift 2
+ else
+ shift
+ fi
+ ;;
+ setup-autocomplete)
+ LAST_SUBCOMMAND="${1}"
+ echo "Setting up autocomplete"
+ echo
+ COMMAND_TO_RUN="perform_setup_autocomplete"
+ shift ;;
+ static-check )
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_static_checks"
+ if [[ "$#" -lt 2 ]]; then
+ if [[ ${RUN_HELP} != "true" ]]; then
+ echo "You should specify static check that you would like to run or 'all' to run all checks."
+ echo
+ echo "One of :"
+ echo
+ echo "${_BREEZE_ALLOWED_STATIC_CHECKS:=}"
+ echo
+ echo "For example:"
+ echo
+ echo "${CMDNAME} static-check mypy"
+ echo
+ exit 1
else
- export PYTHON_MAJOR_MINOR_VERSION=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}
- static_check="${2:-}"
- EXTRA_STATIC_CHECK_OPTIONS+=("--show-diff-on-failure")
- shift 2
+ shift
fi
- ;;
+ else
+ export PYTHON_MAJOR_MINOR_VERSION=${STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION}
+ export STATIC_CHECK="${2:-}"
+ export STATIC_CHECK_ALL_FILES="false"
+ EXTRA_STATIC_CHECK_OPTIONS+=("--show-diff-on-failure")
+ shift 2
+ fi
+ ;;
stop)
- last_subcommand="${1}"
- command_to_run="run_docker_compose"
- docker_compose_command="down"
- EXTRA_DC_OPTIONS+=("--remove-orphans")
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="run_docker_compose"
+ DOCKER_COMPOSE_COMMAND="down"
+ EXTRA_DC_OPTIONS+=("--remove-orphans")
+ shift ;;
restart)
- last_subcommand="${1}"
- command_to_run="run_docker_compose"
- docker_compose_command="down"
- EXTRA_DC_OPTIONS+=("--remove-orphans")
- second_command_to_run="enter_breeze"
- echo "Restarts the environment. Includes emptying the databases."
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="run_docker_compose"
+ DOCKER_COMPOSE_COMMAND="down"
+ EXTRA_DC_OPTIONS+=("--remove-orphans")
+ SECOND_COMMAND_TO_RUN="enter_breeze"
+ echo "Restarts the environment. Includes emptying the databases."
+ shift ;;
tests)
- last_subcommand="${1}"
- if [[ $# -lt 2 ]]; then
- run_help="true"
- else
- shift
- fi
- command_to_run="run_tests"
- ;;
- toggle-suppress-cheatsheet)
- last_subcommand="${1}"
- if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
- rm -f "${SUPPRESS_CHEATSHEET_FILE}"
- else
- touch "${SUPPRESS_CHEATSHEET_FILE}"
- fi
- echo "Toggle suppress cheatsheet"
- echo
+ LAST_SUBCOMMAND="${1}"
+ if [[ $# -lt 2 ]]; then
+ RUN_HELP="true"
+ else
shift
- command_to_run="toggle_suppress_cheatsheet"
- ;;
+ fi
+ COMMAND_TO_RUN="run_tests" ;;
+ toggle-suppress-cheatsheet)
+ LAST_SUBCOMMAND="${1}"
+ if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
+ rm -f "${SUPPRESS_CHEATSHEET_FILE}"
+ else
+ touch "${SUPPRESS_CHEATSHEET_FILE}"
+ fi
+ echo "Toggle suppress cheatsheet"
+ echo
+ shift ;;
toggle-suppress-asciiart)
- last_subcommand="${1}"
- if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
- rm -f "${SUPPRESS_ASCIIART_FILE}"
- else
- touch "${SUPPRESS_ASCIIART_FILE}"
- fi
- echo "Toggle suppress asciiart"
- echo
- shift
- command_to_run="toggle_suppress_asciiart"
- ;;
+ LAST_SUBCOMMAND="${1}"
+ if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
+ rm -f "${SUPPRESS_ASCIIART_FILE}"
+ else
+ touch "${SUPPRESS_ASCIIART_FILE}"
+ fi
+ echo "Toggle suppress asciiart"
+ echo
+ shift ;;
flags)
- flags
- exit 0
- ;;
+ flags
+ exit 0 ;;
help)
- usage
- exit 0
- ;;
+ usage
+ exit 0 ;;
help-all)
- do_help_all
- exit 0
- ;;
+ do_help_all
+ exit 0 ;;
*)
- usage
- echo >&2
- echo >&2 "ERROR: Unknown command ${1}"
- echo >&2
- exit 1
- ;;
- esac
+ usage
+ echo >&2
+ echo >&2 "ERROR: Unknown command ${1}"
+ echo >&2
+ exit 1
+ ;;
+ esac
else
- :
- # By default, start interactive terminal
+ :
+ # By default, start interactive terminal
fi
- if [[ ${run_help} == "true" ]]; then
- if [[ ${last_subcommand} == "" ]]; then
- usage
- flag_footer
- else
- detailed_usage "${last_subcommand}"
- fi
- exit 0
+ if [[ ${RUN_HELP} == "true" ]]; then
+ if [[ ${LAST_SUBCOMMAND} == "" ]]; then
+ usage
+ flag_footer
+ else
+ detailed_usage "${LAST_SUBCOMMAND}"
+ fi
+ exit 0
fi
-
- # EXTRA_DC_OPTIONS is only used by Breeze. It's value is set here as well.
- readonly EXTRA_DC_OPTIONS
-
- # Also Remaining args are set here and set as read-only - no more changes to it.
REMAINING_ARGS+=("$@")
- export REMAINING_ARGS
- readonly REMAINING_ARGS
}
-#######################################################################################################
-#
# Prepares nicely formatted versions of list of allowed and default values defined in Breeze.
# It is used in help command to print the lists in a readable format and fold the lists
-# so that they fit the screen indented_screen_width.
-#
-# Updated global constants:
-# FORMATTED_* constant variables that can be used in Breeze Help output
-#
-#######################################################################################################
+# so that they fit the screen width.
function prepare_formatted_versions() {
- local indent=15
- local list_prefix
- list_prefix=$(printf "%-${indent}s" " ")
- local indented_screen_width=$((SCREEN_WIDTH - indent))
-
- FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS=$(echo "${_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS
-
- FORMATTED_BACKENDS=$(echo "${_BREEZE_ALLOWED_BACKENDS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_BACKENDS
-
- FORMATTED_STATIC_CHECKS=$(echo "${_BREEZE_ALLOWED_STATIC_CHECKS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_STATIC_CHECKS
-
- FORMATTED_INTEGRATIONS=$(echo "${_BREEZE_ALLOWED_INTEGRATIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_INTEGRATIONS
-
- FORMATTED_KUBERNETES_MODES=$(echo "${_BREEZE_ALLOWED_KUBERNETES_MODES=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KUBERNETES_MODES
-
- FORMATTED_KUBERNETES_VERSIONS=$(echo "${_BREEZE_ALLOWED_KUBERNETES_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KUBERNETES_VERSIONS
-
- FORMATTED_KIND_VERSIONS=$(echo "${_BREEZE_ALLOWED_KIND_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KIND_VERSIONS
-
- FORMATTED_HELM_VERSIONS=$(echo "${_BREEZE_ALLOWED_HELM_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_HELM_VERSIONS
-
- FORMATTED_KIND_OPERATIONS=$(echo "${_BREEZE_ALLOWED_KIND_OPERATIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KIND_OPERATIONS
-
- FORMATTED_INSTALL_AIRFLOW_VERSIONS=$(echo "${_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_INSTALL_AIRFLOW_VERSIONS
-
- FORMATTED_POSTGRES_VERSIONS=$(echo "${_BREEZE_ALLOWED_POSTGRES_VERSIONS=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_POSTGRES_VERSIONS
-
- FORMATTED_MYSQL_VERSIONS=$(echo "${_BREEZE_ALLOWED_MYSQL_VERSIONS=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_MYSQL_VERSIONS
-
- FORMATTED_DEFAULT_CI_EXTRAS=$(echo "${DEFAULT_CI_EXTRAS=}" |
- tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
- readonly FORMATTED_DEFAULT_CI_EXTRAS
-
- FORMATTED_DEFAULT_PROD_EXTRAS=$(echo "${DEFAULT_PROD_EXTRAS=}" |
- tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
- readonly FORMATTED_DEFAULT_PROD_EXTRAS
+ INDENT=15
+ LIST_PREFIX=$(printf "%-${INDENT}s" " ")
+ WIDTH=$((SCREEN_WIDTH - INDENT))
+ FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS=$(echo "${_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_BACKENDS=$(echo "${_BREEZE_ALLOWED_BACKENDS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_STATIC_CHECKS=$(echo "${_BREEZE_ALLOWED_STATIC_CHECKS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_INTEGRATIONS=$(echo "${_BREEZE_ALLOWED_INTEGRATIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KUBERNETES_MODES=$(echo "${_BREEZE_ALLOWED_KUBERNETES_MODES=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KUBERNETES_VERSIONS=$(echo "${_BREEZE_ALLOWED_KUBERNETES_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KIND_VERSIONS=$(echo "${_BREEZE_ALLOWED_KIND_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_HELM_VERSIONS=$(echo "${_BREEZE_ALLOWED_HELM_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KIND_OPERATIONS=$(echo "${_BREEZE_ALLOWED_KIND_OPERATIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_INSTALL_AIRFLOW_VERSIONS=$(echo "${_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=""}" | \
+ tr '\n' ' ' | fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_POSTGRES_VERSIONS=$(echo "${_BREEZE_ALLOWED_POSTGRES_VERSIONS=""}" | \
+ tr '\n' ' ' | fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_MYSQL_VERSIONS=$(echo "${_BREEZE_ALLOWED_MYSQL_VERSIONS=""}" | \
+ tr '\n' ' ' | fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_DEFAULT_CI_EXTRAS=$(echo "${DEFAULT_CI_EXTRAS=}" | \
+ tr ',' ' ' | fold -w "${WIDTH}" -s | sed "s/ /,/g; s/^/${LIST_PREFIX}/")
+ FORMATTED_DEFAULT_PROD_EXTRAS=$(echo "${DEFAULT_PROD_EXTRAS=}" | \
+ tr ',' ' ' | fold -w "${WIDTH}" -s | sed "s/ /,/g; s/^/${LIST_PREFIX}/")
}
-#######################################################################################################
-#
# Prepares usage information for all the commands in Breeze.
# Those usage commands are stored in appropriate environment variables.
-#
-# Created constants:
-# USAGE_*
-# DETAILED_USAGE_*
-#
-#######################################################################################################
-
-# shellcheck disable=SC2034,SC2090,SC2089,SC2155
-
function prepare_usage() {
# Note that MacOS uses Bash 3.* and we cannot use associative arrays
export USAGE_SHELL="[Default] Enters interactive shell in the container"
- readonly USAGE_SHELL
export USAGE_EXEC="Execs into running breeze container in new terminal"
- readonly USAGE_EXEC
export USAGE_BUILD_DOCS="Builds documentation in the container"
- readonly USAGE_BUILD_DOCS
export USAGE_BUILD_IMAGE="Builds CI or Production docker image"
- readonly USAGE_BUILD_DOCS
export USAGE_CLEANUP_IMAGE="Cleans up the container image created"
- readonly USAGE_BUILD_DOCS
export USAGE_DOCKER_COMPOSE="Executes specified docker-compose command"
- readonly USAGE_DOCKER_COMPOSE
export USAGE_FLAGS="Shows all breeze's flags"
- readonly USAGE_FLAGS
export USAGE_GENERATE_CONSTRAINTS="Generates pinned constraint files"
- readonly USAGE_GENERATE_CONSTRAINTS
export USAGE_INITIALIZE_LOCAL_VIRTUALENV="Initializes local virtualenv"
- readonly USAGE_INITIALIZE_LOCAL_VIRTUALENV
export USAGE_PREPARE_BACKPORT_README="Prepares backport packages readme files"
- readonly USAGE_PREPARE_BACKPORT_README
export USAGE_PREPARE_BACKPORT_PACKAGES="Prepares backport packages"
- readonly USAGE_PREPARE_BACKPORT_PACKAGES
export USAGE_PUSH_IMAGE="Pushes images to registry"
- readonly USAGE_PUSH_IMAGE
export USAGE_KIND_CLUSTER="Manages KinD cluster on the host"
- readonly USAGE_KIND_CLUSTER
export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze"
- readonly USAGE_SETUP_AUTOCOMPLETE
export USAGE_STOP="Stops the docker-compose environment"
- readonly USAGE_STOP
export USAGE_RESTART="Stops the docker-compose environment including DB cleanup"
- readonly USAGE_RESTART
export USAGE_STATIC_CHECK="Performs selected static check for changed files"
- readonly USAGE_STATIC_CHECK
+ export USAGE_STATIC_CHECK_ALL_FILES="Performs selected static check for all files"
export USAGE_TOGGLE_SUPPRESS_CHEATSHEET="Toggles on/off cheatsheet"
- readonly USAGE_TOGGLE_SUPPRESS_CHEATSHEET
export USAGE_TOGGLE_SUPPRESS_ASCIIART="Toggles on/off asciiart"
- readonly USAGE_TOGGLE_SUPPRESS_ASCIIART
export USAGE_TESTS="Runs selected tests in the container"
- readonly USAGE_TESTS
export USAGE_HELP="Shows this help message"
- readonly USAGE_HELP
export USAGE_HELP_ALL="Shows detailed help for all commands and flags"
- readonly USAGE_HELP_ALL
- export DETAILED_USAGE_SHELL="
+
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_SHELL="
${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
This is default subcommand if no subcommand is used.
@@ -1399,7 +1154,8 @@ ${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
Flags:
$(flag_footer)
"
- readonly DETAILED_USAGE_SHELL
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_SHELL
export DETAILED_USAGE_EXEC="
${CMDNAME} exec [-- <EXTRA_ARGS>]
@@ -1419,8 +1175,8 @@ ${CMDNAME} build-docs [-- <EXTRA_ARGS>]
The possible extra args are: --docs-only, --spellcheck-only, --help
"
- readonly DETAILED_USAGE_BUILD_DOCS
- export DETAILED_USAGE_BUILD_IMAGE="
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_BUILD_IMAGE="
${CMDNAME} build-image [FLAGS]
Builds docker image (CI or production) without entering the container. You can pass
@@ -1444,8 +1200,10 @@ $(flag_build_docker_images)
$(flag_pull_push_docker_images)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_BUILD_IMAGE
- export DETAILED_USAGE_CLEANUP_IMAGE="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_BUILD_IMAGE
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_CLEANUP_IMAGE="
${CMDNAME} cleanup-image [FLAGS]
Removes the breeze-related images created in your local docker image cache. This will
@@ -1457,8 +1215,10 @@ $(flag_airflow_variants)
$(flag_production_image)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_CLEANUP_IMAGE
- export DETAILED_USAGE_DOCKER_COMPOSE="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_CLEANUP_IMAGE
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_DOCKER_COMPOSE="
${CMDNAME} docker-compose [FLAGS] COMMAND [-- <EXTRA_ARGS>]
Run docker-compose command instead of entering the environment. Use 'help' as command
@@ -1472,12 +1232,13 @@ $(flag_airflow_variants)
$(flag_backend_variants)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_DOCKER_COMPOSE
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_DOCKER_COMPOSE
export DETAILED_USAGE_FLAGS="
Explains in detail all the flags that can be used with breeze.
"
- readonly DETAILED_USAGE_FLAGS
- export DETAILED_USAGE_PREPARE_BACKPORT_README="
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PREPARE_BACKPORT_README="
${CMDNAME} prepare-backport-packages [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
Prepares README.md files for backport packages. You can provide (after --) optional version
@@ -1507,8 +1268,9 @@ ${CMDNAME} prepare-backport-packages [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
Flags:
$(flag_verbosity)
"
- readonly DETAILED_USAGE_PREPARE_BACKPORT_README
- export DETAILED_USAGE_GENERATE_CONSTRAINTS="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PREPARE_BACKPORT_README
+ DETAILED_USAGE_GENERATE_CONSTRAINTS="
${CMDNAME} generate-constraints [FLAGS]
Generates pinned constraint files from setup.py. Those files are generated in files folder
@@ -1523,8 +1285,9 @@ Flags:
$(flag_airflow_variants)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_GENERATE_CONSTRAINTS
- export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_GENERATE_CONSTRAINTS
+ DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV="
${CMDNAME} initialize-local-virtualenv [FLAGS]
Initializes locally created virtualenv installing all dependencies of Airflow
@@ -1536,8 +1299,10 @@ ${CMDNAME} initialize-local-virtualenv [FLAGS]
Flags:
$(flag_airflow_variants)
"
- readonly DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV
- export DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES="
${CMDNAME} prepare-backport-packages [FLAGS] [PACKAGE_ID ...]
Prepares backport packages. You can provide (after --) optional list of packages to prepare.
@@ -1565,8 +1330,10 @@ Flags:
$(flag_version_suffix)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES
- export DETAILED_USAGE_PUSH_IMAGE="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PUSH_IMAGE="
${CMDNAME} push_image [FLAGS]
Pushes images to docker registry. You can push the images to DockerHub registry (default)
@@ -1598,8 +1365,9 @@ Flags:
$(flag_pull_push_docker_images)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_PUSH_IMAGE
- export DETAILED_USAGE_KIND_CLUSTER="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PUSH_IMAGE
+ DETAILED_USAGE_KIND_CLUSTER="
${CMDNAME} kind-cluster [FLAGS] OPERATION
Manages host-side Kind Kubernetes cluster that is used to run Kubernetes integration tests.
@@ -1615,7 +1383,7 @@ Flags:
$(flag_airflow_variants)
$(flag_build_docker_images)
"
- readonly DETAILED_USAGE_KIND_CLUSTER
+ export DETAILED_USAGE_KIND_CLUSTER
export DETAILED_USAGE_SETUP_AUTOCOMPLETE="
${CMDNAME} setup-autocomplete
@@ -1623,7 +1391,6 @@ ${CMDNAME} setup-autocomplete
shell and when typing breeze command <TAB> will provide autocomplete for
parameters and values.
"
- readonly DETAILED_USAGE_SETUP_AUTOCOMPLETE
export DETAILED_USAGE_STOP="
${CMDNAME} stop
@@ -1631,8 +1398,7 @@ ${CMDNAME} stop
containers will continue running so that startup time is shorter. But they take quite a lot of
memory and CPU. This command stops all running containers from the environment.
"
- readonly DETAILED_USAGE_STOP
- export DETAILED_USAGE_RESTART="
+ DETAILED_USAGE_RESTART="
${CMDNAME} restart [FLAGS]
Restarts running docker compose environment. When you restart the environment, the docker
@@ -1642,9 +1408,9 @@ ${CMDNAME} restart [FLAGS]
Flags:
$(flag_footer)
"
- readonly DETAILED_USAGE_RESTART
+ export DETAILED_USAGE_RESTART
export DETAILED_USAGE_STATIC_CHECK="
-${CMDNAME} static-check [FLAGS] static_check [-- <EXTRA_ARGS>]
+${CMDNAME} static-check [FLAGS] STATIC_CHECK [-- <EXTRA_ARGS>]
Run selected static checks for currently changed files. You should specify static check that
you would like to run or 'all' to run all checks. One of:
@@ -1662,8 +1428,8 @@ ${FORMATTED_STATIC_CHECKS}
'${CMDNAME} static-check mypy -- --help'
"
- readonly DETAILED_USAGE_STATIC_CHECK
- export DETAILED_USAGE_TESTS="
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_TESTS="
${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
Run the specified unit test target. There might be multiple
@@ -1677,167 +1443,104 @@ ${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
Flags:
$(flag_footer)
"
- readonly DETAILED_USAGE_TESTS
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_TESTS
export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET="
${CMDNAME} toggle-suppress-cheatsheet
Toggles on/off cheatsheet displayed before starting bash shell.
"
- readonly DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET
export DETAILED_USAGE_TOGGLE_SUPPRESS_ASCIIART="
${CMDNAME} toggle-suppress-asciiart
Toggles on/off asciiart displayed before starting bash shell.
"
- readonly DETAILED_USAGE_TOGGLE_SUPPRESS_ASCIIART
export DETAILED_USAGE_HELP="
${CMDNAME} help
Shows general help message for all commands.
"
- readonly DETAILED_USAGE_HELP
export DETAILED_USAGE_HELP_ALL="
${CMDNAME} help-all
Shows detailed help for all commands and flags.
"
- readonly DETAILED_USAGE_HELP_ALL
}
-# shellcheck enable=all
-
-#######################################################################################################
-#
# Gets environment variable value converting the lowercase name of command into variable name
-# Arguments:
-# prefix for the variable to add
-# name of the variable
-# Outputs:
-# Writes the capitalized name of the variable to stdout
-#######################################################################################################
function get_variable_from_lowercase_name() {
- local prefix="${1}"
- local name="${2}"
- local suffix
- suffix="$(echo "${name}" | tr "[:lower:]-" "[:upper:]_")"
- local variable_name="${prefix}_${suffix}"
- echo "${!variable_name}"
+ PREFIX="${1}"
+ NAME="${2}"
+ SUFFIX="$(echo "${NAME}" | tr "[:lower:]-" "[:upper:]_")"
+ VARIABLE_NAME="${PREFIX}_${SUFFIX}"
+ echo "${!VARIABLE_NAME}"
}
-#######################################################################################################
-#
# Gets usage information from lowercase command
-# Arguments:
-# lowercase command name
-# Outputs:
-# Usage information for the command.
-#######################################################################################################
function get_usage() {
- get_variable_from_lowercase_name "USAGE" "${1}"
+ get_variable_from_lowercase_name USAGE "${1}"
}
-#######################################################################################################
-#
+
# Gets detailed usage information from lowercase command
-# Arguments:
-# lowercase command name
-# Outputs:
-# Detailed usage information for the command.
-#######################################################################################################
function get_detailed_usage() {
- get_variable_from_lowercase_name "DETAILED_USAGE" "${1}"
+ get_variable_from_lowercase_name DETAILED_USAGE "${1}"
}
-#######################################################################################################
-#
-# Prints general usage information for all commands.
-#
-# Globals used:
-# CMDNAME
-# BREEZE_COMMANDS
-# BREEZE_EXTRA_ARG_COMMANDS
-# BREEZE_HELP_COMMANDS
-#
-# Outputs:
-# General usage information for all commands.
-#######################################################################################################
+# Prints general usage information
function usage() {
- echo "
+ echo "
Usage: ${CMDNAME} [FLAGS] [COMMAND] -- <EXTRA_ARGS>
-By default the script enters the CI container and drops you to bash shell, but you can choose
-one of the commands to run specific actions instead.
-
-Add --help after each command to see details:
+By default the script enters IT environment and drops you to bash shell, but you can choose one
+of the commands to run specific actions instead. Add --help after each command to see details:
Commands without arguments:
"
- for subcommand in ${BREEZE_COMMANDS}; do
- printf " %-40s %s\n" "${subcommand}" "$(get_usage "${subcommand}")"
- done
- echo "
+ for SUBCOMMAND in ${BREEZE_COMMANDS}
+ do
+ printf " %-40s %s\n" "${SUBCOMMAND}" "$(get_usage "${SUBCOMMAND}")"
+ done
+ echo "
Commands with arguments:
"
- for subcommand in ${BREEZE_EXTRA_ARG_COMMANDS}; do
- printf " %-30s%-10s %s\n" "${subcommand}" "<ARG>" "$(get_usage "${subcommand}")"
- done
- echo "
+ for SUBCOMMAND in ${BREEZE_EXTRA_ARG_COMMANDS}
+ do
+ printf " %-30s%-10s %s\n" "${SUBCOMMAND}" "<ARG>" "$(get_usage "${SUBCOMMAND}")"
+ done
+ echo "
Help commands:
"
- for subcommand in ${BREEZE_HELP_COMMANDS}; do
- printf " %-40s %s\n" "${subcommand}" "$(get_usage "${subcommand}")"
- done
- echo
+ for SUBCOMMAND in ${BREEZE_HELP_COMMANDS}
+ do
+ printf " %-40s %s\n" "${SUBCOMMAND}" "$(get_usage "${SUBCOMMAND}")"
+ done
+ echo
}
-#######################################################################################################
-#
-# Prints detailed usage for command specified
-#
-# Argument:
-# Command name.
-#
-# Outputs:
-# Detailed usage information for the command
-#######################################################################################################
# Prints detailed usage for command specified
function detailed_usage() {
- subcommand=${1}
- echo "
+ SUBCOMMAND=${1}
+ echo "
-Detailed usage for command: ${subcommand}
+Detailed usage for command: ${SUBCOMMAND}
-$(get_detailed_usage "${subcommand}")
+$(get_detailed_usage "${SUBCOMMAND}")
"
}
-#######################################################################################################
-#
-# Prints flag footer - common to all commands..
-#
-# Outputs:
-# Footer common for all commands.
-#######################################################################################################
+# Prints flag footer
function flag_footer() {
- echo "
+ echo "
Run '${CMDNAME} flags' to see all applicable flags.
"
}
-#######################################################################################################
-#
# Prints flags for different variants of airflow to use
-#
-# Global constants used:
-# FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_airflow_variants() {
- echo "
+ echo "
-p, --python <PYTHON_MAJOR_MINOR_VERSION>
Python version used for the image. This is always major/minor version.
@@ -1850,20 +1553,9 @@ ${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints flags for different backend to use
-#
-# Global constants used:
-# FORMATTED_BACKENDS
-# FORMATTED_POSTGRES_VERSIONS
-# FORMATTED_MYSQL_VERSIONS
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_backend_variants() {
- echo "
+ echo "
-b, --backend <BACKEND>
Backend to use for tests - it determines which database is used.
One of:
@@ -1884,33 +1576,17 @@ ${FORMATTED_MYSQL_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints production image flags
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_production_image() {
- echo "
+ echo "
-I, --production-image
Use production image for entering the environment and builds (not for tests).
"
}
-#######################################################################################################
-#
# Prints additional breeze action flags
-#
-# Global constants used:
-# FORMATTED_INTEGRATIONS
-#
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_breeze_actions() {
- echo "
+ echo "
-d, --db-reset
Resets the database at entry to the environment. It will drop all the tables
and data and recreate the DB from scratch even if 'restart' command was not used.
@@ -1928,22 +1604,9 @@ ${FORMATTED_INTEGRATIONS}
"
}
-#######################################################################################################
-#
# Prints Kubernetes action flags
-#
-# Global constants used:
-# FORMATTED_KUBERNETES_MODE
-# FORMATTED_KUBERNETES_VERSIONS
-# FORMATTED_KIND_VERSIONS
-# FORMATTED_HELM_VERSIONS
-#
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_kubernetes_configuration() {
- echo "
+ echo "
Configuration for the KinD Kubernetes cluster and tests:
-K, --kubernetes-mode <KUBERNETES_MODE>
@@ -1981,34 +1644,18 @@ ${FORMATTED_HELM_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints flags that determine what is the source mounting scheme
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_local_file_mounting() {
- echo "
+ echo "
-l, --skip-mounting-local-sources
Skips mounting local volume with sources - you get exactly what is in the
docker image rather than your current local sources of Airflow.
"
}
-#######################################################################################################
-#
# Prints flags that allow to choose different airflow variants
-#
-# Global constants used:
-# FORMATTED_INSTALL_AIRFLOW_VERSIONS
-#
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_choose_different_airflow_version() {
- echo "
+ echo "
-a, --install-airflow-version <INSTALL_AIRFLOW_VERSION>
If specified, installs Airflow directly from PIP released version. This happens at
image building time in production image and at container entering time for CI image. One of:
@@ -2021,15 +1668,9 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints flags that allow to set assumed answers to questions
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_assume_answers_to_questions() {
- echo "
+ echo "
-y, --assume-yes
Assume 'yes' answer to all questions.
@@ -2041,30 +1682,18 @@ function flag_assume_answers_to_questions() {
"
}
-#######################################################################################################
-#
# Prints flags that are used for credential forwarding
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_credentials() {
- echo "
+ echo "
-f, --forward-credentials
Forwards host credentials to docker container. Use with care as it will make
your credentials available to everything you install in Docker.
"
}
-#######################################################################################################
-#
# Prints flags that control verbosity
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_verbosity() {
- echo "
+ echo "
-v, --verbose
Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
debugging - when you run breeze with --verbose flags you will be able to see the commands
@@ -2075,33 +1704,16 @@ function flag_verbosity() {
"
}
-#######################################################################################################
-#
-# Prints information about help flag
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_help() {
- echo "
+ echo "
-h, --help
Shows detailed help message for the command specified.
"
}
-#######################################################################################################
-#
# Prints flags controlling docker build process
-#
-# Global constants used:
-# FORMATTED_DEFAULT_CI_EXTRAS
-# FORMATTED_DEFAULT_PROD_EXTRAS
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_build_docker_images() {
- echo "
+ echo "
-F, --force-build-images
Forces building of the local docker images. The images are rebuilt
automatically for the first time or when changes are detected in
@@ -2170,21 +1782,9 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
"
}
-#######################################################################################################
-#
# Prints flags controlling docker pull and push process
-#
-# Global constants used:
-# _BREEZE_DEFAULT_DOCKERHUB_USER
-# _BREEZE_DEFAULT_DOCKERHUB_REPO
-# _BREEZE_DEFAULT_GITHUB_REPOSITORY
-# _BREEZE_DEFAULT_GITHUB_IMAGE_ID
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_pull_push_docker_images() {
- echo "
+ echo "
-D, --dockerhub-user
DockerHub user used to pull, push and build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_USER:=}.
@@ -2216,15 +1816,9 @@ function flag_pull_push_docker_images() {
"
}
-#######################################################################################################
-#
# Prints flags that control version of generated packages
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_version_suffix() {
- echo "
+ echo "
-S, --version-suffix-for-pypi
Adds optional suffix to the version in the generated backport package. It can be used
to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI.
@@ -2235,15 +1829,11 @@ function flag_version_suffix() {
"
}
-#######################################################################################################
-#
+
+
# Prints all flags
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flags() {
- echo "
+ echo "
$(print_line)
Summary of all flags supported by Breeze:
@@ -2306,13 +1896,7 @@ $(flag_help)
"
}
-#######################################################################################################
-#
-# Prints header line filling screen indented_screen_width - only when VERBOSE is set
-#
-# Outputs:
-# Prints header line.
-#######################################################################################################
+# Prints header line filling screen width - only when VERBOSE is set
function print_header_line() {
if [ ${VERBOSE:="false"} == "true" ]; then
echo
@@ -2321,121 +1905,68 @@ function print_header_line() {
fi
}
-#######################################################################################################
-#
-# Prints separation line filling screen indented_screen_width
-#
-# Outputs:
-# Prints line.
-#######################################################################################################
-function print_line() {
+
+# Prints separation line filling screen width
+function print_line {
printf '#%.0s' $(seq "${SCREEN_WIDTH}")
}
-# Prints star line filling screen indented_screen_width
-function print_star_line() {
+# Prints star line filling screen width
+function print_star_line {
printf '*%.0s' $(seq "${SCREEN_WIDTH}")
}
-#######################################################################################################
-#
-# Reads saved environment variables. Some of the variables are stored across session so that once
-# you use them you do not have to use it next time. That makes those flags persistent.
-#
-# An example of it is `--backend` or '--kubernetes-mode' flags.
-#
-# Note that PYTHON_MAJOR_MINOR_VERSION is not read here - it is read at the
-# `setup_default_breeze_variables` method because it is needed
-# to determine the right images to use and set several variables that depend on the Python version
-#
-# Uses:
-# _BREEZE_DEFAULT_* corresponding defaults for variables it reads
-#
-# Modified and used global constants:
-#
-# BACKEND
-# KUBERNETES_MODE
-# KUBERNETES_VERSION
-# KIND_VERSION
-# HELM_VERSION
-# POSTGRES_VERSION
-# MYSQL_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-#
-#######################################################################################################
-
-function read_saved_environment_variables() {
- BACKEND="${BACKEND:=$(read_from_file BACKEND)}"
- BACKEND=${BACKEND:-${_BREEZE_DEFAULT_BACKEND}}
+# Reads save environment variables. Some of the variables are stored across session so that once
+# you use them you do not have to use it next time. That makes those flags persistent
+# An example of it is `--backend` or '--kubernetes-mode' flags. Note that PYTHON_MAJOR_MINOR_VERSION is
+# not read here - it is read at the `setup_default_breeze_variables` method because it is needed
+# t determine the right images to use and set several variables that depend on the Python version
+function read_saved_environment_variables {
+ export BACKEND="${BACKEND:=$(read_from_file BACKEND)}"
+ export BACKEND=${BACKEND:-${_BREEZE_DEFAULT_BACKEND}}
- KUBERNETES_MODE="${KUBERNETES_MODE:=$(read_from_file KUBERNETES_MODE)}"
- KUBERNETES_MODE=${KUBERNETES_MODE:=${_BREEZE_DEFAULT_KUBERNETES_MODE}}
+ export KUBERNETES_MODE="${KUBERNETES_MODE:=$(read_from_file KUBERNETES_MODE)}"
+ export KUBERNETES_MODE=${KUBERNETES_MODE:=${_BREEZE_DEFAULT_KUBERNETES_MODE}}
- KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(read_from_file KUBERNETES_VERSION)}"
- KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_BREEZE_DEFAULT_KUBERNETES_VERSION}}
+ export KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(read_from_file KUBERNETES_VERSION)}"
+ export KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_BREEZE_DEFAULT_KUBERNETES_VERSION}}
- KIND_VERSION="${KIND_VERSION:=$(read_from_file KIND_VERSION)}"
- KIND_VERSION=${KIND_VERSION:=${_BREEZE_DEFAULT_KIND_VERSION}}
+ export KIND_VERSION="${KIND_VERSION:=$(read_from_file KIND_VERSION)}"
+ export KIND_VERSION=${KIND_VERSION:=${_BREEZE_DEFAULT_KIND_VERSION}}
- HELM_VERSION="${HELM_VERSION:=$(read_from_file HELM_VERSION)}"
- HELM_VERSION=${HELM_VERSION:=${_BREEZE_DEFAULT_HELM_VERSION}}
+ export HELM_VERSION="${HELM_VERSION:=$(read_from_file HELM_VERSION)}"
+ export HELM_VERSION=${HELM_VERSION:=${_BREEZE_DEFAULT_HELM_VERSION}}
- POSTGRES_VERSION="${POSTGRES_VERSION:=$(read_from_file POSTGRES_VERSION)}"
- POSTGRES_VERSION=${POSTGRES_VERSION:=${_BREEZE_DEFAULT_POSTGRES_VERSION}}
+ export POSTGRES_VERSION="${POSTGRES_VERSION:=$(read_from_file POSTGRES_VERSION)}"
+ export POSTGRES_VERSION=${POSTGRES_VERSION:=${_BREEZE_DEFAULT_POSTGRES_VERSION}}
- MYSQL_VERSION="${MYSQL_VERSION:=$(read_from_file MYSQL_VERSION)}"
- MYSQL_VERSION=${MYSQL_VERSION:=${_BREEZE_DEFAULT_MYSQL_VERSION}}
+ export MYSQL_VERSION="${MYSQL_VERSION:=$(read_from_file MYSQL_VERSION)}"
+ export MYSQL_VERSION=${MYSQL_VERSION:=${_BREEZE_DEFAULT_MYSQL_VERSION}}
# Here you read DockerHub user/account that you use
# You can populate your own images in DockerHub this way and work with the,
# You can override it with "--dockerhub-user" option and it will be stored in .build directory
- DOCKERHUB_USER="${DOCKERHUB_USER:=$(read_from_file DOCKERHUB_USER)}"
- DOCKERHUB_USER="${DOCKERHUB_USER:=${_BREEZE_DEFAULT_DOCKERHUB_USER}}"
+ export DOCKERHUB_USER="${DOCKERHUB_USER:=$(read_from_file DOCKERHUB_USER)}"
+ export DOCKERHUB_USER="${DOCKERHUB_USER:=${_BREEZE_DEFAULT_DOCKERHUB_USER}}"
# Here you read DockerHub repo that you use
# You can populate your own images in DockerHub this way and work with them
# You can override it with "--dockerhub-repo" option and it will be stored in .build directory
- DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(read_from_file DOCKERHUB_REPO)}"
- DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_BREEZE_DEFAULT_DOCKERHUB_REPO}}"
+ export DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(read_from_file DOCKERHUB_REPO)}"
+ export DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_BREEZE_DEFAULT_DOCKERHUB_REPO}}"
}
-#######################################################################################################
-#
# Checks if variables are correctly set and if they are - saves them so that they can be used across
-# sessions. In case we are installing Airflow 1.10, the constants are set to match 1.10 line.
-#
-# In case the variables are matching expected values they are saved in ".build/VARIABLE_NAME" for
-# later reuse. If not, error is printed and the saved file is cleaned, so that next time
-# default value can be used.
-#
-# Used Global constants:
-# INSTALL_AIRFLOW_REFERENCE
-# INSTALL_AIRFLOW_VERSION
-# BACKEND
-# KUBERNETES_MODE
-# KUBERNETES_VERSION
-# KIND_VERSION
-# HELM_VERSION
-# POSTGRES_VERSION
-# MYSQL_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-#
-# Updated Global constants:
-# BRANCH_NAME
-#
-# Output: saved variable files in .build,
-#######################################################################################################
+# sessions.
function check_and_save_all_params() {
- check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
+ check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
- if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
- if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *1_10* ]]; then
+ if [[ "${INSTALL_AIRFLOW_REFERENCE:=}" != "" ]]; then
+ if [[ ${INSTALL_AIRFLOW_REFERENCE} == *1_10* ]]; then
export BRANCH_NAME="v1-10-test"
fi
- elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
- if [[ ${INSTALL_AIRFLOW_VERSION=} == *1.10* ]]; then
+ elif [[ "${INSTALL_AIRFLOW_VERSION:=}" != "" ]]; then
+ if [[ ${INSTALL_AIRFLOW_VERSION} == *1.10* ]]; then
export BRANCH_NAME="v1-10-test"
fi
fi
@@ -2452,33 +1983,22 @@ function check_and_save_all_params() {
fi
fi
- check_and_save_allowed_param "BACKEND" "backend" "--backend"
- check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
- check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
- check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
- check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
- check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
- check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
+ check_and_save_allowed_param "BACKEND" "backend" "--backend"
+ check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
+ check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
+ check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
+ check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
+ check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
+ check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
# Can't verify those - they can be anything, so let's just save them
save_to_file DOCKERHUB_USER
save_to_file DOCKERHUB_REPO
}
-#######################################################################################################
-#
# Prints cheatsheet if it is not suppressed
-#
-# Used global constants:
-#
-# AIRFLOW_SOURCES
-# WEBSERVER_HOST_PORT
-# POSTGRES_HOST_PORT
-# MYSQL_HOST_PORT
-#
-#######################################################################################################
function print_cheatsheet() {
- if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${command_to_run} == "enter_breeze" ]]; then
+ if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${COMMAND_TO_RUN} == "enter_breeze" ]]; then
echo
echo
print_line
@@ -2517,18 +2037,12 @@ function print_cheatsheet() {
fi
}
-#######################################################################################################
-#
# Prints setup instruction in case we find that autocomplete is not set
# also prints how to toggle asciiart/cheatsheet
-#
-# Used global constants:
-# CMDNAME
-#######################################################################################################
-function print_setup_instructions() {
- if [[ ${command_to_run} == "enter_breeze" ]]; then
+function print_setup_instructions {
+ if [[ ${COMMAND_TO_RUN} == "enter_breeze" ]] ; then
# shellcheck disable=SC2034 # Unused variables left for comp_breeze usage
- if ! typeset -f "_comp_breeze" >/dev/null; then
+ if ! typeset -f "_comp_breeze" > /dev/null; then
print_line
echo
echo " You can setup autocomplete by running '${CMDNAME} setup-autocomplete'"
@@ -2549,360 +2063,237 @@ function print_setup_instructions() {
fi
}
-#######################################################################################################
-#
# Checks that pre-commit is installed and upgrades it if needed
-# this is used in case static check command is used.
-#
-#######################################################################################################
-function make_sure_precommit_is_installed() {
+# this is used in case static check command is used
+function make_sure_precommit_is_installed {
echo
echo "Making sure pre-commit is installed"
echo
- local pip_binary
if command -v pip3 >/dev/null; then
- pip_binary=pip3
+ PIP_BIN=pip3
elif command -v pip >/dev/null; then
- pip_binary=pip
+ PIP_BIN=pip
else
- echo >&2
- echo >&2 "ERROR: You need to have pip or pip3 in your PATH"
- echo >&2
- S
- exit 1
+ echo >&2
+ echo >&2 "ERROR: You need to have pip or pip3 in your PATH"
+ echo >&2
+ S
+ exit 1
fi
- "${pip_binary}" install --upgrade pre-commit >/dev/null 2>&1
+ "${PIP_BIN}" install --upgrade pre-commit >/dev/null 2>&1
# Add ~/.local/bin to the path in case pip is run outside of virtualenv
export PATH="${PATH}":~/.local/bin
}
-#######################################################################################################
-#
# Removes CI and PROD images and cleans up the flag that indicates that the image was already built
-#
-# Used global constants:
-# PYTHON_BASE_IMAGE
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# BUILT_CI_IMAGE_FLAG_FILE
-#
-#######################################################################################################
-function remove_images() {
- docker rmi "${PYTHON_BASE_IMAGE}" || true
- docker rmi "${AIRFLOW_CI_IMAGE}" || true
+function remove_images {
+ docker rmi "${PYTHON_BASE_IMAGE}" || true
+ docker rmi "${AIRFLOW_CI_IMAGE}" || true
docker rmi "${AIRFLOW_PROD_IMAGE}" || true
- rm -f "${BUILT_CI_IMAGE_FLAG_FILE}"
+ rm -f "${BUILT_IMAGE_FLAG_FILE}"
}
-#######################################################################################################
-#
# Runs chosen static checks
-#
-# Uses variables:
-# static_check
-#
-#######################################################################################################
-function run_static_checks() {
- if [[ ${static_check} == "all" ]]; then
+function run_static_checks {
+ if [[ ${STATIC_CHECK} == "all" ]]; then
echo
echo "Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
echo
pre-commit run "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
- elif [[ ${static_check} == "all-but-pylint" ]]; then
+ elif [[ ${STATIC_CHECK} == "all-but-pylint" ]]; then
echo
echo "Setting SKIP=pylint. Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
echo
echo
SKIP=pylint pre-commit run "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
else
- if [[ ${static_check} == *"pylint"* || ${static_check} == *"mypy"* || ${static_check} == *"flake8"* ]]; then
- echo
- echo "Running build pre-commit before running the requested static check"
- echo
- pre-commit run "build"
- fi
echo
- echo "Running: pre-commit run" "${static_check}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
+ echo "Running: pre-commit run" "${STATIC_CHECK}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
echo
- pre-commit run "${static_check}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
+ pre-commit run "${STATIC_CHECK}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
fi
}
-#######################################################################################################
-#
-# Runs Build before a command if needed - the right build command will be determined and used
-# depending on which command you are trying ton
-#
-# Used global constants:
-# PRODUCTION_IMAGE
-# KIND_CLUSTER_OPERATION
-# FORMATTED_KIND_OPERATIONS
-#
-# Used variables:
-# command_to_run
-#
-#######################################################################################################
-function run_build_command() {
- case "${command_to_run}" in
- run_tests | run_docker_compose)
- prepare_ci_build
- rebuild_ci_image_if_needed
- ;;
- enter_breeze)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- else
+# Runs Build before a command - it will check and if needed rebuild necessary image, depending on the
+# command chosen
+function run_build_command {
+ case "${COMMAND_TO_RUN}" in
+ run_tests|run_docker_compose)
prepare_ci_build
rebuild_ci_image_if_needed
- fi
- ;;
- build_docs | perform_static_checks | perform_generate_constraints | perform_prepare_backport_readme | perform_prepare_backport_packages)
- prepare_ci_build
- rebuild_ci_image_if_needed
- ;;
- perform_push_image)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- else
+ ;;
+ enter_breeze)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ else
+ prepare_ci_build
+ rebuild_ci_image_if_needed
+ fi
+ ;;
+ build_docs|perform_static_checks|perform_generate_constraints|perform_prepare_backport_readme|perform_prepare_backport_packages)
prepare_ci_build
rebuild_ci_image_if_needed
- fi
- ;;
- build_image)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- build_prod_images
- else
- prepare_ci_build
- calculate_md5sum_for_all_files
- build_ci_image
- update_all_md5_files
- build_ci_image_manifest
- fi
- ;;
- cleanup_image | run_exec)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- else
- prepare_ci_build
- fi
- ;;
- perform_initialize_local_virtualenv | perform_setup_autocomplete | toggle_suppress_cheatsheet | toggle_suppress_asciiart ) ;;
-
- manage_kind_cluster)
- if [[ ${KIND_CLUSTER_OPERATION} == "start" ]]; then
- echo "Starts KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]]; then
- echo "Stops KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "restart" ]]; then
- echo "Restarts KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "recreate" ]]; then
- echo "Recreates KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "status" ]]; then
- echo "Checks status of KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "deploy" ]]; then
- echo "Deploys Airflow to KinD cluster"
- prepare_prod_build
- build_prod_images
- elif [[ ${KIND_CLUSTER_OPERATION} == "test" ]]; then
- echo "Run Kubernetes tests with the KinD cluster "
- elif [[ ${KIND_CLUSTER_OPERATION} == "shell" ]]; then
- echo "Enter an interactive shell for kubernetes testing"
- elif [[ -z ${KIND_CLUSTER_OPERATION=} ]]; then
- echo
- echo "Please provide an operation to run"
- echo
- echo "Should be one of:"
- echo "${FORMATTED_KIND_OPERATIONS}"
- echo
- exit 1
- else
- echo
- echo "ERROR: Unknown Kind Kubernetes cluster operation: '${KIND_CLUSTER_OPERATION}'"
- echo
- echo "Should be one of:"
- echo "${FORMATTED_KIND_OPERATIONS}"
- echo
- exit 1
- fi
- ;;
- *)
- echo >&2
- echo >&2 "ERROR: Unknown command to run ${command_to_run}"
- echo >&2
- exit 1
- ;;
+ ;;
+ perform_push_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ else
+ prepare_ci_build
+ rebuild_ci_image_if_needed
+ fi
+ ;;
+ build_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ build_prod_images
+ else
+ prepare_ci_build
+ calculate_md5sum_for_all_files
+ build_ci_image
+ update_all_md5_files
+ build_ci_image_manifest
+ fi
+ ;;
+ cleanup_image|run_exec)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ else
+ prepare_ci_build
+ fi
+ ;;
+ perform_initialize_local_virtualenv|perform_setup_autocomplete)
+ ;;
+ manage_kind_cluster)
+ if [[ ${KIND_CLUSTER_OPERATION} == "start" ]] ; then
+ echo "Starts KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]] ; then
+ echo "Stops KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "restart" ]] ; then
+ echo "Restarts KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "recreate" ]] ; then
+ echo "Recreates KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "status" ]] ; then
+ echo "Checks status of KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "deploy" ]] ; then
+ echo "Deploys Airflow to KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "test" ]] ; then
+ echo "Run Kubernetes tests with the KinD cluster "
+ elif [[ ${KIND_CLUSTER_OPERATION} == "shell" ]] ; then
+ echo "Enter an interactive shell for kubernetes testing"
+ else
+ echo "ERROR: Unknown Kind Kubernetes cluster operation: '${KIND_CLUSTER_OPERATION}'"
+ echo
+ echo "Should be one of:"
+ echo "${FORMATTED_KIND_OPERATIONS}"
+ echo
+ exit 1
+ fi
+ ;;
+ *)
+ echo >&2
+ echo >&2 "ERROR: Unknown command to run ${COMMAND_TO_RUN}"
+ echo >&2
+ exit 1
+ ;;
esac
}
-#######################################################################################################
-#
# Runs the actual command - depending on the command chosen it will use the right
-# Convenient script and run the right command for the script.
-#
-# Used variables:
-# command_to_run
-#
-# Used global constants:
-# PRODUCTION_IMAGE
-# SCRIPTS_CI_DIR
-# BUILD_CACHE_DIR
-# KIND_CLUSTER_OPERATION
-# EXTRA_DC_OPTIONS
-#
-# Set Global variables:
-# RUN_TESTS
-#######################################################################################################
-function run_breeze_command() {
+# Convenient script and run the right command with it
+function run_breeze_command {
set +u
- local dc_run_file
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}"
- else
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
- fi
- case "${command_to_run}" in
- enter_breeze)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- "${dc_run_file}" run --service-ports --rm airflow "${@}"
- "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
- else
- "${dc_run_file}" run --service-ports --rm airflow "${@}"
- "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
- "${SCRIPTS_CI_DIR}/tools/ci_clear_tmp.sh"
- fi
- ;;
- run_exec)
- # Unfortunately `docker-compose exec` does not support exec'ing into containers started with run :(
- # so we have to find it manually
- set +e
- local airflow_testing_container
- airflow_testing_container=$("${dc_run_file}" ps | grep airflow | awk '{print $1}' 2>/dev/null)
- : "${airflow_testing_container:?"ERROR! Breeze must be running in order to exec into running container"}"
- set -e
- docker exec -it "${airflow_testing_container}" \
- "/opt/airflow/scripts/in_container/entrypoint_exec.sh" "${@}"
- ;;
- run_tests)
- export RUN_TESTS="true"
- readonly RUN_TESTS
- "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" run --service-ports --rm airflow "$@"
- ;;
- run_docker_compose)
- set +u
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}"
- else
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
- fi
- "${dc_run_file}" "${docker_compose_command}" "${EXTRA_DC_OPTIONS[@]}" "$@"
- set -u
- ;;
- perform_static_checks)
- make_sure_precommit_is_installed
- run_static_checks "${@}"
- ;;
- build_image) ;;
-
- cleanup_image)
- remove_images
- ;;
- perform_generate_constraints)
- run_generate_constraints
- ;;
- perform_prepare_backport_packages)
- run_prepare_backport_packages "${@}"
- ;;
- perform_prepare_backport_readme)
- run_prepare_backport_readme "${@}"
- ;;
- perform_push_image)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- push_prod_images
- else
- push_ci_images
- fi
- ;;
- perform_initialize_local_virtualenv)
- initialize_virtualenv
- ;;
- perform_setup_autocomplete)
- setup_autocomplete
- ;;
- manage_kind_cluster)
- make_sure_kubernetes_tools_are_installed
- get_kind_cluster_name
- perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
- ;;
- build_docs)
- run_docs "${@}"
- ;;
- toggle_suppress_cheatsheet)
- if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
- echo
- echo "Cheatsheet disabled"
- echo
- else
- echo
- echo "Cheatsheet enabled"
- echo
- fi
- ;;
- toggle_suppress_asciiart)
- if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
- echo
- echo "ASCIIart disabled"
- echo
- else
- echo
- echo "ASCIIart enabled"
- echo
- fi
- ;;
- *)
- echo >&2
- echo >&2 "ERROR: Unknown command to run ${command_to_run}"
- echo >&2
- ;;
+ case "${COMMAND_TO_RUN}" in
+ enter_breeze)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ "${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}" run --service-ports --rm airflow "${@}"
+ "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
+ else
+ "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" run --service-ports --rm airflow "${@}"
+ "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
+ "${SCRIPTS_CI_DIR}/tools/ci_clear_tmp.sh"
+ fi
+ ;;
+ run_exec)
+ # Unfortunately docker-compose exec does not support exec'ing into containers started with run :(
+ set +e
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}"
+ else
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}"
+ fi
+ AIRFLOW_TESTING_CONTAINER=$("${DC_FILE}" ps | \
+ grep airflow | awk '{print $1}' 2>/dev/null)
+ : "${AIRFLOW_TESTING_CONTAINER:?"ERROR! Breeze must be running in order to exec into running container"}"
+ set -e
+ docker exec -it "${AIRFLOW_TESTING_CONTAINER}" \
+ "/opt/airflow/scripts/in_container/entrypoint_exec.sh" "${@}"
+ ;;
+ run_tests)
+ export RUN_TESTS="true"
+ "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" run --service-ports --rm airflow "$@"
+ ;;
+ run_docker_compose)
+ set +u
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}"
+ else
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}"
+ fi
+ "${DC_FILE}" "${DOCKER_COMPOSE_COMMAND}" "${EXTRA_DC_OPTIONS[@]}" "$@"
+ set -u
+ ;;
+ perform_static_checks)
+ make_sure_precommit_is_installed
+ run_static_checks "${@}"
+ ;;
+ build_image)
+ ;;
+ cleanup_image)
+ remove_images
+ ;;
+ perform_generate_constraints)
+ run_generate_constraints
+ ;;
+ perform_prepare_backport_packages)
+ run_prepare_backport_packages "${@}"
+ ;;
+ perform_prepare_backport_readme)
+ run_prepare_backport_readme "${@}"
+ ;;
+ perform_push_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ push_prod_images
+ else
+ push_ci_images
+ fi
+ ;;
+ perform_initialize_local_virtualenv)
+ initialize_virtualenv
+ ;;
+ perform_setup_autocomplete)
+ setup_autocomplete
+ ;;
+ manage_kind_cluster)
+ make_sure_kubernetes_tools_are_installed
+ initialize_kind_variables
+ perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
+ ;;
+ build_docs)
+ run_docs "${@}"
+ ;;
+ *)
+ echo >&2
+ echo >&2 "ERROR: Unknown command to run ${COMMAND_TO_RUN}"
+ echo >&2
+ ;;
esac
set -u
}
-#######################################################################################################
-#
-# We have different versions of images depending on the python version used. We keep up with the
-# Latest patch-level changes in Python (this is done automatically during CI builds) so we have
-# To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor
-# version of python in X.Y format (3.6, 3.7, 3.8 etc).
-#
-# In Breeze the precedence of setting the version is as follows:
-# 1. --python flag (if set, it will explicitly override it in the next step)
-# 2. PYTHON_MAJOR_MINOR_VERSION exported from outside
-# 3. last used version stored in ./build/PYTHON_MAJOR_MINOR_VERSION
-# 4. DEFAULT_PYTHON_MAJOR_MINOR_VERSION from scripts/ci/libraries/_initialization.sh
-#
-# Here points 2. and 3. are realised. If result is empty string , the 4. will be set in
-# the next step (basic_sanity_checks() is called and the version is still not set by then)
-# finally, if --python flag is specified, it will override whatever is set above.
-#
-# We need to run after initialize_common_environment (so that read_from_file function is present)
-# But before we set the default value for Python
-#
-# Used and modified global constants:
-# PYTHON_MAJOR_MINOR_VERSION
-#######################################################################################################
-function determine_python_version_to_use_in_breeze() {
- PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
- export PYTHON_MAJOR_MINOR_VERSION
-}
-
-
-setup_default_breeze_constants
+setup_default_breeze_variables
initialize_common_environment
-determine_python_version_to_use_in_breeze
-
basic_sanity_checks
script_start
@@ -2924,12 +2315,6 @@ read_saved_environment_variables
check_and_save_all_params
-determine_docker_cache_strategy
-
-get_docker_image_names
-
-make_constants_read_only
-
sanitize_mounted_files
prepare_command_files
@@ -2944,11 +2329,11 @@ print_cheatsheet
print_setup_instructions
-set +u # Account for an empty array
+set +u # Account for an empty array
run_breeze_command "${REMAINING_ARGS[@]}"
-set +u # Account for an empty array
-if [[ -n ${second_command_to_run} ]]; then
- command_to_run=${second_command_to_run}
+set +u # Account for an empty array
+if [[ -n ${SECOND_COMMAND_TO_RUN} ]]; then
+ COMMAND_TO_RUN=${SECOND_COMMAND_TO_RUN}
run_breeze_command "${REMAINING_ARGS[@]}"
fi
diff --git a/breeze-complete b/breeze-complete
index 336ad0f..d30beb5 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -28,23 +28,6 @@ _BREEZE_ALLOWED_MYSQL_VERSIONS="5.7 8"
_BREEZE_ALLOWED_POSTGRES_VERSIONS="9.6 10"
_BREEZE_ALLOWED_KIND_OPERATIONS="start stop restart status deploy test shell"
-# Default values for the flags used
-
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_BACKEND="sqlite"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_KUBERNETES_MODE="image"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_KUBERNETES_VERSION="v1.18.6"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_KIND_VERSION="v0.8.0"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_HELM_VERSION="v3.2.4"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_POSTGRES_VERSION="9.6"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_MYSQL_VERSION="5.7"
-
_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=$(cat <<-EOF
1.10.12
1.10.11
@@ -204,6 +187,9 @@ function _get_known_values_breeze() {
static-check)
_BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_STATIC_CHECKS}
;;
+ static-check-all-files)
+ _BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_STATIC_CHECKS}
+ ;;
-A | --install-airflow-version)
_BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS}
;;
diff --git a/confirm b/confirm
index e796737..50ab15f 100755
--- a/confirm
+++ b/confirm
@@ -17,7 +17,7 @@
# under the License.
set -euo pipefail
-if [[ -n "${FORCE_ANSWER_TO_QUESTIONS=}" ]]; then
+if [[ "${FORCE_ANSWER_TO_QUESTIONS:=""}" != "" ]]; then
RESPONSE=${FORCE_ANSWER_TO_QUESTIONS}
echo
echo "Forcing response '${FORCE_ANSWER_TO_QUESTIONS}' to '${1}?'"
diff --git a/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
index 1619396..d756dac 100755
--- a/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
+++ b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/backport_packages/ci_prepare_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
index f30b4a7..e1c8bd0 100755
--- a/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
+++ b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/backport_packages/ci_prepare_backport_readme.sh b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
index 8bf481c..b869128 100755
--- a/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
+++ b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
index 190659a..11f009a 100755
--- a/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
+++ b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -31,6 +33,8 @@ function run_test_package_import_all_classes() {
"--" "/opt/airflow/scripts/in_container/run_test_package_import_all_classes.sh"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
index d923683..2602c2b 100755
--- a/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
+++ b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -26,6 +28,8 @@ function run_test_package_installation_separately() {
"--" "/opt/airflow/scripts/in_container/run_test_package_installation_separately.sh"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/constraints/ci_branch_constraints.sh b/scripts/ci/constraints/ci_branch_constraints.sh
index 4cc7227..5bcc688 100755
--- a/scripts/ci/constraints/ci_branch_constraints.sh
+++ b/scripts/ci/constraints/ci_branch_constraints.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
if [[ ${GITHUB_REF} == 'refs/heads/master' ]]; then
echo "::set-output name=branch::constraints-master"
elif [[ ${GITHUB_REF} == 'refs/heads/v1-10-test' ]]; then
diff --git a/scripts/ci/constraints/ci_commit_constraints.sh b/scripts/ci/constraints/ci_commit_constraints.sh
index f950eaf..f20e834 100755
--- a/scripts/ci/constraints/ci_commit_constraints.sh
+++ b/scripts/ci/constraints/ci_commit_constraints.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
cp -v ./artifacts/constraints-*/constraints*.txt repo/
cd repo || exit 1
git config --local user.email "dev@airflow.apache.org"
diff --git a/scripts/ci/constraints/ci_generate_constraints.sh b/scripts/ci/constraints/ci_generate_constraints.sh
index 946d010..1b7ee4f 100755
--- a/scripts/ci/constraints/ci_generate_constraints.sh
+++ b/scripts/ci/constraints/ci_generate_constraints.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/docs/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
index 0058454..13925ad 100755
--- a/scripts/ci/docs/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index 4fa23a9..aba2dcb 100755
--- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -21,6 +21,7 @@
# Builds or waits for the CI image in the CI environment
# Depending on "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_ci_image_on_ci() {
+ get_environment_for_builds_on_ci
prepare_ci_build
rm -rf "${BUILD_CACHE_DIR}"
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index 65a4da4..9f16bf8 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -21,6 +21,7 @@
# Builds or waits for the PROD image in the CI environment
# Depending on the "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_prod_images_on_ci() {
+ get_environment_for_builds_on_ci
prepare_prod_build
rm -rf "${BUILD_CACHE_DIR}"
diff --git a/scripts/ci/images/ci_push_ci_images.sh b/scripts/ci/images/ci_push_ci_images.sh
index 48d9ec5..2d69d9e 100755
--- a/scripts/ci/images/ci_push_ci_images.sh
+++ b/scripts/ci/images/ci_push_ci_images.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/images/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh
index 5b6f800..db439d8 100755
--- a/scripts/ci/images/ci_push_production_images.sh
+++ b/scripts/ci/images/ci_push_production_images.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh
index a1dd705..74bb3ef 100755
--- a/scripts/ci/images/ci_wait_for_all_ci_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_ci_images.sh
@@ -15,18 +15,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
-echo
-echo "Airflow sources: ${AIRFLOW_SOURCES}"
-echo
+
+# This is hook build used by DockerHub. We are also using it
+# on CI to potentially rebuild (and refresh layers that
+# are not cached) Docker images that are used to run CI jobs
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
+ echo "This script should not be called!"
+ echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY} and GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
echo
exit 1
fi
@@ -40,14 +39,7 @@ echo "Check if jq is installed"
echo
command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
-echo
-echo "The jq version $(jq --version)"
-echo
-
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
-initialize_common_environment
+jq --version
for PYTHON_MAJOR_MINOR_VERSION in "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}"
do
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh
index 9cb4b34..51e1ca9 100755
--- a/scripts/ci/images/ci_wait_for_all_prod_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -15,18 +15,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
-echo
-echo "Airflow sources: ${AIRFLOW_SOURCES}"
-echo
+
+# This is hook build used by DockerHub. We are also using it
+# on CI to potentially rebuild (and refresh layers that
+# are not cached) Docker images that are used to run CI jobs
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
+ echo "This script should not be called!"
+ echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY} and GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
echo
exit 1
fi
@@ -38,17 +37,9 @@ echo
echo
echo "Check if jq is installed"
echo
-command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
-
-echo
-echo "The jq version $(jq --version)"
-echo
-
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
-initialize_common_environment
+command -v jq
+jq --version
for PYTHON_MAJOR_MINOR_VERSION in "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}"
do
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
diff --git a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
index f8393f2..d6d653d 100755
--- a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
+++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
@@ -19,10 +19,18 @@
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
set -euo pipefail
-add_trap "dump_kind_logs" EXIT HUP INT TERM
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
+export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
+export KUBERNETES_MODE=${KUBERNETES_MODE:="image"}
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
+
+get_environment_for_builds_on_ci
+initialize_kind_variables
make_sure_kubernetes_tools_are_installed
-get_kind_cluster_name
prepare_prod_build
build_prod_images
build_image_for_kubernetes_tests
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index e5e7615..78a61f8 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -18,10 +18,10 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-make_sure_kubernetes_tools_are_installed
-get_kind_cluster_name
-
-add_trap dump_kind_logs EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
INTERACTIVE="false"
@@ -70,6 +70,9 @@ else
fi
+get_environment_for_builds_on_ci
+initialize_kind_variables
+
cd "${AIRFLOW_SOURCES}" || exit 1
VIRTUALENV_PATH="${BUILD_CACHE_DIR}/.kubernetes_venv"
diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
index 3688c03..dca8dbd 100755
--- a/scripts/ci/libraries/_all_libs.sh
+++ b/scripts/ci/libraries/_all_libs.sh
@@ -17,17 +17,11 @@
# under the License.
LIBRARIES_DIR=$(dirname "${BASH_SOURCE[0]}")
-export LIBRARIES_DIR
-readonly LIBRARIES_DIR
-SCRIPTS_CI_DIR=$(cd "${LIBRARIES_DIR}/.." && pwd)
-export SCRIPTS_CI_DIR
-readonly SCRIPTS_CI_DIR
-
-# shellcheck source=scripts/ci/libraries/_traps.sh
-. "${LIBRARIES_DIR}"/_traps.sh
# shellcheck source=scripts/ci/libraries/_initialization.sh
. "${LIBRARIES_DIR}"/_initialization.sh
+
+
# shellcheck source=scripts/ci/libraries/_sanity_checks.sh
. "${LIBRARIES_DIR}"/_sanity_checks.sh
# shellcheck source=scripts/ci/libraries/_build_images.sh
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 7439f03..e94b920 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
+
# For remote installation of airflow (from GitHub or Pypi) when building the image, you need to
# pass build flags depending on the version and method of the installation (for example to
# get proper requirement constraint files)
@@ -55,7 +56,7 @@ function add_build_args_for_remote_install() {
# Retrieves version of airflow stored in the production image (used to display the actual
# Version we use if it was build from PyPI or GitHub
function get_airflow_version_from_production_image() {
- VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
+ VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
}
# Removes the "Forced answer" (yes/no/quit) given previously, unless you specifically want to remember it.
@@ -110,20 +111,17 @@ function confirm_image_rebuild() {
print_info "CI environment - forcing rebuild for image ${THE_IMAGE_TYPE}."
print_info
RES="0"
- elif [[ -n "${FORCE_ANSWER_TO_QUESTIONS=}" ]]; then
+ elif [[ -n "${FORCE_ANSWER_TO_QUESTIONS:=""}" ]]; then
print_info
print_info "Forcing answer '${FORCE_ANSWER_TO_QUESTIONS}'"
print_info
case "${FORCE_ANSWER_TO_QUESTIONS}" in
- [yY][eE][sS] | [yY])
- RES="0"
- ;;
- [qQ][uU][iI][tT] | [qQ])
- RES="2"
- ;;
- *)
- RES="1"
- ;;
+ [yY][eE][sS]|[yY])
+ RES="0" ;;
+ [qQ][uU][iI][tT]|[qQ])
+ RES="2" ;;
+ *)
+ RES="1" ;;
esac
elif [[ -t 0 ]]; then
# Check if this script is run interactively with stdin open and terminal attached
@@ -161,7 +159,7 @@ function confirm_image_rebuild() {
# Force "no" also to subsequent questions so that if you answer it once, you are not asked
# For all other pre-commits and you will continue using the images you already have
export FORCE_ANSWER_TO_QUESTIONS="no"
- echo 'export FORCE_ANSWER_TO_QUESTIONS="no"' >"${LAST_FORCE_ANSWER_FILE}"
+ echo 'export FORCE_ANSWER_TO_QUESTIONS="no"' > "${LAST_FORCE_ANSWER_FILE}"
elif [[ ${RES} == "2" ]]; then
echo >&2
echo >&2 "ERROR: The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. "
@@ -183,11 +181,11 @@ function confirm_image_rebuild() {
# We cannot use docker registry APIs as they are available only with authorisation
# But this image can be pulled without authentication
function build_ci_image_manifest() {
- docker inspect "${AIRFLOW_CI_IMAGE}" >"manifests/${AIRFLOW_CI_BASE_TAG}.json"
+ docker inspect "${AIRFLOW_CI_IMAGE}" > "manifests/${AIRFLOW_CI_BASE_TAG}.json"
docker build \
- --build-arg AIRFLOW_CI_BASE_TAG="${AIRFLOW_CI_BASE_TAG}" \
- --tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \
- -f- . <<EOF
+ --build-arg AIRFLOW_CI_BASE_TAG="${AIRFLOW_CI_BASE_TAG}" \
+ --tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \
+ -f- . <<EOF
ARG AIRFLOW_CI_BASE_TAG
FROM scratch
@@ -216,7 +214,7 @@ function get_local_image_info() {
return
fi
set -e
- # Create manifest from the local manifest image
+ # Create manifest from the local manifest image
docker cp "local-airflow-manifest:${AIRFLOW_CI_BASE_TAG}.json" "${TMP_MANIFEST_LOCAL_JSON}"
sed 's/ *//g' "${TMP_MANIFEST_LOCAL_JSON}" | grep '^"sha256:' >"${TMP_MANIFEST_LOCAL_SHA}"
docker rm --force "local-airflow-manifest"
@@ -233,10 +231,10 @@ function get_local_image_info() {
function get_remote_image_info() {
set +e
# Pull remote manifest image
- if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"; then
- echo >&2
- echo >&2 "Remote docker registry unreachable"
- echo >&2
+ if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"; then
+ >&2 echo
+ >&2 echo "Remote docker registry unreachable"
+ >&2 echo
REMOTE_DOCKER_REGISTRY_UNREACHABLE="true"
return
fi
@@ -257,7 +255,7 @@ function get_remote_image_info() {
"${TMP_MANIFEST_REMOTE_JSON}"
# Filter everything except SHAs of image layers
sed 's/ *//g' "${TMP_MANIFEST_REMOTE_JSON}" | grep '^"sha256:' >"${TMP_MANIFEST_REMOTE_SHA}"
- docker rm --force "$(cat "${TMP_CONTAINER_ID}")"
+ docker rm --force "$( cat "${TMP_CONTAINER_ID}")"
}
# The Number determines the cut-off between local building time and pull + build time.
@@ -280,7 +278,7 @@ MAGIC_CUT_OFF_NUMBER_OF_LAYERS=36
# Compares layers from both remote and local image and set FORCE_PULL_IMAGES to true in case
# More than the last NN layers are different.
function compare_layers() {
- NUM_DIFF=$(diff -y --suppress-common-lines "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_SHA}" |
+ NUM_DIFF=$(diff -y --suppress-common-lines "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_SHA}" | \
wc -l || true)
rm -f "${TMP_MANIFEST_REMOTE_JSON}" "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_JSON}" "${TMP_MANIFEST_LOCAL_SHA}"
echo
@@ -288,7 +286,7 @@ function compare_layers() {
echo
# This is where setup py is rebuilt - it will usually take a looooot of time to build it, so it is
# Better to pull here
- if ((NUM_DIFF >= MAGIC_CUT_OFF_NUMBER_OF_LAYERS)); then
+ if (( NUM_DIFF >= MAGIC_CUT_OFF_NUMBER_OF_LAYERS )); then
echo
echo
echo "WARNING! Your image and the dockerhub image differ significantly"
@@ -304,6 +302,7 @@ function compare_layers() {
fi
}
+
# Prints summary of the build parameters
function print_build_info() {
print_info
@@ -311,48 +310,28 @@ function print_build_info() {
print_info
}
-function get_docker_image_names() {
+function get_base_image_version() {
# python image version to use
- export PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}}
-
- # Python base image to use
- export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
-
- # CI image base tag
- export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
-
- # CI image to build
- export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
-
- # Default CI image
- export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-ci"
-
- # Base production image tag - used to build kubernetes tag as well
- export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
-
- # PROD image to build
- export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}"
-
- # PROD build segment
- export AIRFLOW_PROD_BUILD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-build"
-
- # PROD Kubernetes image to build
- export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}}
+}
- # PROD default image
- export AIRFLOW_PROD_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}"
- # File that is touched when the CI image is built for the first time locally
- export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
-}
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
# DockerHub user etc. the variables are set so that other functions can use those variables.
function prepare_ci_build() {
+ get_base_image_version
+ # We use pulled docker image cache by default for CI images to speed up the builds
+ export DOCKER_CACHE=${DOCKER_CACHE:="pulled"}
+ print_info
+ print_info "Using ${DOCKER_CACHE} cache strategy for the build."
+ print_info
+ export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
+ export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- if [[ -n ${GITHUB_TOKEN=} ]]; then
+ if [[ ${GITHUB_TOKEN:=} != "" ]]; then
echo "${GITHUB_TOKEN}" | docker login \
--username "${GITHUB_USERNAME}" \
--password-stdin \
@@ -362,6 +341,10 @@ function prepare_ci_build() {
export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${GITHUB_REPOSITORY_LOWERCASE}/${AIRFLOW_CI_BASE_TAG}"
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${GITHUB_REPOSITORY_LOWERCASE}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
fi
+ export AIRFLOW_BUILD_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}/${AIRFLOW_CI_BASE_TAG}"
+ export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-ci"
+ export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
+ export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then
export DEFAULT_PROD_IMAGE="${AIRFLOW_CI_IMAGE_DEFAULT}"
else
@@ -369,25 +352,24 @@ function prepare_ci_build() {
fi
export THE_IMAGE_TYPE="CI"
export IMAGE_DESCRIPTION="Airflow CI"
-
- # Those constants depend on the type of image run so they are only made constants here
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}"
- readonly AIRFLOW_EXTRAS
-
+ export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
+ export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
+ export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}"
+ export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
- readonly AIRFLOW_IMAGE
-
go_to_airflow_sources
fix_group_permissions
}
+
# Only rebuilds CI image if needed. It checks if the docker image build is needed
# because any of the important source files (from scripts/ci/libraries/_initialization.sh) has
# changed or in any of the edge cases (docker image removed, .build cache removed etc.
# In case rebuild is needed, it determines (by comparing layers in local and remote image)
# Whether pull is needed before rebuild.
function rebuild_ci_image_if_needed() {
- if [[ -f "${BUILT_CI_IMAGE_FLAG_FILE}" ]]; then
+ if [[ -f "${BUILT_IMAGE_FLAG_FILE}" ]]; then
print_info
print_info "${THE_IMAGE_TYPE} image already built locally."
print_info
@@ -399,7 +381,7 @@ function rebuild_ci_image_if_needed() {
export FORCE_BUILD_IMAGES="true"
fi
- if [[ ${CHECK_IMAGE_FOR_REBUILD} == "false" ]]; then
+ if [[ ${CHECK_IMAGE_FOR_REBUILD:="true"} == "false" ]]; then
print_info
print_info "Skip checking for rebuilds of the CI image but checking if it needs to be pulled"
print_info
@@ -407,9 +389,9 @@ function rebuild_ci_image_if_needed() {
return
fi
- local needs_docker_build="false"
+ NEEDS_DOCKER_BUILD="false"
check_if_docker_build_is_needed
- if [[ ${needs_docker_build} == "true" ]]; then
+ if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
if [[ ${SKIP_CHECK_REMOTE_IMAGE:=} != "true" && ${DOCKER_CACHE} == "pulled" ]]; then
# Check if remote image is different enough to force pull
# This is an optimisation pull vs. build time. When there
@@ -457,16 +439,17 @@ function rebuild_ci_image_if_needed() {
fi
}
+
# Interactive version of confirming the ci image that is used in pre-commits
# it displays additional information - what the user should do in order to bring the local images
# back to state that pre-commit will be happy with
function rebuild_ci_image_if_needed_and_confirmed() {
- local needs_docker_build="false"
+ NEEDS_DOCKER_BUILD="false"
THE_IMAGE_TYPE="CI"
check_if_docker_build_is_needed
- if [[ ${needs_docker_build} == "true" ]]; then
+ if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
print_info
print_info "Docker image build is needed!"
print_info
@@ -476,7 +459,7 @@ function rebuild_ci_image_if_needed_and_confirmed() {
print_info
fi
- if [[ "${needs_docker_build}" == "true" ]]; then
+ if [[ "${NEEDS_DOCKER_BUILD}" == "true" ]]; then
echo
echo "Some of your images need to be rebuild because important files (like package list) has changed."
echo
@@ -505,13 +488,13 @@ function rebuild_ci_image_if_needed_and_confirmed() {
# selected by Breeze flags or environment variables.
function build_ci_image() {
print_build_info
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "Preparing ${AIRFLOW_CI_IMAGE}.
- " >"${DETECTED_TERMINAL}"
+ " > "${DETECTED_TERMINAL}"
spin "${OUTPUT_LOG}" &
SPIN_PID=$!
# shellcheck disable=SC2064
- add_trap "kill ${SPIN_PID}" INT TERM HUP EXIT
+ trap "kill ${SPIN_PID}" SIGINT SIGTERM
fi
pull_ci_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
@@ -528,29 +511,29 @@ function build_ci_image() {
echo >&2
exit 1
fi
- if [[ -n ${SPIN_PID=} ]]; then
+ if [[ -n ${SPIN_PID:=""} ]]; then
kill -HUP "${SPIN_PID}" || true
wait "${SPIN_PID}" || true
- echo >"${DETECTED_TERMINAL}"
+ echo > "${DETECTED_TERMINAL}"
fi
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "Preparing ${AIRFLOW_CI_IMAGE}.
- " >"${DETECTED_TERMINAL}"
+ " > "${DETECTED_TERMINAL}"
spin "${OUTPUT_LOG}" &
SPIN_PID=$!
# shellcheck disable=SC2064
- add_trap "kill ${SPIN_PID}" EXIT HUP INT TERM
+ trap "kill ${SPIN_PID}" SIGINT SIGTERM
fi
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "
Docker building ${AIRFLOW_CI_IMAGE}.
-" >"${DETECTED_TERMINAL}"
+" > "${DETECTED_TERMINAL}"
fi
set +u
docker build \
--build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
--build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
- --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
+ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
@@ -565,27 +548,33 @@ Docker building ${AIRFLOW_CI_IMAGE}.
--target "main" \
. -f Dockerfile.ci
set -u
- if [[ -n "${DEFAULT_CI_IMAGE=}" ]]; then
+ if [[ -n "${DEFAULT_CI_IMAGE:=}" ]]; then
docker tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}"
fi
- if [[ -n ${SPIN_PID=} ]]; then
+ if [[ -n ${SPIN_PID:=""} ]]; then
kill -HUP "${SPIN_PID}" || true
wait "${SPIN_PID}" || true
- echo >"${DETECTED_TERMINAL}"
+ echo > "${DETECTED_TERMINAL}"
fi
}
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
# DockerHub user etc. the variables are set so that other functions can use those variables.
function prepare_prod_build() {
- if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
+ get_base_image_version
+ # We use local docker image cache by default for Production images
+ export DOCKER_CACHE=${DOCKER_CACHE:="local"}
+ print_info
+ print_info "Using ${DOCKER_CACHE} cache strategy for the build."
+ print_info
+ if [[ "${INSTALL_AIRFLOW_REFERENCE:=}" != "" ]]; then
# When --install-airflow-reference is used then the image is build from github tag
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
"--build-arg" "AIRFLOW_INSTALL_SOURCES=https://github.com/apache/airflow/archive/${INSTALL_AIRFLOW_REFERENCE}.tar.gz#egg=apache-airflow"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_REFERENCE}"
add_build_args_for_remote_install
- elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
+ elif [[ "${INSTALL_AIRFLOW_VERSION:=}" != "" ]]; then
# When --install-airflow-version is used then the image is build from PIP package
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
"--build-arg" "AIRFLOW_INSTALL_SOURCES=apache-airflow"
@@ -598,6 +587,13 @@ function prepare_prod_build() {
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
)
fi
+
+ export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ export AIRFLOW_PROD_BUILD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-build"
+ export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}"
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ export AIRFLOW_PROD_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}"
+ export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then
export DEFAULT_CI_IMAGE="${AIRFLOW_PROD_IMAGE_DEFAULT}"
else
@@ -605,16 +601,15 @@ function prepare_prod_build() {
fi
export THE_IMAGE_TYPE="PROD"
export IMAGE_DESCRIPTION="Airflow production"
-
- # Those constants depend on the type of image run so they are only made constants here
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_PROD_EXTRAS}"}"
- readonly AIRFLOW_EXTRAS
-
+ export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
+ export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
+ export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}"
+ export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
- readonly AIRFLOW_IMAGE
if [[ ${USE_GITHUB_REGISTRY="false"} == "true" ]]; then
- if [[ -n ${GITHUB_TOKEN=} ]]; then
+ if [[ ${GITHUB_TOKEN:=} != "" ]]; then
echo "${GITHUB_TOKEN}" | docker login \
--username "${GITHUB_USERNAME}" \
--password-stdin \
@@ -638,13 +633,14 @@ function prepare_prod_build() {
function build_prod_images() {
print_build_info
- if [[ ${SKIP_BUILDING_PROD_IMAGE} == "true" ]]; then
+ if [[ ${SKIP_BUILDING_PROD_IMAGE:="false"} == "true" ]]; then
print_info
print_info "Skip building production image. Assume the one we have is good!"
print_info
return
fi
+
pull_prod_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
@@ -662,9 +658,9 @@ function build_prod_images() {
"--cache-from" "${AIRFLOW_PROD_BUILD_IMAGE}"
)
else
- echo >&2
- echo >&2 "Error - thee ${DOCKER_CACHE} cache is unknown!"
- echo >&2
+ >&2 echo
+ >&2 echo "Error - thee ${DOCKER_CACHE} cache is unknown!"
+ >&2 echo
exit 1
fi
set +u
@@ -712,7 +708,7 @@ function build_prod_images() {
# $1 - image name to wait for
# $2 - suffix of the image to wait for
# $3, $4, ... - target tags to tag the image with
-function wait_for_image_tag() {
+function wait_for_image_tag {
IMAGE_NAME="${1}"
IMAGE_SUFFIX=${2}
shift 2
@@ -723,7 +719,7 @@ function wait_for_image_tag() {
echo
while true; do
docker pull "${IMAGE_TO_WAIT_FOR}" || true
- if [[ -z "$(docker images -q "${IMAGE_TO_WAIT_FOR}" 2>/dev/null || true)" ]]; then
+ if [[ "$(docker images -q "${IMAGE_TO_WAIT_FOR}" 2> /dev/null)" == "" ]]; then
echo
echo "The image ${IMAGE_TO_WAIT_FOR} is not yet available. Waiting"
echo
@@ -735,30 +731,14 @@ function wait_for_image_tag() {
echo
echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${IMAGE_NAME}."
echo
- docker tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}"
+ docker tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}"
for TARGET_TAG in "${@}"; do
echo
echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${TARGET_TAG}."
echo
- docker tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}"
+ docker tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}"
done
break
fi
done
}
-
-# We use pulled docker image cache by default for CI images to speed up the builds
-# and local to speed up iteration on kerberos tests
-function determine_docker_cache_strategy() {
- if [[ -z "${DOCKER_CACHE=}" ]]; then
- if [[ "${PRODUCTION_IMAGE}" == "true" ]]; then
- export DOCKER_CACHE="local"
- else
- export DOCKER_CACHE="pulled"
- fi
- fi
- readonly DOCKER_CACHE
- print_info
- print_info "Using ${DOCKER_CACHE} cache strategy for the build."
- print_info
-}
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 6e6bc95..e2d4d97 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -17,50 +17,26 @@
# under the License.
# Needs to be declared outside function in MacOS
-# shellcheck disable=SC2034
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS=()
-# Creates directories for Breeze
-function create_directories() {
+# Very basic variables that MUST be set
+function initialize_base_variables() {
# This folder is mounted to inside the container in /files folder. This is the way how
# We can exchange DAGs, scripts, packages etc with the container environment
export FILES_DIR="${AIRFLOW_SOURCES}/files"
- readonly FILES_DIR
-
- # Directory where all the build cache is stored - we keep there status of all the docker images
- # As well as hashes of the important files, but also we generate build scripts there that are
- # Used to execute the commands for breeze
- export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
- export BUILD_CACHE_DIR
- readonly BUILD_CACHE_DIR
-
- # Create those folders above in case they do not exist
- mkdir -p "${BUILD_CACHE_DIR}" >/dev/null
- mkdir -p "${FILES_DIR}" >/dev/null
+ # Temporary dir used well ... temporarily
+ export TMP_DIR="${AIRFLOW_SOURCES}/tmp"
# By default we are not in CI environment GitHub Actions sets CI to "true"
export CI="${CI="false"}"
# Create useful directories if not yet created
+ mkdir -p "${TMP_DIR}"
+ mkdir -p "${FILES_DIR}"
mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache"
mkdir -p "${AIRFLOW_SOURCES}/logs"
mkdir -p "${AIRFLOW_SOURCES}/dist"
- CACHE_TMP_FILE_DIR=$(mktemp -d)
- export CACHE_TMP_FILE_DIR
- readonly CACHE_TMP_FILE_DIR
-
- if [[ ${SKIP_CACHE_DELETION=} != "true" ]]; then
- add_trap "rm -rf -- '${CACHE_TMP_FILE_DIR}'" EXIT HUP INT TERM
- fi
-
- OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log"
- export OUTPUT_LOG
- readonly OUTPUT_LOG
-}
-
-# Very basic variables that MUST be set
-function initialize_base_variables() {
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
@@ -77,13 +53,17 @@ function initialize_base_variables() {
# Python version and avoids problems with root-owned .pyc files in host
export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"}
- # By default we build CI images but we can switch to production image with PRODUCTION_IMAGE="true"
- export PRODUCTION_IMAGE="false"
+ # Sets to the build cache directory - status of build and convenience scripts are stored there
+ BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
+ export BUILD_CACHE_DIR
# Currently supported major/minor versions of python
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS+=("3.6" "3.7" "3.8")
export CURRENT_PYTHON_MAJOR_MINOR_VERSIONS
+ # default python Major/Minor version
+ PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
+
# If set to true, the database will be reset at entry. Works for Postgres and MySQL
export DB_RESET=${DB_RESET:="false"}
@@ -91,12 +71,8 @@ function initialize_base_variables() {
AIRFLOW_VERSION=$(grep version "${AIRFLOW_SOURCES}/airflow/version.py" | awk '{print $3}' | sed "s/['+]//g")
export AIRFLOW_VERSION
- # Whether credentials should be forwarded to inside the docker container
- export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"}
-
- # If no Airflow Home defined - fallback to ${HOME}/airflow
- AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow}
- export AIRFLOW_HOME_DIR
+ # default version of python used to tag the "master" and "latest" images in DockerHub
+ export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6
}
# Determine current branch
@@ -104,7 +80,6 @@ function initialize_branch_variables() {
# Default branch used - this will be different in different branches
export DEFAULT_BRANCH="master"
export DEFAULT_CONSTRAINTS_BRANCH="constraints-master"
- readonly DEFAULT_CONSTRAINTS_BRANCH
# Default branch name for triggered builds is the one configured in default branch
# We need to read it here as it comes from _common_values.sh
@@ -256,7 +231,6 @@ function initialize_image_build_variables() {
# Default extras used for building CI image
export DEFAULT_CI_EXTRAS="devel_ci"
- # Default build id
export CI_BUILD_ID="${CI_BUILD_ID:="0"}"
# Default extras used for building Production image. The master of this information is in the Dockerfile
@@ -277,24 +251,13 @@ function initialize_image_build_variables() {
# Checks if the image should be rebuilt
export CHECK_IMAGE_FOR_REBUILD="${CHECK_IMAGE_FOR_REBUILD:="true"}"
- # Skips building production images altogether (assume they are already built)
- export SKIP_BUILDING_PROD_IMAGE="${SKIP_BUILDING_PROD_IMAGE:="false"}"
-
- # Additional airflow extras on top of the default ones
- export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
- # Additional python dependencies on top of the default ones
- export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
- # additional development apt dependencies on top of the default ones
- export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}"
- # additional runtime apt dependencies on top of the default ones
- export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
-
}
# Determine version suffixes used to build backport packages
function initialize_version_suffixes_for_package_building() {
# Version suffix for PyPI packaging
export VERSION_SUFFIX_FOR_PYPI=""
+
# Artifact name suffix for SVN packaging
export VERSION_SUFFIX_FOR_SVN=""
}
@@ -303,63 +266,46 @@ function initialize_version_suffixes_for_package_building() {
function initialize_kubernetes_variables() {
# By default we assume the kubernetes cluster is not being started
export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
+
# Default Kubernetes version
export DEFAULT_KUBERNETES_VERSION="v1.18.6"
- readonly DEFAULT_KUBERNETES_VERSION
+
# Default KinD version
export DEFAULT_KIND_VERSION="v0.8.0"
- readonly DEFAULT_KIND_VERSION
+
# Default Helm version
export DEFAULT_HELM_VERSION="v3.2.4"
- readonly DEFAULT_HELM_VERSION
+
# Namespace where airflow is installed via helm
export HELM_AIRFLOW_NAMESPACE="airflow"
- # Kind version
- export KIND_VERSION=${KIND_VERSION:=${DEFAULT_KIND_VERSION}}
- # Helm version
- export HELM_VERSION=${HELM_VERSION:=${DEFAULT_HELM_VERSION}}
- # Kubectl version
- export KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}}
- # Local Kind path
- export KIND_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kind"
- readonly KIND_BINARY_PATH
- # Local Helm path
- export HELM_BINARY_PATH="${BUILD_CACHE_DIR}/bin/helm"
- readonly HELM_BINARY_PATH
- # local Kubectl path
- export KUBECTL_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kubectl"
- readonly KUBECTL_BINARY_PATH
+
+ COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")"
+ export COMMIT_SHA
+
+ export CI_BUILD_ID="0"
}
function initialize_git_variables() {
# SHA of the commit for the current sources
- COMMIT_SHA="$(git rev-parse HEAD 2>/dev/null || echo "Unknown")"
+ COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")"
export COMMIT_SHA
}
function initialize_github_variables() {
# Defaults for interacting with GitHub
- export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
-
+ export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+ GITHUB_REPOSITORY_LOWERCASE="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
+ export GITHUB_REPOSITORY_LOWERCASE
export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"}
+ export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
- export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
- GITHUB_REPOSITORY_LOWERCASE="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
- export GITHUB_REPOSITORY_LOWERCASE
-
- # Used only in CI environment
- export GITHUB_TOKEN="${GITHUB_TOKEN=""}"
- export GITHUB_USERNAME="${GITHUB_USERNAME=""}"
-
-
}
# Common environment that is initialized by both Breeze and CI scripts
function initialize_common_environment() {
- create_directories
initialize_base_variables
initialize_branch_variables
initialize_available_integrations
@@ -375,15 +321,6 @@ function initialize_common_environment() {
initialize_github_variables
}
-function set_default_python_version_if_empty() {
- # default version of python used to tag the "master" and "latest" images in DockerHub
- export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6
-
- # default python Major/Minor version
- export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}}
-
-}
-
function summarize_ci_environment() {
cat <<EOF
@@ -493,7 +430,7 @@ function get_environment_for_builds_on_ci() {
export CI_SOURCE_BRANCH="${GITHUB_HEAD_REF}"
BRANCH_EXISTS=$(git ls-remote --heads \
"https://github.com/${CI_SOURCE_REPO}.git" "${CI_SOURCE_BRANCH}" || true)
- if [[ -z ${BRANCH_EXISTS=} ]]; then
+ if [[ ${BRANCH_EXISTS} == "" ]]; then
print_info
print_info "https://github.com/${CI_SOURCE_REPO}.git Branch ${CI_SOURCE_BRANCH} does not exist"
print_info
@@ -510,114 +447,32 @@ function get_environment_for_builds_on_ci() {
fi
else
# CI PR settings
- export CI_TARGET_REPO="${CI_TARGET_REPO="apache/airflow"}"
- export CI_TARGET_BRANCH="${DEFAULT_BRANCH="master"}"
- export CI_BUILD_ID="${CI_BUILD_ID="0"}"
- export CI_JOB_ID="${CI_JOB_ID="0"}"
- export CI_EVENT_TYPE="${CI_EVENT_TYPE="pull_request"}"
- export CI_REF="${CI_REF="refs/head/master"}"
-
- export CI_SOURCE_REPO="${CI_SOURCE_REPO="apache/airflow"}"
- export CI_SOURCE_BRANCH="${DEFAULT_BRANCH="master"}"
+ export CI_TARGET_REPO="${CI_TARGET_REPO:="apache/airflow"}"
+ export CI_TARGET_BRANCH="${DEFAULT_BRANCH:="master"}"
+ export CI_BUILD_ID="${CI_BUILD_ID:="0"}"
+ export CI_JOB_ID="${CI_JOB_ID:="0"}"
+ export CI_EVENT_TYPE="${CI_EVENT_TYPE:="pull_request"}"
+ export CI_REF="${CI_REF:="refs/head/master"}"
+
+ export CI_SOURCE_REPO="${CI_SOURCE_REPO:="apache/airflow"}"
+ export CI_SOURCE_BRANCH="${DEFAULT_BRANCH:="master"}"
+
+ # Github Registry settings (in Github Actions they come from .github/workflow/ci.yml)
+ export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="true"}
+ export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"}
+ export GITHUB_REPOSITORY="${GITHUB_REPOSITORY:="apache/airflow"}"
+ export GITHUB_USERNAME="${GITHUB_USERNAME=""}"
+ export GITHUB_TOKEN="${GITHUB_TOKEN=""}"
+ export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
+ export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
+ export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
+
+ # DockerHub settings
+ export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
+ export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
fi
if [[ ${VERBOSE} == "true" && ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then
summarize_ci_environment
fi
}
-
-# shellcheck disable=SC2034
-
-# By the time this method is run, nearly all constants have been already set to the final values
-# so we can set them as readonly.
-function make_constants_read_only() {
- # Set the arguments as read-only
- readonly PYTHON_MAJOR_MINOR_VERSION
-
- readonly WEBSERVER_HOST_PORT
- readonly POSTGRES_HOST_PORT
- readonly MYSQL_HOST_PORT
-
-
- readonly HOST_USER_ID
- readonly HOST_GROUP_ID
- readonly HOST_AIRFLOW_SOURCES
- readonly HOST_HOME
- readonly HOST_OS
-
- readonly INTEGRATIONS
-
- readonly ENABLE_KIND_CLUSTER
- readonly KUBERNETES_MODE
- readonly KUBERNETES_VERSION
- readonly KIND_VERSION
- readonly HELM_VERSION
- readonly KUBECTL_VERSION
-
- readonly BACKEND
- readonly POSTGRES_VERSION
- readonly MYSQL_VERSION
-
- readonly MOUNT_LOCAL_SOURCES
-
- readonly INSTALL_AIRFLOW_VERSION
- readonly INSTALL_AIRFLOW_REFERENCE
-
- readonly DB_RESET
- readonly VERBOSE
-
- readonly PRODUCTION_IMAGE
-
- # The FORCE_* variables are missing here because they are not constant - they are just exported variables.
- # Their value might change during the script execution - for example when during the
- # pre-commit the answer is "no", we set the FORCE_ANSWER_TO_QUESTIONS to "no"
- # for all subsequent questions. Also in CI environment we first force pulling and building
- # the images but then we disable it so that in subsequent steps the image is reused.
- # similarly CHECK_IMAGE_FOR_REBUILD variable.
-
- readonly SKIP_BUILDING_PROD_IMAGE
- readonly CI_BUILD_ID
- readonly CI_JOB_ID
-
- # AIRFLOW_EXTRAS are made readonly by the time the image is built (either PROD or CI)
- readonly ADDITIONAL_AIRFLOW_EXTRAS
- readonly ADDITIONAL_PYTHON_DEPS
- readonly ADDITIONAL_DEV_DEPS
- readonly ADDITIONAL_RUNTIME_DEPS
-
- readonly DOCKERHUB_USER
- readonly DOCKERHUB_REPO
- readonly DOCKER_CACHE
-
- readonly USE_GITHUB_REGISTRY
- readonly GITHUB_REGISTRY
- readonly GITHUB_REGISTRY_WAIT_FOR_IMAGE
- readonly GITHUB_REGISTRY_PULL_IMAGE_TAG
- readonly GITHUB_REGISTRY_PUSH_IMAGE_TAG
-
- readonly GITHUB_REPOSITORY
- readonly GITHUB_REPOSITORY_LOWERCASE
- readonly GITHUB_TOKEN
- readonly GITHUB_USERNAME
-
-
- readonly FORWARD_CREDENTIALS
- readonly USE_GITHUB_REGISTRY
-
- readonly EXTRA_STATIC_CHECK_OPTIONS
-
- readonly VERSION_SUFFIX_FOR_PYPI
- readonly VERSION_SUFFIX_FOR_SVN
-
- readonly PYTHON_BASE_IMAGE_VERSION
- readonly PYTHON_BASE_IMAGE
- readonly AIRFLOW_CI_BASE_TAG
- readonly AIRFLOW_CI_IMAGE
- readonly AIRFLOW_CI_IMAGE_DEFAULT
- readonly AIRFLOW_PROD_BASE_TAG
- readonly AIRFLOW_PROD_IMAGE
- readonly AIRFLOW_PROD_BUILD_IMAGE
- readonly AIRFLOW_PROD_IMAGE_KUBERNETES
- readonly AIRFLOW_PROD_IMAGE_DEFAULT
- readonly BUILT_CI_IMAGE_FLAG_FILE
-}
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index aeb540a..f48902a 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -17,13 +17,11 @@
# under the License.
-function get_kind_cluster_name(){
+function initialize_kind_variables(){
# Name of the KinD cluster to connect to
export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
- readonly KIND_CLUSTER_NAME
# Name of the KinD cluster to connect to when referred to via kubectl
export KUBECTL_CLUSTER_NAME=kind-${KIND_CLUSTER_NAME}
- readonly KUBECTL_CLUSTER_NAME
}
function dump_kind_logs() {
@@ -34,56 +32,59 @@ function dump_kind_logs() {
echo "EXIT_CODE is ${EXIT_CODE:=}"
local DUMP_DIR_NAME DUMP_DIR
- DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}
+ DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}
DUMP_DIR="/tmp/${DUMP_DIR_NAME}"
kind --name "${KIND_CLUSTER_NAME}" export logs "${DUMP_DIR}"
}
function make_sure_kubernetes_tools_are_installed() {
SYSTEM=$(uname -s| tr '[:upper:]' '[:lower:]')
-
+ KIND_VERSION=${KIND_VERSION:=${DEFAULT_KIND_VERSION}}
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${SYSTEM}-amd64"
+ KIND_PATH="${BUILD_CACHE_DIR}/bin/kind"
+ HELM_VERSION=${HELM_VERSION:=${DEFAULT_HELM_VERSION}}
+ HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz"
+ HELM_PATH="${BUILD_CACHE_DIR}/bin/helm"
+ KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}}
+ KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${SYSTEM}/amd64/kubectl"
+ KUBECTL_PATH="${BUILD_CACHE_DIR}/bin/kubectl"
mkdir -pv "${BUILD_CACHE_DIR}/bin"
- if [[ -f "${KIND_BINARY_PATH}" ]]; then
- DOWNLOADED_KIND_VERSION=v"$(${KIND_BINARY_PATH} --version | awk '{ print $3 }')"
+ if [[ -f "${KIND_PATH}" ]]; then
+ DOWNLOADED_KIND_VERSION=v"$(${KIND_PATH} --version | awk '{ print $3 }')"
echo "Currently downloaded kind version = ${DOWNLOADED_KIND_VERSION}"
fi
- if [[ ! -f "${KIND_BINARY_PATH}" || ${DOWNLOADED_KIND_VERSION} != "${KIND_VERSION}" ]]; then
+ if [[ ! -f "${KIND_PATH}" || ${DOWNLOADED_KIND_VERSION} != "${KIND_VERSION}" ]]; then
echo
echo "Downloading Kind version ${KIND_VERSION}"
- curl --fail --location "${KIND_URL}" --output "${KIND_BINARY_PATH}"
- chmod a+x "${KIND_BINARY_PATH}"
+ curl --fail --location "${KIND_URL}" --output "${KIND_PATH}"
+ chmod a+x "${KIND_PATH}"
else
echo "Kind version ok"
echo
fi
-
- KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${SYSTEM}/amd64/kubectl"
- if [[ -f "${KUBECTL_BINARY_PATH}" ]]; then
- DOWNLOADED_KUBECTL_VERSION="$(${KUBECTL_BINARY_PATH} version --client=true --short | awk '{ print $3 }')"
+ if [[ -f "${KUBECTL_PATH}" ]]; then
+ DOWNLOADED_KUBECTL_VERSION="$(${KUBECTL_PATH} version --client=true --short | awk '{ print $3 }')"
echo "Currently downloaded kubectl version = ${DOWNLOADED_KUBECTL_VERSION}"
fi
- if [[ ! -f "${KUBECTL_BINARY_PATH}" || ${DOWNLOADED_KUBECTL_VERSION} != "${KUBECTL_VERSION}" ]]; then
+ if [[ ! -f "${KUBECTL_PATH}" || ${DOWNLOADED_KUBECTL_VERSION} != "${KUBECTL_VERSION}" ]]; then
echo
echo "Downloading Kubectl version ${KUBECTL_VERSION}"
- curl --fail --location "${KUBECTL_URL}" --output "${KUBECTL_BINARY_PATH}"
- chmod a+x "${KUBECTL_BINARY_PATH}"
+ curl --fail --location "${KUBECTL_URL}" --output "${KUBECTL_PATH}"
+ chmod a+x "${KUBECTL_PATH}"
else
echo "Kubectl version ok"
echo
fi
-
- HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz"
- if [[ -f "${HELM_BINARY_PATH}" ]]; then
- DOWNLOADED_HELM_VERSION="$(${HELM_BINARY_PATH} version --template '{{.Version}}')"
+ if [[ -f "${HELM_PATH}" ]]; then
+ DOWNLOADED_HELM_VERSION="$(${HELM_PATH} version --template '{{.Version}}')"
echo "Currently downloaded helm version = ${DOWNLOADED_HELM_VERSION}"
fi
- if [[ ! -f "${HELM_BINARY_PATH}" || ${DOWNLOADED_HELM_VERSION} != "${HELM_VERSION}" ]]; then
+ if [[ ! -f "${HELM_PATH}" || ${DOWNLOADED_HELM_VERSION} != "${HELM_VERSION}" ]]; then
echo
echo "Downloading Helm version ${HELM_VERSION}"
curl --location "${HELM_URL}" |
- tar -xvz -O "${SYSTEM}-amd64/helm" >"${HELM_BINARY_PATH}"
- chmod a+x "${HELM_BINARY_PATH}"
+ tar -xvz -O "${SYSTEM}-amd64/helm" >"${HELM_PATH}"
+ chmod a+x "${HELM_PATH}"
else
echo "Helm version ok"
echo
@@ -127,7 +128,7 @@ function perform_kind_cluster_operation() {
ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell recreate ]"
set +u
- if [[ -z "${1=}" ]]; then
+ if [[ -z "${1}" ]]; then
echo >&2
echo >&2 "Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS}"
echo >&2
@@ -178,6 +179,11 @@ function perform_kind_cluster_operation() {
echo
echo "Deploying Airflow to KinD"
echo
+ get_environment_for_builds_on_ci
+ make_sure_kubernetes_tools_are_installed
+ initialize_kind_variables
+ prepare_prod_build
+ build_prod_images
build_image_for_kubernetes_tests
load_image_to_kind_cluster
deploy_airflow_with_helm
diff --git a/scripts/ci/libraries/_md5sum.sh b/scripts/ci/libraries/_md5sum.sh
index 0ef2d79..c2e1ff3 100644
--- a/scripts/ci/libraries/_md5sum.sh
+++ b/scripts/ci/libraries/_md5sum.sh
@@ -79,7 +79,7 @@ function update_all_md5_files() {
move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
done
mkdir -pv "${BUILD_CACHE_DIR}/${BRANCH_NAME}"
- touch "${BUILT_CI_IMAGE_FLAG_FILE}"
+ touch "${BUILT_IMAGE_FLAG_FILE}"
}
@@ -121,13 +121,13 @@ function check_if_docker_build_is_needed() {
if [[ ${FORCE_BUILD_IMAGES:=""} == "true" ]]; then
print_info "Docker image build is forced for ${THE_IMAGE_TYPE} image"
calculate_md5sum_for_all_files
- needs_docker_build="true"
+ export NEEDS_DOCKER_BUILD="true"
else
calculate_md5sum_for_all_files
if [[ ${FILES_MODIFIED} == "true" ]]; then
- needs_docker_build="true"
+ export NEEDS_DOCKER_BUILD="true"
fi
- if [[ ${needs_docker_build} == "true" ]]; then
+ if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
print_info "Docker image build is needed for ${THE_IMAGE_TYPE} image!"
else
print_info "Docker image build is not needed for ${THE_IMAGE_TYPE} image!"
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 6e8b1a1..ad13cb2 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -26,7 +26,7 @@ function pull_image_if_not_present_or_forced() {
IMAGE_HASH=$(docker images -q "${IMAGE_TO_PULL}" 2> /dev/null || true)
local PULL_IMAGE=${FORCE_PULL_IMAGES}
- if [[ -z "${IMAGE_HASH=}" ]]; then
+ if [[ "${IMAGE_HASH}" == "" ]]; then
PULL_IMAGE="true"
fi
if [[ "${PULL_IMAGE}" == "true" ]]; then
@@ -62,13 +62,15 @@ function pull_image_github_dockerhub() {
# Pulls CI image in case caching strategy is "pulled" and the image needs to be pulled
function pull_ci_images_if_needed() {
+ # Whether to force pull images to populate cache
+ export FORCE_PULL_IMAGES=${FORCE_PULL_IMAGES:="false"}
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
echo
echo "Force pull base image ${PYTHON_BASE_IMAGE}"
echo
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "
Docker pulling ${PYTHON_BASE_IMAGE}.
" > "${DETECTED_TERMINAL}"
@@ -95,6 +97,9 @@ Docker pulling ${PYTHON_BASE_IMAGE}.
# Pulls PROD image in case caching strategy is "pulled" and the image needs to be pulled
function pull_prod_images_if_needed() {
+ # Whether to force pull images to populate cache
+ export FORCE_PULL_IMAGES=${FORCE_PULL_IMAGES:="false"}
+
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
echo
@@ -128,7 +133,7 @@ function push_ci_images_to_dockerhub() {
docker push "${AIRFLOW_CI_IMAGE}"
docker tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
docker push "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
- if [[ -n ${DEFAULT_CI_IMAGE=} ]]; then
+ if [[ -n ${DEFAULT_CI_IMAGE:=""} ]]; then
# Only push default image to DockerHub registry if it is defined
docker push "${DEFAULT_CI_IMAGE}"
fi
@@ -143,7 +148,7 @@ function push_ci_images_to_github() {
AIRFLOW_CI_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}"
docker push "${AIRFLOW_CI_TAGGED_IMAGE}"
- if [[ -n ${GITHUB_SHA=} ]]; then
+ if [[ ${GITHUB_SHA:=} != "" ]]; then
# Also push image to GitHub registry with commit SHA
AIRFLOW_CI_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${COMMIT_SHA}"
docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_SHA_IMAGE}"
@@ -171,7 +176,7 @@ function push_ci_images() {
function push_prod_images_to_dockerhub () {
# Prod image
docker push "${AIRFLOW_PROD_IMAGE}"
- if [[ -n ${DEFAULT_PROD_IMAGE=} ]]; then
+ if [[ -n ${DEFAULT_PROD_IMAGE:=""} ]]; then
docker push "${DEFAULT_PROD_IMAGE}"
fi
# Prod build image
@@ -189,7 +194,7 @@ function push_prod_images_to_github () {
AIRFLOW_PROD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}"
docker push "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
- if [[ -n ${COMMIT_SHA=} ]]; then
+ if [[ ${COMMIT_SHA:=} != "" ]]; then
# Also push image to GitHub registry with commit SHA
AIRFLOW_PROD_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}"
docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_SHA_IMAGE}"
@@ -237,19 +242,23 @@ function wait_for_github_registry_image() {
GITHUB_API_CALL="${GITHUB_API_ENDPOINT}/${IMAGE_NAME}/manifests/${IMAGE_TAG}"
while true; do
- curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" 2>/dev/null > "${OUTPUT_LOG}"
- local digest
- digest=$(jq '.config.digest' < "${OUTPUT_LOG}")
- echo -n "."
- if [[ ${digest} != "null" ]]; then
- echo -e " \e[32mOK.\e[0m"
+ echo
+ echo "Calling ${GITHUB_API_CALL} with ${GITHUB_USERNAME}"
+ echo
+ curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" > "${OUTPUT_LOG}"
+ echo
+ DIGEST=$(jq '.config.digest' < "${OUTPUT_LOG}")
+ if [[ ${DIGEST} != "null" ]]; then
+ echo
+ echo "Found digest: '${DIGEST}'"
+ echo
break
fi
+ jq -C '.' < "${OUTPUT_LOG}"
sleep 10
done
echo
- echo
echo "Found ${IMAGE_NAME}:${IMAGE_TAG} image"
- echo "Digest: '${digest}'"
+ echo "Digest: '${DIGEST}'"
echo
}
diff --git a/scripts/ci/libraries/_sanity_checks.sh b/scripts/ci/libraries/_sanity_checks.sh
index b250cf2..d2bc01e 100644
--- a/scripts/ci/libraries/_sanity_checks.sh
+++ b/scripts/ci/libraries/_sanity_checks.sh
@@ -49,6 +49,25 @@ function sanitize_mounted_files() {
#
# Most useful is out.log file in this directory storing verbose output of the scripts.
#
+function create_cache_directory() {
+ CACHE_TMP_FILE_DIR=$(mktemp -d)
+ export CACHE_TMP_FILE_DIR
+
+ if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then
+ trap 'rm -rf -- "${CACHE_TMP_FILE_DIR}"' INT TERM HUP
+ fi
+
+ OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log"
+ export OUTPUT_LOG
+}
+
+# Removes the cache temporary directory
+function remove_cache_directory() {
+ if [[ -d ${CACHE_TMP_FILE_DIR} ]]; then
+ rm -rf "${CACHE_TMP_FILE_DIR}"
+ fi
+}
+
#
# Checks if core utils required in the host system are installed and explain what needs to be done if not
#
@@ -71,8 +90,6 @@ function check_if_coreutils_installed() {
set -e
CMDNAME="$(basename -- "$0")"
- export CMDNAME
- readonly CMDNAME
#################### Parsing options/arguments
if [[ ${GETOPT_RETVAL} != 4 || "${STAT_PRESENT}" != "0" || "${MD5SUM_PRESENT}" != "0" ]]; then
@@ -122,7 +139,7 @@ function assert_not_in_container() {
echo >&2 "You are inside the Airflow docker container!"
echo >&2 "You should only run this script from the host."
echo >&2 "Learn more about how we develop and test airflow in:"
- echo >&2 "https://github.com/apache/airflow/blob/master/TESTING.rst"
+ echo >&2 "https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst"
echo >&2
exit 1
fi
@@ -142,8 +159,8 @@ function go_to_airflow_sources {
#
function basic_sanity_checks() {
assert_not_in_container
- set_default_python_version_if_empty
go_to_airflow_sources
check_if_coreutils_installed
+ create_cache_directory
sanitize_mounted_files
}
diff --git a/scripts/ci/libraries/_script_init.sh b/scripts/ci/libraries/_script_init.sh
index 4e11e27..804037e 100755
--- a/scripts/ci/libraries/_script_init.sh
+++ b/scripts/ci/libraries/_script_init.sh
@@ -18,11 +18,18 @@
set -euo pipefail
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
-readonly AIRFLOW_SOURCES
+_CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+SCRIPTS_CI_DIR="$(cd "${_CURRENT_DIR}"/.. && pwd)"
+export SCRIPTS_CI_DIR
+
+# Sets to where airflow sources are located
+AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${SCRIPTS_CI_DIR}/../../" && pwd)}
+export AIRFLOW_SOURCES
+
# shellcheck source=scripts/ci/libraries/_all_libs.sh
-. "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
+. "${SCRIPTS_CI_DIR}"/libraries/_all_libs.sh
initialize_common_environment
@@ -30,12 +37,4 @@ basic_sanity_checks
script_start
-determine_docker_cache_strategy
-
-get_environment_for_builds_on_ci
-
-get_docker_image_names
-
-make_constants_read_only
-
-add_trap script_end EXIT HUP INT TERM
+trap script_end EXIT
diff --git a/scripts/ci/libraries/_start_end.sh b/scripts/ci/libraries/_start_end.sh
index 635ad88..4dcb150 100644
--- a/scripts/ci/libraries/_start_end.sh
+++ b/scripts/ci/libraries/_start_end.sh
@@ -79,4 +79,5 @@ function script_end {
print_info "Exit code ${EXIT_CODE}"
print_info
fi
+ remove_cache_directory
}
diff --git a/scripts/ci/libraries/_traps.sh b/scripts/ci/libraries/_traps.sh
deleted file mode 100644
index be2e31b..0000000
--- a/scripts/ci/libraries/_traps.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#######################################################################################################
-#
-# Adds trap to the traps already set.
-#
-# Arguments:
-# trap to set
-# .... list of signals to handle
-#######################################################################################################
-function add_trap() {
- trap="${1}"
- shift
- for signal in "${@}"
- do
- # adding trap to exiting trap
- local handlers
- handlers="$( trap -p "${signal}" | cut -f2 -d \' )"
- # shellcheck disable=SC2064
- trap "${handlers}${handlers:+;}${trap}" "${signal}"
- done
-}
diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh
index aa4f096..77d2fa1 100644
--- a/scripts/ci/libraries/_verbosity.sh
+++ b/scripts/ci/libraries/_verbosity.sh
@@ -24,13 +24,12 @@ function check_verbose_setup {
fi
}
-DOCKER_BINARY_PATH="${DOCKER_BINARY_PATH:=$(command -v docker || echo "/bin/docker")}"
-export DOCKER_BINARY_PATH
+DOCKER_BINARY="${DOCKER_BINARY:=$(command -v docker || echo "/bin/docker")}"
+export DOCKER_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be
# printed before execution
function docker {
- set +e
if [[ ${VERBOSE:="false"} == "true" && \
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
${VERBOSE_COMMANDS:=} != "true" && \
@@ -39,62 +38,55 @@ function docker {
>&2 echo "docker" "${@}"
fi
if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then
- ${DOCKER_BINARY_PATH} "${@}" >>"${OUTPUT_LOG}" 2>&1
+ ${DOCKER_BINARY} "${@}" >>"${OUTPUT_LOG}" 2>&1
else
- ${DOCKER_BINARY_PATH} "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
+ ${DOCKER_BINARY} "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
fi
- res="$?"
- if [[ ${res} == "0" ]]; then
+ EXIT_CODE="$?"
+ if [[ ${EXIT_CODE} == "0" ]]; then
# No matter if "set -e" is used the log will be removed on success.
# This way in the output log we only see the most recent failed command and what was echoed before
rm -f "${OUTPUT_LOG}"
fi
- set -e
- return ${res}
+ return "${EXIT_CODE}"
}
-HELM_BINARY_PATH="${HELM_BINARY_PATH:=$(command -v helm || echo "/bin/helm")}"
-export HELM_BINARY_PATH
+HELM_BINARY="${HELM_BINARY:=$(command -v helm || echo "/bin/helm")}"
+export HELM_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all helm commands run will be
# printed before execution
function helm {
- set +e
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "helm" "${@}"
fi
- ${HELM_BINARY_PATH} "${@}" | tee -a "${OUTPUT_LOG}"
- local res="$?"
- if [[ ${res} == "0" ]]; then
+ ${HELM_BINARY} "${@}" | tee -a "${OUTPUT_LOG}"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
rm -f "${OUTPUT_LOG}"
fi
- set -e
- return ${res}
}
-KUBECTL_BINARY_PATH=${KUBECTL_BINARY_PATH:=$(command -v kubectl || echo "/bin/kubectl")}
-export KUBECTL_BINARY_PATH
+KUBECTL_BINARY=${KUBECTL_BINARY:=$(command -v kubectl || echo "/bin/kubectl")}
+export KUBECTL_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kubectl commands run will be
# printed before execution
function kubectl {
- set +e
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "kubectl" "${@}"
fi
- ${KUBECTL_BINARY_PATH} "${@}" | tee -a "${OUTPUT_LOG}"
- local res="$?"
- if [[ ${res} == "0" ]]; then
+ ${KUBECTL_BINARY} "${@}" | tee -a "${OUTPUT_LOG}"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
rm -f "${OUTPUT_LOG}"
fi
- set -e
- return ${res}
}
-KIND_BINARY_PATH="${KIND_BINARY_PATH:=$(command -v kind || echo "/bin/kind")}"
-export KIND_BINARY_PATH
+KIND_BINARY="${KIND_BINARY:=$(command -v kind || echo "/bin/kind")}"
+export KIND_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kind commands run will be
# printed before execution
@@ -104,7 +96,7 @@ function kind {
>&2 echo "kind" "${@}"
fi
# kind outputs nice output on terminal.
- ${KIND_BINARY_PATH} "${@}"
+ ${KIND_BINARY} "${@}"
}
# Prints verbose information in case VERBOSE variable is set
diff --git a/scripts/ci/openapi/client_codegen_diff.sh b/scripts/ci/openapi/client_codegen_diff.sh
index 5260680..76ba677 100755
--- a/scripts/ci/openapi/client_codegen_diff.sh
+++ b/scripts/ci/openapi/client_codegen_diff.sh
@@ -17,6 +17,7 @@
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
set -e
TARGET_REMOTE=origin
diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
index 92ec950..f8651a9 100755
--- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
@@ -17,37 +17,21 @@
# under the License.
set -euo pipefail
PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export PRE_COMMIT_DIR
-readonly PRE_COMMIT_DIR
-
AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
-export AIRFLOW_SOURCES
-readonly AIRFLOW_SOURCES
-
cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
TMP_FILE=$(mktemp)
-export TMP_FILE
-readonly TMP_FILE
-
TMP_OUTPUT=$(mktemp)
-export TMP_OUTPUT
-readonly TMP_OUTPUT
echo "
.. code-block:: text
" >"${TMP_FILE}"
export MAX_SCREEN_WIDTH=100
-readonly MAX_SCREEN_WIDTH
-
export FORCE_SCREEN_WIDTH="true"
-readonly FORCE_SCREEN_WIDTH
-
export VERBOSE="false"
-readonly VERBOSE
./breeze help-all | sed 's/^/ /' | sed 's/ *$//' >>"${TMP_FILE}"
diff --git a/scripts/ci/pre_commit/pre_commit_check_integrations.sh b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
index abeee51..b204944 100755
--- a/scripts/ci/pre_commit/pre_commit_check_integrations.sh
+++ b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
@@ -16,6 +16,9 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
+PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
+cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
@@ -23,8 +26,6 @@ export SKIP_CHECK_REMOTE_IMAGE="true"
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-cd "${AIRFLOW_SOURCES}"
-
. breeze-complete
if [[ "${AVAILABLE_INTEGRATIONS} all" != "${_BREEZE_ALLOWED_INTEGRATIONS}" ]]; then
diff --git a/scripts/ci/pre_commit/pre_commit_ci_build.sh b/scripts/ci/pre_commit/pre_commit_ci_build.sh
index 646377e..56e7cbe 100755
--- a/scripts/ci/pre_commit/pre_commit_ci_build.sh
+++ b/scripts/ci/pre_commit/pre_commit_ci_build.sh
@@ -20,6 +20,7 @@ export REMEMBER_LAST_ANSWER="${2}"
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
index c24ee29..c687992 100755
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -15,16 +15,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -euo pipefail
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
TMP_OUTPUT=$(mktemp)
# Remove temp file if it's hanging around
-add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM
+trap 'rm -rf -- "${TMP_OUTPUT}" 2>/dev/null' EXIT
LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml"
diff --git a/scripts/ci/pre_commit/pre_commit_mermaid.sh b/scripts/ci/pre_commit/pre_commit_mermaid.sh
index b364511..fc52c91 100755
--- a/scripts/ci/pre_commit/pre_commit_mermaid.sh
+++ b/scripts/ci/pre_commit/pre_commit_mermaid.sh
@@ -16,11 +16,9 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
+AIRFLOW_SOURCES="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )"
export NO_TERMINAL_OUTPUT_FROM_SCRIPTS="true"
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
if ! command -v npm; then
echo 'You need to have npm installed in order to generate .mermaid graphs automatically.'
echo
@@ -28,7 +26,8 @@ if ! command -v npm; then
exit 0
fi
-tmp_file="${CACHE_TMP_FILE_DIR}/tmp.mermaid"
+tmp_file=$(mktemp)
+
cd "${AIRFLOW_SOURCES}"
MERMAID_INSTALLATION_DIR="${AIRFLOW_SOURCES}/.build/mermaid/"
@@ -43,7 +42,7 @@ else
fi
# shellcheck disable=SC2064
-add_trap "rm -rf '${tmp_file}'" EXIT HUP INT TERM
+trap "rm -rf ${tmp_file}" EXIT
for file in "${@}"
do
diff --git a/scripts/ci/static_checks/bat_tests.sh b/scripts/ci/static_checks/bat_tests.sh
index a5163d0..5b74885 100755
--- a/scripts/ci/static_checks/bat_tests.sh
+++ b/scripts/ci/static_checks/bat_tests.sh
@@ -16,6 +16,10 @@
# specific language governing permissions and limitations
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+initialize_common_environment
+
function run_bats_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
diff --git a/scripts/ci/static_checks/flake8.sh b/scripts/ci/static_checks/flake8.sh
index e14924e..f6613ab 100755
--- a/scripts/ci/static_checks/flake8.sh
+++ b/scripts/ci/static_checks/flake8.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -34,6 +36,8 @@ function run_flake8() {
fi
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/mypy.sh b/scripts/ci/static_checks/mypy.sh
index 47770c4..01e03cf 100755
--- a/scripts/ci/static_checks/mypy.sh
+++ b/scripts/ci/static_checks/mypy.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -31,6 +33,8 @@ function run_mypy() {
"--" "/opt/airflow/scripts/in_container/run_mypy.sh" "${FILES[@]}"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/pylint.sh b/scripts/ci/static_checks/pylint.sh
index 6d30f5c..dddb6d9 100755
--- a/scripts/ci/static_checks/pylint.sh
+++ b/scripts/ci/static_checks/pylint.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -33,6 +35,8 @@ function run_pylint() {
fi
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/refresh_pylint_todo.sh b/scripts/ci/static_checks/refresh_pylint_todo.sh
index 0473410..3657277 100755
--- a/scripts/ci/static_checks/refresh_pylint_todo.sh
+++ b/scripts/ci/static_checks/refresh_pylint_todo.sh
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export FORCE_ANSWER_TO_QUESTIONS="quit"
+export FORCE_ANSWER_TO_QUESTIONS=quit
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -26,6 +26,8 @@ function refresh_pylint_todo() {
/opt/airflow/scripts/in_container/refresh_pylint_todo.sh
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/run_static_checks.sh b/scripts/ci/static_checks/run_static_checks.sh
index 6400c8d..a5382e8 100755
--- a/scripts/ci/static_checks/run_static_checks.sh
+++ b/scripts/ci/static_checks/run_static_checks.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -25,6 +27,8 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
exit
fi
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index 9ef3c23..8cc0e3e 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export VERBOSE=${VERBOSE:="false"}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -25,6 +27,7 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
exit
fi
+
function run_airflow_testing_in_docker() {
set +u
set +e
@@ -73,6 +76,8 @@ function run_airflow_testing_in_docker() {
return "${EXIT_CODE}"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
@@ -116,7 +121,7 @@ if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml")
fi
-if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE} ]]; then
+if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml")
fi
diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh
index 7ef4873..2c62c74 100755
--- a/scripts/ci/tools/ci_clear_tmp.sh
+++ b/scripts/ci/tools/ci_clear_tmp.sh
@@ -15,12 +15,28 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
#
# Fixes ownership for files created inside container (files owned by root will be owned by host user)
#
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+export AIRFLOW_CI_IMAGE=\
+${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci
+
+export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE}
+export WEBSERVER_HOST_PORT=28080
+HOST_USER_ID="$(id -ur)"
+HOST_GROUP_ID="$(id -gr)"
+HOST_OS="$(uname -s)"
+
+export HOST_USER_ID
+export HOST_GROUP_ID
+export HOST_OS
+
docker-compose \
-f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
-f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \
diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh
index 94bcaf1..9a5a46f 100755
--- a/scripts/ci/tools/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_count_changed_files.sh
@@ -21,9 +21,12 @@
# Parameters
# $1: Revision to compare
# $2: Pattern to match
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
if [[ ${CI_EVENT_TYPE} == "push" ]]; then
echo
echo "Always run all tests on push"
diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh
index 975a29b..b534b20 100755
--- a/scripts/ci/tools/ci_fix_ownership.sh
+++ b/scripts/ci/tools/ci_fix_ownership.sh
@@ -19,9 +19,25 @@
#
# Fixes ownership for files created inside container (files owned by root will be owned by host user)
#
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+export AIRFLOW_CI_IMAGE=\
+${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci
+
+export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE}
+export WEBSERVER_HOST_PORT=28080
+HOST_USER_ID="$(id -ur)"
+HOST_GROUP_ID="$(id -gr)"
+HOST_OS="$(uname -s)"
+
+export HOST_USER_ID
+export HOST_GROUP_ID
+export HOST_OS
+export BACKEND="sqlite"
+
docker-compose \
-f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
-f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \
diff --git a/scripts/in_container/_in_container_script_init.sh b/scripts/in_container/_in_container_script_init.sh
index 2abb02a..76aa32a 100755
--- a/scripts/in_container/_in_container_script_init.sh
+++ b/scripts/in_container/_in_container_script_init.sh
@@ -28,4 +28,4 @@ in_container_basic_sanity_check
in_container_script_start
-add_trap in_container_script_end EXIT HUP INT TERM
+trap in_container_script_end EXIT
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 6ede33f..fd058a5 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -16,27 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-#######################################################################################################
-#
-# Adds trap to the traps already set.
-#
-# Arguments:
-# trap to set
-# .... list of signals to handle
-#######################################################################################################
-function add_trap() {
- trap="${1}"
- shift
- for signal in "${@}"
- do
- # adding trap to exiting trap
- local handlers
- handlers="$( trap -p "${signal}" | cut -f2 -d \' )"
- # shellcheck disable=SC2064
- trap "${handlers}${handlers:+;}${trap}" "${signal}"
- done
-}
-
function assert_in_container() {
export VERBOSE=${VERBOSE:="false"}
if [[ ! -f /.dockerenv ]]; then
@@ -61,7 +40,7 @@ function in_container_script_end() {
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
if [[ "${PRINT_INFO_FROM_SCRIPTS=="true"}" == "true" ]] ;then
- if [[ -n ${OUT_FILE_PRINTED_ON_ERROR=} ]]; then
+ if [[ -n ${OUT_FILE_PRINTED_ON_ERROR:=} ]]; then
echo " ERROR ENCOUNTERED!"
echo
echo " Output:"
@@ -271,7 +250,7 @@ function setup_kerberos() {
}
function dump_airflow_logs() {
- DUMP_FILE=/files/airflow_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}.log.tar.gz
+ DUMP_FILE=/files/airflow_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}.log.tar.gz
echo "###########################################################################################"
echo " Dumping logs from all the airflow tasks"
echo "###########################################################################################"
diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh
index 31ee76f..0a1fd14 100755
--- a/scripts/in_container/check_environment.sh
+++ b/scripts/in_container/check_environment.sh
@@ -101,7 +101,7 @@ function resetdb_if_requested() {
echo "==============================================================================================="
echo " Checking integrations and backends"
echo "==============================================================================================="
-if [[ -n ${BACKEND=} ]]; then
+if [[ -n ${BACKEND:=} ]]; then
check_db_backend 20
echo "-----------------------------------------------------------------------------------------------"
fi
@@ -125,7 +125,7 @@ fi
resetdb_if_requested
-if [[ -n ${DISABLED_INTEGRATIONS=} ]]; then
+if [[ ${DISABLED_INTEGRATIONS} != "" ]]; then
echo
echo "Disabled integrations:${DISABLED_INTEGRATIONS}"
echo
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index 25fb480..302d30c 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -35,7 +35,7 @@ echo
echo "Airflow home: ${AIRFLOW_HOME}"
echo "Airflow sources: ${AIRFLOW_SOURCES}"
echo "Airflow core SQL connection: ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}"
-if [[ -n "${AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS=}" ]]; then
+if [[ -n "${AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS:=}" ]]; then
echo "Airflow collation for IDs: ${AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS}"
fi
@@ -57,7 +57,7 @@ else
export RUN_AIRFLOW_1_10="false"
fi
-if [[ -z ${INSTALL_AIRFLOW_VERSION=} ]]; then
+if [[ ${INSTALL_AIRFLOW_VERSION} == "" ]]; then
if [[ ! -d "${AIRFLOW_SOURCES}/airflow/www/node_modules" ]]; then
echo
echo "Installing node modules as they are not yet installed (Sources mounted from Host)"
@@ -192,7 +192,7 @@ if [[ ${#@} -gt 0 && -n "$1" ]]; then
TESTS_TO_RUN=("${@}")
fi
-if [[ -n ${RUN_INTEGRATION_TESTS=} ]]; then
+if [[ -n ${RUN_INTEGRATION_TESTS:=""} ]]; then
# Integration tests
for INT in ${RUN_INTEGRATION_TESTS}
do
diff --git a/scripts/in_container/prod/entrypoint_prod.sh b/scripts/in_container/prod/entrypoint_prod.sh
index 98708b7..5e3b241 100755
--- a/scripts/in_container/prod/entrypoint_prod.sh
+++ b/scripts/in_container/prod/entrypoint_prod.sh
@@ -44,7 +44,7 @@ function verify_db_connection {
echo DB_BACKEND="${DB_BACKEND:=${DETECTED_DB_BACKEND}}"
- if [[ -z "${DETECTED_DB_PORT=}" ]]; then
+ if [[ -z "${DETECTED_DB_PORT}" ]]; then
if [[ ${DB_BACKEND} == "postgres"* ]]; then
DETECTED_DB_PORT=5432
elif [[ ${DB_BACKEND} == "mysql"* ]]; then
@@ -106,7 +106,7 @@ verify_db_connection "${AIRFLOW__CORE__SQL_ALCHEMY_CONN}"
AIRFLOW__CELERY__BROKER_URL=${AIRFLOW__CELERY__BROKER_URL:=}
-if [[ -n ${AIRFLOW__CELERY__BROKER_URL=} ]] && \
+if [[ -n ${AIRFLOW__CELERY__BROKER_URL} ]] && \
[[ ${AIRFLOW_COMMAND} =~ ^(scheduler|worker|flower)$ ]]; then
verify_db_connection "${AIRFLOW__CELERY__BROKER_URL}"
fi
diff --git a/scripts/in_container/run_cli_tool.sh b/scripts/in_container/run_cli_tool.sh
index 706098e..2093373 100755
--- a/scripts/in_container/run_cli_tool.sh
+++ b/scripts/in_container/run_cli_tool.sh
@@ -18,19 +18,19 @@
# under the License.
set -euo pipefail
-if [ -z "${AIRFLOW_CI_IMAGE=}" ]; then
+if [ -z "${AIRFLOW_CI_IMAGE}" ]; then
>&2 echo "Missing environment variable AIRFLOW_CI_IMAGE"
exit 1
fi
-if [ -z "${HOST_AIRFLOW_SOURCES=}" ]; then
+if [ -z "${HOST_AIRFLOW_SOURCES}" ]; then
>&2 echo "Missing environment variable HOST_AIRFLOW_SOURCES"
exit 1
fi
-if [ -z "${HOST_USER_ID=}" ]; then
+if [ -z "${HOST_USER_ID}" ]; then
>&2 echo "Missing environment variable HOST_USER_ID"
exit 1
fi
-if [ -z "${HOST_GROUP_ID=}" ]; then
+if [ -z "${HOST_GROUP_ID}" ]; then
>&2 echo "Missing environment variable HOST_GROUP_ID"
exit 1
fi
@@ -65,7 +65,7 @@ env > "${ENV_TMP_FILE}"
cleanup() {
rm "${ENV_TMP_FILE}"
}
-trap cleanup EXIT HUP INT TERM
+trap cleanup EXIT
CONTAINER_ID="$(head -n 1 < /proc/self/cgroup | cut -d ":" -f 3 | cut -d "/" -f 3)"
diff --git a/scripts/in_container/run_docs_build.sh b/scripts/in_container/run_docs_build.sh
index 1304ee8..c98cb6e 100755
--- a/scripts/in_container/run_docs_build.sh
+++ b/scripts/in_container/run_docs_build.sh
@@ -18,7 +18,10 @@
# shellcheck source=scripts/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
sudo rm -rf "${AIRFLOW_SOURCES}/docs/_build/*"
sudo rm -rf "${AIRFLOW_SOURCES}/docs/_api/*"
diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh
index d361b0e..484c472 100755
--- a/scripts/in_container/run_generate_constraints.sh
+++ b/scripts/in_container/run_generate_constraints.sh
@@ -18,7 +18,10 @@
# shellcheck source=scripts/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
CONSTRAINTS_DIR="/files/constraints-${PYTHON_MAJOR_MINOR_VERSION}"
diff --git a/scripts/in_container/run_prepare_backport_packages.sh b/scripts/in_container/run_prepare_backport_packages.sh
index 4911f81..6ebef75 100755
--- a/scripts/in_container/run_prepare_backport_packages.sh
+++ b/scripts/in_container/run_prepare_backport_packages.sh
@@ -20,7 +20,11 @@
LIST_OF_DIRS_FILE=$(mktemp)
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
+
cd "${AIRFLOW_SOURCES}/airflow/providers" || exit 1
find . -type d | sed 's/.\///; s/\//\./g' | grep -E 'hooks|operators|sensors|secrets' \
@@ -153,7 +157,7 @@ cd "${AIRFLOW_SOURCES}" || exit 1
pushd dist
-if [[ -n ${VERSION_SUFFIX_FOR_SVN=} ]]; then
+if [[ ${VERSION_SUFFIX_FOR_SVN} != "" ]]; then
for FILE in *.tar.gz
do
mv "${FILE}" "${FILE//\.tar\.gz/${VERSION_SUFFIX_FOR_SVN}-bin.tar.gz}"
diff --git a/scripts/in_container/run_prepare_backport_readme.sh b/scripts/in_container/run_prepare_backport_readme.sh
index eeeee8d..bdb8edc 100755
--- a/scripts/in_container/run_prepare_backport_readme.sh
+++ b/scripts/in_container/run_prepare_backport_readme.sh
@@ -20,7 +20,10 @@
OUT_FILE_PRINTED_ON_ERROR=$(mktemp)
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
cd "${AIRFLOW_SOURCES}" || exit 1
diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash
index 2df8786..e2f174e 100644
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/bats_utils.bash
@@ -17,17 +17,12 @@
AIRFLOW_SOURCES=$(pwd)
export AIRFLOW_SOURCES
-readonly AIRFLOW_SOURCES
+export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-export DOCKER_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/docker.sh
-export KUBECTL_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/kubectl.sh
-export KIND_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/kind.sh
-export HELM_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/helm.sh
-export SKIP_IN_CONTAINER_CHECK="true"
+export DOCKER_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/docker.sh
+export KUBECTL_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/kubectl.sh
+export KIND_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/kind.sh
+export HELM_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/helm.sh
# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "scripts/ci/libraries/_all_libs.sh"
-
-initialize_common_environment
-
-basic_sanity_checks
+source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
diff --git a/tests/bats/test_breeze_params.bats b/tests/bats/test_breeze_params.bats
index 0795ee2..7d051b4 100644
--- a/tests/bats/test_breeze_params.bats
+++ b/tests/bats/test_breeze_params.bats
@@ -25,10 +25,12 @@ setup() {
}
teardown() {
+ load bats_utils
rm -f "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
}
@test "Test missing value for a parameter" {
+ load bats_utils
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") - <<EOF
@@ -41,6 +43,10 @@ EOF
}
@test "Test wrong value for a parameter but proper stored in the .build/PARAM" {
+ load bats_utils
+
+ initialize_common_environment
+
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=x
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
@@ -57,6 +63,10 @@ EOF
}
@test "Test wrong value for a parameter stored in the .build/PARAM" {
+ load bats_utils
+
+ initialize_common_environment
+
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=x
echo "x" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
@@ -75,6 +85,10 @@ EOF
@test "Test correct value for a parameter" {
+ load bats_utils
+
+ initialize_common_environment
+
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=a
run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
@@ -85,6 +99,10 @@ EOF
}
@test "Test correct value for a parameter from multi-line values" {
+ load bats_utils
+
+ initialize_common_environment
+
_BREEZE_ALLOWED_TEST_PARAMS=$(cat <<-EOF
a
b
@@ -102,6 +120,10 @@ EOF
@test "Test read_parameter from missing file" {
+ load bats_utils
+
+ initialize_common_environment
+
run read_from_file TEST_PARAM
[ -z "${TEST_FILE}" ]
diff <(echo "${output}") <(echo "")
@@ -110,6 +132,10 @@ EOF
}
@test "Test read_parameter from file" {
+ load bats_utils
+
+ initialize_common_environment
+
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
run read_from_file TEST_PARAM
diff <(echo "${output}") <(echo "a")
diff --git a/tests/bats/test_empty_test.bats b/tests/bats/test_empty_test.bats
index 2f79819..308d78d 100644
--- a/tests/bats/test_empty_test.bats
+++ b/tests/bats/test_empty_test.bats
@@ -20,6 +20,8 @@
@test "empty test" {
load bats_utils
+ initialize_common_environment
+
run pwd
[ "${status}" == 0 ]
}
diff --git a/tests/bats/test_local_mounts.bats b/tests/bats/test_local_mounts.bats
index aef961f..97cf984 100644
--- a/tests/bats/test_local_mounts.bats
+++ b/tests/bats/test_local_mounts.bats
@@ -21,6 +21,8 @@
@test "convert volume list to docker params" {
load bats_utils
+ initialize_common_environment
+
read -r -a RES <<< "$(convert_local_mounts_to_docker_params)"
[[ ${#RES[@]} -gt 0 ]] # Array should be non-zero length
[airflow] 01/02: Revert "Add packages to function names in bash
(#10670)"
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch revert-breeze
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d9c4263ad40054ca6b0aa33532f19fc2d087f1fd
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Wed Sep 2 17:01:26 2020 +0100
Revert "Add packages to function names in bash (#10670)"
This reverts commit cc551ba793344800d2d396c13d7fd0c8eed97352.
---
BREEZE.rst | 2 +-
breeze | 426 ++++++++++-----------
.../ci_prepare_backport_packages.sh | 6 +-
.../ci_prepare_backport_readme.sh | 6 +-
...ci_test_backport_packages_import_all_classes.sh | 4 +-
...ci_test_backport_packages_install_separately.sh | 4 +-
scripts/ci/constraints/ci_generate_constraints.sh | 6 +-
scripts/ci/docs/ci_docs.sh | 6 +-
scripts/ci/images/ci_build_dockerhub.sh | 12 +-
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 12 +-
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 8 +-
scripts/ci/images/ci_push_ci_images.sh | 4 +-
scripts/ci/images/ci_push_production_images.sh | 4 +-
scripts/ci/images/ci_wait_for_all_ci_images.sh | 4 +-
scripts/ci/images/ci_wait_for_all_prod_images.sh | 6 +-
.../ci/kubernetes/ci_deploy_app_to_kubernetes.sh | 20 +-
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 6 +-
scripts/ci/libraries/_build_images.sh | 190 ++++-----
scripts/ci/libraries/_initialization.sh | 102 ++---
scripts/ci/libraries/_kind.sh | 48 +--
scripts/ci/libraries/_local_mounts.sh | 6 +-
scripts/ci/libraries/_md5sum.sh | 44 +--
scripts/ci/libraries/_parameters.sh | 8 +-
scripts/ci/libraries/_permissions.sh | 22 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 52 +--
scripts/ci/libraries/_pylint.sh | 2 +-
scripts/ci/libraries/_runs.sh | 8 +-
scripts/ci/libraries/_sanity_checks.sh | 40 +-
scripts/ci/libraries/_script_init.sh | 16 +-
scripts/ci/libraries/_spinner.sh | 2 +-
scripts/ci/libraries/_start_end.sh | 44 +--
scripts/ci/libraries/_traps.sh | 2 +-
scripts/ci/libraries/_verbosity.sh | 13 +-
scripts/ci/pre_commit/pre_commit_ci_build.sh | 6 +-
.../ci/pre_commit/pre_commit_local_yml_mounts.sh | 6 +-
scripts/ci/pre_commit/pre_commit_mermaid.sh | 2 +-
scripts/ci/static_checks/flake8.sh | 4 +-
scripts/ci/static_checks/mypy.sh | 4 +-
scripts/ci/static_checks/pylint.sh | 6 +-
scripts/ci/static_checks/refresh_pylint_todo.sh | 4 +-
scripts/ci/static_checks/run_static_checks.sh | 4 +-
scripts/ci/testing/ci_run_airflow_testing.sh | 4 +-
tests/bats/bats_utils.bash | 4 +-
tests/bats/test_breeze_params.bats | 14 +-
tests/bats/test_local_mounts.bats | 2 +-
45 files changed, 601 insertions(+), 594 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 01dbc0b..5d87345 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1046,7 +1046,7 @@ This is the current syntax for `./breeze <./breeze>`_:
####################################################################################################
- usage: breeze [FLAGS] [COMMAND] -- <EXTRA_ARGS>
+ Usage: breeze [FLAGS] [COMMAND] -- <EXTRA_ARGS>
By default the script enters the CI container and drops you to bash shell, but you can choose
one of the commands to run specific actions instead.
diff --git a/breeze b/breeze
index 7230c6e..d03e556 100755
--- a/breeze
+++ b/breeze
@@ -78,7 +78,7 @@ export EXTRA_STATIC_CHECK_OPTIONS
# scripts/ci/libraries/_initialization.sh and breeze-complete script (which sets-up auto-complete).
#
#######################################################################################################
-function breeze::setup_default_breeze_constants() {
+function setup_default_breeze_constants() {
# Indicates that we are inside Breeze environment
export BREEZE=true
readonly BREEZE
@@ -183,7 +183,7 @@ function breeze::setup_default_breeze_constants() {
# OSTYPE
#
#######################################################################################################
-function breeze::initialize_virtualenv() {
+function initialize_virtualenv() {
# Check if we are inside virtualenv
set +e
echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" |
@@ -256,7 +256,7 @@ function breeze::initialize_virtualenv() {
# OSTYPE
#
#######################################################################################################
-function breeze::setup_autocomplete() {
+function setup_autocomplete() {
echo "Installing bash/zsh completion for local user"
echo
"${AIRFLOW_SOURCES}/confirm" "This will create ~/.bash_completion.d/ directory and modify ~/.*rc files"
@@ -340,7 +340,7 @@ EOF
#
# Prints information about the current configuration of Breeze - if you enter breeze interactively
# and you did not suppress cheatsheet or asciiart, it also prints those. It also prints values
-# of constants set by breeze::read_saved_environment_variables() function and other initialization functions.
+# of constants set by read_saved_environment_variables() function and other initialization functions.
#
# Used globals:
#
@@ -363,7 +363,7 @@ EOF
# Prints the information about the build to stdout.
#
#######################################################################################################
-function breeze::print_badge() {
+function print_badge() {
local backend_version=""
if [[ ${BACKEND} == "postgres" ]]; then
backend_version="${POSTGRES_VERSION}"
@@ -421,7 +421,7 @@ EOF
Branch name: ${BRANCH_NAME}
Docker image: ${AIRFLOW_PROD_IMAGE}
- Airflow source version: $(build_images::get_airflow_version_from_production_image)
+ Airflow source version: $(get_airflow_version_from_production_image)
EOF
else
cat <<EOF
@@ -526,7 +526,7 @@ EOF
# Creates the convenience command file that can be run to use the docker command.
#
#######################################################################################################
-function breeze::prepare_command_file() {
+function prepare_command_file() {
local file="${1}"
local command="${2}"
local compose_file="${3}"
@@ -588,7 +588,7 @@ EOF
# BUILT_CI_IMAGE_FLAG_FILE
#
#######################################################################################################
-function breeze::prepare_command_files() {
+function prepare_command_files() {
local main_ci_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
local main_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
@@ -638,11 +638,11 @@ function breeze::prepare_command_files() {
readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD
# Prepare script for "run docker compose CI command"
- breeze::prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" \
+ prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" \
"\"\${@}\"" "${compose_ci_file}" "${AIRFLOW_CI_IMAGE}"
# Prepare script for "run docker compose PROD command"
- breeze::prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}" \
+ prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}" \
"\"\${@}\"" "${compose_prod_file}" "${AIRFLOW_PROD_IMAGE}"
}
@@ -658,25 +658,25 @@ function breeze::prepare_command_files() {
# Prints detailed help for all commands to stdout.
#
#######################################################################################################
-function breeze::do_help_all() {
+function do_help_all() {
echo
- breeze::print_line
- breeze::usage
- breeze::print_line
+ print_line
+ usage
+ print_line
echo
echo
echo "Detailed usage"
echo
- breeze::print_line
+ print_line
echo
local subcommand
for subcommand in ${ALL_BREEZE_COMMANDS}; do
- breeze::detailed_usage "${subcommand}"
- breeze::print_line
+ detailed_usage "${subcommand}"
+ print_line
echo
done
echo
- breeze::flags
+ flags
}
#######################################################################################################
@@ -684,11 +684,11 @@ function breeze::do_help_all() {
# Parses all arguments that can be passed to Breeze command - that includes command to run and flags.
#
# Updated global constants:
-# By the end of this function, all the constants from `initialization::make_constants_read_only` function are set
+# By the end of this function, all the constants from `make_constants_read_only` function are set
# and they are set as read-only.
#
#######################################################################################################
-function breeze::parse_arguments() {
+function parse_arguments() {
set -u
local params
if ! params=$(getopt \
@@ -725,7 +725,7 @@ function breeze::parse_arguments() {
;;
-i | --integration)
local integration=${2}
- parameters::check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
+ check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
echo "Integration: ${integration}"
if [[ ${integration} == "all" ]]; then
for integration in ${_BREEZE_ALLOWED_INTEGRATIONS}; do
@@ -1185,15 +1185,15 @@ function breeze::parse_arguments() {
exit 0
;;
help)
- breeze::usage
+ usage
exit 0
;;
help-all)
- breeze::do_help_all
+ do_help_all
exit 0
;;
*)
- breeze::usage
+ usage
echo >&2
echo >&2 "ERROR: Unknown command ${1}"
echo >&2
@@ -1207,10 +1207,10 @@ function breeze::parse_arguments() {
if [[ ${run_help} == "true" ]]; then
if [[ ${last_subcommand} == "" ]]; then
- breeze::usage
- breeze::flag_footer
+ usage
+ flag_footer
else
- breeze::detailed_usage "${last_subcommand}"
+ detailed_usage "${last_subcommand}"
fi
exit 0
fi
@@ -1234,7 +1234,7 @@ function breeze::parse_arguments() {
# FORMATTED_* constant variables that can be used in Breeze Help output
#
#######################################################################################################
-function breeze::prepare_formatted_versions() {
+function prepare_formatted_versions() {
local indent=15
local list_prefix
list_prefix=$(printf "%-${indent}s" " ")
@@ -1310,7 +1310,7 @@ function breeze::prepare_formatted_versions() {
# shellcheck disable=SC2034,SC2090,SC2089,SC2155
-function breeze::prepare_usage() {
+function prepare_usage() {
# Note that MacOS uses Bash 3.* and we cannot use associative arrays
export USAGE_SHELL="[Default] Enters interactive shell in the container"
readonly USAGE_SHELL
@@ -1397,7 +1397,7 @@ ${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
--github-image-id 209845560' - pull/use image with RUN_ID
Flags:
-$(breeze::flag_footer)
+$(flag_footer)
"
readonly DETAILED_USAGE_SHELL
export DETAILED_USAGE_EXEC="
@@ -1437,12 +1437,12 @@ ${CMDNAME} build-image [FLAGS]
specific COMMIT_SHA tag or RUN_ID.
Flags:
-$(breeze::flag_airflow_variants)
-$(breeze::flag_choose_different_airflow_version)
-$(breeze::flag_production_image)
-$(breeze::flag_build_docker_images)
-$(breeze::flag_pull_push_docker_images)
-$(breeze::flag_verbosity)
+$(flag_airflow_variants)
+$(flag_choose_different_airflow_version)
+$(flag_production_image)
+$(flag_build_docker_images)
+$(flag_pull_push_docker_images)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_BUILD_IMAGE
export DETAILED_USAGE_CLEANUP_IMAGE="
@@ -1453,9 +1453,9 @@ ${CMDNAME} cleanup-image [FLAGS]
with --all) to reclaim that space.
Flags:
-$(breeze::flag_airflow_variants)
-$(breeze::flag_production_image)
-$(breeze::flag_verbosity)
+$(flag_airflow_variants)
+$(flag_production_image)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_CLEANUP_IMAGE
export DETAILED_USAGE_DOCKER_COMPOSE="
@@ -1468,9 +1468,9 @@ ${CMDNAME} docker-compose [FLAGS] COMMAND [-- <EXTRA_ARGS>]
'${CMDNAME} docker-compose pull -- --ignore-pull-failures'
Flags:
-$(breeze::flag_airflow_variants)
-$(breeze::flag_backend_variants)
-$(breeze::flag_verbosity)
+$(flag_airflow_variants)
+$(flag_backend_variants)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_DOCKER_COMPOSE
export DETAILED_USAGE_FLAGS="
@@ -1505,7 +1505,7 @@ ${CMDNAME} prepare-backport-packages [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
'.' for example 'apache.hive'
Flags:
-$(breeze::flag_verbosity)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_PREPARE_BACKPORT_README
export DETAILED_USAGE_GENERATE_CONSTRAINTS="
@@ -1520,8 +1520,8 @@ ${CMDNAME} generate-constraints [FLAGS]
CRON job in CI automatically.
Flags:
-$(breeze::flag_airflow_variants)
-$(breeze::flag_verbosity)
+$(flag_airflow_variants)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_GENERATE_CONSTRAINTS
export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV="
@@ -1534,7 +1534,7 @@ ${CMDNAME} initialize-local-virtualenv [FLAGS]
activated before running this command.
Flags:
-$(breeze::flag_airflow_variants)
+$(flag_airflow_variants)
"
readonly DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV
export DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES="
@@ -1562,8 +1562,8 @@ ${CMDNAME} prepare-backport-packages [FLAGS] [PACKAGE_ID ...]
for example 'apache.hive'
Flags:
-$(breeze::flag_version_suffix)
-$(breeze::flag_verbosity)
+$(flag_version_suffix)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES
export DETAILED_USAGE_PUSH_IMAGE="
@@ -1595,8 +1595,8 @@ ${CMDNAME} push_image [FLAGS]
--github-image-id 209845560' - to push with RUN_ID
Flags:
-$(breeze::flag_pull_push_docker_images)
-$(breeze::flag_verbosity)
+$(flag_pull_push_docker_images)
+$(flag_verbosity)
"
readonly DETAILED_USAGE_PUSH_IMAGE
export DETAILED_USAGE_KIND_CLUSTER="
@@ -1612,8 +1612,8 @@ ${CMDNAME} kind-cluster [FLAGS] OPERATION
${FORMATTED_KIND_OPERATIONS}
Flags:
-$(breeze::flag_airflow_variants)
-$(breeze::flag_build_docker_images)
+$(flag_airflow_variants)
+$(flag_build_docker_images)
"
readonly DETAILED_USAGE_KIND_CLUSTER
export DETAILED_USAGE_SETUP_AUTOCOMPLETE="
@@ -1640,7 +1640,7 @@ ${CMDNAME} restart [FLAGS]
especially useful if you switch between different versions of Airflow.
Flags:
-$(breeze::flag_footer)
+$(flag_footer)
"
readonly DETAILED_USAGE_RESTART
export DETAILED_USAGE_STATIC_CHECK="
@@ -1675,7 +1675,7 @@ ${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
'${CMDNAME} tests tests
Flags:
-$(breeze::flag_footer)
+$(flag_footer)
"
readonly DETAILED_USAGE_TESTS
export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET="
@@ -1715,7 +1715,7 @@ ${CMDNAME} help-all
# Outputs:
# Writes the capitalized name of the variable to stdout
#######################################################################################################
-function breeze::get_variable_from_lowercase_name() {
+function get_variable_from_lowercase_name() {
local prefix="${1}"
local name="${2}"
local suffix
@@ -1730,10 +1730,10 @@ function breeze::get_variable_from_lowercase_name() {
# Arguments:
# lowercase command name
# Outputs:
-# usage information for the command.
+# Usage information for the command.
#######################################################################################################
-function breeze::get_usage() {
- breeze::get_variable_from_lowercase_name "USAGE" "${1}"
+function get_usage() {
+ get_variable_from_lowercase_name "USAGE" "${1}"
}
#######################################################################################################
@@ -1744,8 +1744,8 @@ function breeze::get_usage() {
# Outputs:
# Detailed usage information for the command.
#######################################################################################################
-function breeze::get_detailed_usage() {
- breeze::get_variable_from_lowercase_name "DETAILED_USAGE" "${1}"
+function get_detailed_usage() {
+ get_variable_from_lowercase_name "DETAILED_USAGE" "${1}"
}
#######################################################################################################
@@ -1761,10 +1761,10 @@ function breeze::get_detailed_usage() {
# Outputs:
# General usage information for all commands.
#######################################################################################################
-function breeze::usage() {
+function usage() {
echo "
-usage: ${CMDNAME} [FLAGS] [COMMAND] -- <EXTRA_ARGS>
+Usage: ${CMDNAME} [FLAGS] [COMMAND] -- <EXTRA_ARGS>
By default the script enters the CI container and drops you to bash shell, but you can choose
one of the commands to run specific actions instead.
@@ -1774,19 +1774,19 @@ Add --help after each command to see details:
Commands without arguments:
"
for subcommand in ${BREEZE_COMMANDS}; do
- printf " %-40s %s\n" "${subcommand}" "$(breeze::get_usage "${subcommand}")"
+ printf " %-40s %s\n" "${subcommand}" "$(get_usage "${subcommand}")"
done
echo "
Commands with arguments:
"
for subcommand in ${BREEZE_EXTRA_ARG_COMMANDS}; do
- printf " %-30s%-10s %s\n" "${subcommand}" "<ARG>" "$(breeze::get_usage "${subcommand}")"
+ printf " %-30s%-10s %s\n" "${subcommand}" "<ARG>" "$(get_usage "${subcommand}")"
done
echo "
Help commands:
"
for subcommand in ${BREEZE_HELP_COMMANDS}; do
- printf " %-40s %s\n" "${subcommand}" "$(breeze::get_usage "${subcommand}")"
+ printf " %-40s %s\n" "${subcommand}" "$(get_usage "${subcommand}")"
done
echo
}
@@ -1802,13 +1802,13 @@ Help commands:
# Detailed usage information for the command
#######################################################################################################
# Prints detailed usage for command specified
-function breeze::detailed_usage() {
+function detailed_usage() {
subcommand=${1}
echo "
Detailed usage for command: ${subcommand}
-$(breeze::get_detailed_usage "${subcommand}")
+$(get_detailed_usage "${subcommand}")
"
}
@@ -1820,7 +1820,7 @@ $(breeze::get_detailed_usage "${subcommand}")
# Outputs:
# Footer common for all commands.
#######################################################################################################
-function breeze::flag_footer() {
+function flag_footer() {
echo "
Run '${CMDNAME} flags' to see all applicable flags.
"
@@ -1836,7 +1836,7 @@ Run '${CMDNAME} flags' to see all applicable flags.
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_airflow_variants() {
+function flag_airflow_variants() {
echo "
-p, --python <PYTHON_MAJOR_MINOR_VERSION>
Python version used for the image. This is always major/minor version.
@@ -1862,7 +1862,7 @@ ${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS}
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_backend_variants() {
+function flag_backend_variants() {
echo "
-b, --backend <BACKEND>
Backend to use for tests - it determines which database is used.
@@ -1891,7 +1891,7 @@ ${FORMATTED_MYSQL_VERSIONS}
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_production_image() {
+function flag_production_image() {
echo "
-I, --production-image
Use production image for entering the environment and builds (not for tests).
@@ -1909,7 +1909,7 @@ function breeze::flag_production_image() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_breeze_actions() {
+function flag_breeze_actions() {
echo "
-d, --db-reset
Resets the database at entry to the environment. It will drop all the tables
@@ -1942,7 +1942,7 @@ ${FORMATTED_INTEGRATIONS}
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_kubernetes_configuration() {
+function flag_kubernetes_configuration() {
echo "
Configuration for the KinD Kubernetes cluster and tests:
@@ -1988,7 +1988,7 @@ ${FORMATTED_HELM_VERSIONS}
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_local_file_mounting() {
+function flag_local_file_mounting() {
echo "
-l, --skip-mounting-local-sources
Skips mounting local volume with sources - you get exactly what is in the
@@ -2007,7 +2007,7 @@ function breeze::flag_local_file_mounting() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_choose_different_airflow_version() {
+function flag_choose_different_airflow_version() {
echo "
-a, --install-airflow-version <INSTALL_AIRFLOW_VERSION>
If specified, installs Airflow directly from PIP released version. This happens at
@@ -2028,7 +2028,7 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS}
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_assume_answers_to_questions() {
+function flag_assume_answers_to_questions() {
echo "
-y, --assume-yes
Assume 'yes' answer to all questions.
@@ -2048,7 +2048,7 @@ function breeze::flag_assume_answers_to_questions() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_credentials() {
+function flag_credentials() {
echo "
-f, --forward-credentials
Forwards host credentials to docker container. Use with care as it will make
@@ -2063,7 +2063,7 @@ function breeze::flag_credentials() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_verbosity() {
+function flag_verbosity() {
echo "
-v, --verbose
Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
@@ -2082,7 +2082,7 @@ function breeze::flag_verbosity() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_help() {
+function flag_help() {
echo "
-h, --help
Shows detailed help message for the command specified.
@@ -2100,7 +2100,7 @@ function breeze::flag_help() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_build_docker_images() {
+function flag_build_docker_images() {
echo "
-F, --force-build-images
Forces building of the local docker images. The images are rebuilt
@@ -2183,7 +2183,7 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_pull_push_docker_images() {
+function flag_pull_push_docker_images() {
echo "
-D, --dockerhub-user
DockerHub user used to pull, push and build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_USER:=}.
@@ -2223,7 +2223,7 @@ function breeze::flag_pull_push_docker_images() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flag_version_suffix() {
+function flag_version_suffix() {
echo "
-S, --version-suffix-for-pypi
Adds optional suffix to the version in the generated backport package. It can be used
@@ -2242,67 +2242,67 @@ function breeze::flag_version_suffix() {
# Outputs:
# Flag information.
#######################################################################################################
-function breeze::flags() {
+function flags() {
echo "
-$(breeze::print_line)
+$(print_line)
Summary of all flags supported by Breeze:
-$(breeze::print_star_line)
+$(print_star_line)
Choose Airflow variant
-$(breeze::flag_airflow_variants)
+$(flag_airflow_variants)
-$(breeze::print_star_line)
+$(print_star_line)
Choose backend to run for Airflow
-$(breeze::flag_backend_variants)
+$(flag_backend_variants)
-$(breeze::print_star_line)
+$(print_star_line)
Enable production image
-$(breeze::flag_production_image)
+$(flag_production_image)
-$(breeze::print_star_line)
+$(print_star_line)
Additional actions executed while entering breeze
-$(breeze::flag_breeze_actions)
+$(flag_breeze_actions)
-$(breeze::print_star_line)
+$(print_star_line)
Kind kubernetes and Kubernetes tests configuration(optional)
-$(breeze::flag_kubernetes_configuration)
+$(flag_kubernetes_configuration)
-$(breeze::print_star_line)
+$(print_star_line)
Manage mounting local files
-$(breeze::flag_local_file_mounting)
+$(flag_local_file_mounting)
-$(breeze::print_star_line)
+$(print_star_line)
Assume answers to questions
-$(breeze::flag_assume_answers_to_questions)
+$(flag_assume_answers_to_questions)
-$(breeze::print_star_line)
+$(print_star_line)
Choose different Airflow version to install or run
-$(breeze::flag_choose_different_airflow_version)
+$(flag_choose_different_airflow_version)
-$(breeze::print_star_line)
+$(print_star_line)
Credentials
-$(breeze::flag_credentials)
+$(flag_credentials)
-$(breeze::print_star_line)
+$(print_star_line)
Flags for building Docker images (both CI and production)
-$(breeze::flag_build_docker_images)
+$(flag_build_docker_images)
-$(breeze::print_star_line)
+$(print_star_line)
Flags for pulling/pushing Docker images (both CI and production)
-$(breeze::flag_pull_push_docker_images)
+$(flag_pull_push_docker_images)
-$(breeze::print_star_line)
+$(print_star_line)
Flags for generation of the backport packages
-$(breeze::flag_version_suffix)
+$(flag_version_suffix)
-$(breeze::print_star_line)
+$(print_star_line)
Increase verbosity of the scripts
-$(breeze::flag_verbosity)
+$(flag_verbosity)
-$(breeze::print_star_line)
+$(print_star_line)
Print detailed help message
-$(breeze::flag_help)
+$(flag_help)
"
}
@@ -2313,7 +2313,7 @@ $(breeze::flag_help)
# Outputs:
# Prints header line.
#######################################################################################################
-function breeze::print_header_line() {
+function print_header_line() {
if [ ${VERBOSE:="false"} == "true" ]; then
echo
printf '=%.0s' $(seq "${SCREEN_WIDTH}")
@@ -2328,12 +2328,12 @@ function breeze::print_header_line() {
# Outputs:
# Prints line.
#######################################################################################################
-function breeze::print_line() {
+function print_line() {
printf '#%.0s' $(seq "${SCREEN_WIDTH}")
}
# Prints star line filling screen indented_screen_width
-function breeze::print_star_line() {
+function print_star_line() {
printf '*%.0s' $(seq "${SCREEN_WIDTH}")
}
@@ -2365,38 +2365,38 @@ function breeze::print_star_line() {
#
#######################################################################################################
-function breeze::read_saved_environment_variables() {
- BACKEND="${BACKEND:=$(parameters::read_from_file BACKEND)}"
+function read_saved_environment_variables() {
+ BACKEND="${BACKEND:=$(read_from_file BACKEND)}"
BACKEND=${BACKEND:-${_BREEZE_DEFAULT_BACKEND}}
- KUBERNETES_MODE="${KUBERNETES_MODE:=$(parameters::read_from_file KUBERNETES_MODE)}"
+ KUBERNETES_MODE="${KUBERNETES_MODE:=$(read_from_file KUBERNETES_MODE)}"
KUBERNETES_MODE=${KUBERNETES_MODE:=${_BREEZE_DEFAULT_KUBERNETES_MODE}}
- KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(parameters::read_from_file KUBERNETES_VERSION)}"
+ KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(read_from_file KUBERNETES_VERSION)}"
KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_BREEZE_DEFAULT_KUBERNETES_VERSION}}
- KIND_VERSION="${KIND_VERSION:=$(parameters::read_from_file KIND_VERSION)}"
+ KIND_VERSION="${KIND_VERSION:=$(read_from_file KIND_VERSION)}"
KIND_VERSION=${KIND_VERSION:=${_BREEZE_DEFAULT_KIND_VERSION}}
- HELM_VERSION="${HELM_VERSION:=$(parameters::read_from_file HELM_VERSION)}"
+ HELM_VERSION="${HELM_VERSION:=$(read_from_file HELM_VERSION)}"
HELM_VERSION=${HELM_VERSION:=${_BREEZE_DEFAULT_HELM_VERSION}}
- POSTGRES_VERSION="${POSTGRES_VERSION:=$(parameters::read_from_file POSTGRES_VERSION)}"
+ POSTGRES_VERSION="${POSTGRES_VERSION:=$(read_from_file POSTGRES_VERSION)}"
POSTGRES_VERSION=${POSTGRES_VERSION:=${_BREEZE_DEFAULT_POSTGRES_VERSION}}
- MYSQL_VERSION="${MYSQL_VERSION:=$(parameters::read_from_file MYSQL_VERSION)}"
+ MYSQL_VERSION="${MYSQL_VERSION:=$(read_from_file MYSQL_VERSION)}"
MYSQL_VERSION=${MYSQL_VERSION:=${_BREEZE_DEFAULT_MYSQL_VERSION}}
# Here you read DockerHub user/account that you use
# You can populate your own images in DockerHub this way and work with the,
# You can override it with "--dockerhub-user" option and it will be stored in .build directory
- DOCKERHUB_USER="${DOCKERHUB_USER:=$(parameters::read_from_file DOCKERHUB_USER)}"
+ DOCKERHUB_USER="${DOCKERHUB_USER:=$(read_from_file DOCKERHUB_USER)}"
DOCKERHUB_USER="${DOCKERHUB_USER:=${_BREEZE_DEFAULT_DOCKERHUB_USER}}"
# Here you read DockerHub repo that you use
# You can populate your own images in DockerHub this way and work with them
# You can override it with "--dockerhub-repo" option and it will be stored in .build directory
- DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(parameters::read_from_file DOCKERHUB_REPO)}"
+ DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(read_from_file DOCKERHUB_REPO)}"
DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_BREEZE_DEFAULT_DOCKERHUB_REPO}}"
}
@@ -2427,8 +2427,8 @@ function breeze::read_saved_environment_variables() {
#
# Output: saved variable files in .build,
#######################################################################################################
-function breeze::check_and_save_all_params() {
- parameters::check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
+function check_and_save_all_params() {
+ check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *1_10* ]]; then
@@ -2452,17 +2452,17 @@ function breeze::check_and_save_all_params() {
fi
fi
- parameters::check_and_save_allowed_param "BACKEND" "backend" "--backend"
- parameters::check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
- parameters::check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
- parameters::check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
- parameters::check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
- parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
- parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
+ check_and_save_allowed_param "BACKEND" "backend" "--backend"
+ check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
+ check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
+ check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
+ check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
+ check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
+ check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
# Can't verify those - they can be anything, so let's just save them
- parameters::save_to_file DOCKERHUB_USER
- parameters::save_to_file DOCKERHUB_REPO
+ save_to_file DOCKERHUB_USER
+ save_to_file DOCKERHUB_REPO
}
#######################################################################################################
@@ -2477,17 +2477,17 @@ function breeze::check_and_save_all_params() {
# MYSQL_HOST_PORT
#
#######################################################################################################
-function breeze::print_cheatsheet() {
+function print_cheatsheet() {
if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${command_to_run} == "enter_breeze" ]]; then
echo
echo
- breeze::print_line
+ print_line
echo
echo " Airflow Breeze CHEATSHEET"
echo
set +e
if ! command -v breeze; then
- breeze::print_line
+ print_line
echo
echo " Adding breeze to your path:"
echo " When you exit the environment, you can add sources of Airflow to the path - you can"
@@ -2497,7 +2497,7 @@ function breeze::print_cheatsheet() {
echo
fi
set -e
- breeze::print_line
+ print_line
echo
echo " Port forwarding:"
@@ -2525,23 +2525,23 @@ function breeze::print_cheatsheet() {
# Used global constants:
# CMDNAME
#######################################################################################################
-function breeze::print_setup_instructions() {
+function print_setup_instructions() {
if [[ ${command_to_run} == "enter_breeze" ]]; then
# shellcheck disable=SC2034 # Unused variables left for comp_breeze usage
if ! typeset -f "_comp_breeze" >/dev/null; then
- breeze::print_line
+ print_line
echo
echo " You can setup autocomplete by running '${CMDNAME} setup-autocomplete'"
echo
echo
fi
- breeze::print_line
+ print_line
echo
echo " You can toggle ascii/cheatsheet by running:"
echo " * ${CMDNAME} toggle-suppress-cheatsheet"
echo " * ${CMDNAME} toggle-suppress-asciiart"
echo
- breeze::print_line
+ print_line
echo
echo
echo
@@ -2555,7 +2555,7 @@ function breeze::print_setup_instructions() {
# this is used in case static check command is used.
#
#######################################################################################################
-function breeze::make_sure_precommit_is_installed() {
+function make_sure_precommit_is_installed() {
echo
echo "Making sure pre-commit is installed"
echo
@@ -2587,7 +2587,7 @@ function breeze::make_sure_precommit_is_installed() {
# BUILT_CI_IMAGE_FLAG_FILE
#
#######################################################################################################
-function breeze::remove_images() {
+function remove_images() {
docker rmi "${PYTHON_BASE_IMAGE}" || true
docker rmi "${AIRFLOW_CI_IMAGE}" || true
docker rmi "${AIRFLOW_PROD_IMAGE}" || true
@@ -2602,7 +2602,7 @@ function breeze::remove_images() {
# static_check
#
#######################################################################################################
-function breeze::run_static_checks() {
+function run_static_checks() {
if [[ ${static_check} == "all" ]]; then
echo
echo "Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
@@ -2642,49 +2642,49 @@ function breeze::run_static_checks() {
# command_to_run
#
#######################################################################################################
-function breeze::run_build_command() {
+function run_build_command() {
case "${command_to_run}" in
run_tests | run_docker_compose)
- build_images::prepare_ci_build
- build_images::rebuild_ci_image_if_needed
+ prepare_ci_build
+ rebuild_ci_image_if_needed
;;
enter_breeze)
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- build_images::prepare_prod_build
+ prepare_prod_build
else
- build_images::prepare_ci_build
- build_images::rebuild_ci_image_if_needed
+ prepare_ci_build
+ rebuild_ci_image_if_needed
fi
;;
build_docs | perform_static_checks | perform_generate_constraints | perform_prepare_backport_readme | perform_prepare_backport_packages)
- build_images::prepare_ci_build
- build_images::rebuild_ci_image_if_needed
+ prepare_ci_build
+ rebuild_ci_image_if_needed
;;
perform_push_image)
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- build_images::prepare_prod_build
+ prepare_prod_build
else
- build_images::prepare_ci_build
- build_images::rebuild_ci_image_if_needed
+ prepare_ci_build
+ rebuild_ci_image_if_needed
fi
;;
build_image)
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- build_images::prepare_prod_build
- build_images::build_prod_images
+ prepare_prod_build
+ build_prod_images
else
- build_images::prepare_ci_build
- md5sum::calculate_md5sum_for_all_files
- build_images::build_ci_image
- md5sum::update_all_md5
- build_images::build_ci_image_manifest
+ prepare_ci_build
+ calculate_md5sum_for_all_files
+ build_ci_image
+ update_all_md5_files
+ build_ci_image_manifest
fi
;;
cleanup_image | run_exec)
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- build_images::prepare_prod_build
+ prepare_prod_build
else
- build_images::prepare_ci_build
+ prepare_ci_build
fi
;;
perform_initialize_local_virtualenv | perform_setup_autocomplete | toggle_suppress_cheatsheet | toggle_suppress_asciiart ) ;;
@@ -2702,8 +2702,8 @@ function breeze::run_build_command() {
echo "Checks status of KinD cluster"
elif [[ ${KIND_CLUSTER_OPERATION} == "deploy" ]]; then
echo "Deploys Airflow to KinD cluster"
- build_images::prepare_prod_build
- build_images::build_prod_images
+ prepare_prod_build
+ build_prod_images
elif [[ ${KIND_CLUSTER_OPERATION} == "test" ]]; then
echo "Run Kubernetes tests with the KinD cluster "
elif [[ ${KIND_CLUSTER_OPERATION} == "shell" ]]; then
@@ -2753,7 +2753,7 @@ function breeze::run_build_command() {
# Set Global variables:
# RUN_TESTS
#######################################################################################################
-function breeze::run_breeze_command() {
+function run_breeze_command() {
set +u
local dc_run_file
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
@@ -2799,43 +2799,43 @@ function breeze::run_breeze_command() {
set -u
;;
perform_static_checks)
- breeze::make_sure_precommit_is_installed
- breeze::run_static_checks "${@}"
+ make_sure_precommit_is_installed
+ run_static_checks "${@}"
;;
build_image) ;;
cleanup_image)
- breeze::remove_images
+ remove_images
;;
perform_generate_constraints)
- runs::run_generate_constraints
+ run_generate_constraints
;;
perform_prepare_backport_packages)
- runs::run_prepare_backport_packages "${@}"
+ run_prepare_backport_packages "${@}"
;;
perform_prepare_backport_readme)
- runs::run_prepare_backport_readme "${@}"
+ run_prepare_backport_readme "${@}"
;;
perform_push_image)
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- push_pull_remove_images::push_prod_images
+ push_prod_images
else
- push_pull_remove_images::push_ci_images
+ push_ci_images
fi
;;
perform_initialize_local_virtualenv)
- breeze::initialize_virtualenv
+ initialize_virtualenv
;;
perform_setup_autocomplete)
- breeze::setup_autocomplete
+ setup_autocomplete
;;
manage_kind_cluster)
- kind::make_sure_kubernetes_tools_are_installed
- kind::get_kind_cluster_name
- kind::perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
+ make_sure_kubernetes_tools_are_installed
+ get_kind_cluster_name
+ perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
;;
build_docs)
- runs::run_docs "${@}"
+ run_docs "${@}"
;;
toggle_suppress_cheatsheet)
if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
@@ -2882,73 +2882,73 @@ function breeze::run_breeze_command() {
# 4. DEFAULT_PYTHON_MAJOR_MINOR_VERSION from scripts/ci/libraries/_initialization.sh
#
# Here points 2. and 3. are realised. If result is empty string , the 4. will be set in
-# the next step (sanity_checks::basic_sanity_checks() is called and the version is still not set by then)
+# the next step (basic_sanity_checks() is called and the version is still not set by then)
# finally, if --python flag is specified, it will override whatever is set above.
#
-# We need to run after initialization::initialize_common_environment (so that parameters::read_from_file function is present)
+# We need to run after initialize_common_environment (so that read_from_file function is present)
# But before we set the default value for Python
#
# Used and modified global constants:
# PYTHON_MAJOR_MINOR_VERSION
#######################################################################################################
-function breeze::determine_python_version_to_use_in_breeze() {
- PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(parameters::read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
+function determine_python_version_to_use_in_breeze() {
+ PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
export PYTHON_MAJOR_MINOR_VERSION
}
-breeze::setup_default_breeze_constants
+setup_default_breeze_constants
-initialization::initialize_common_environment
+initialize_common_environment
-breeze::determine_python_version_to_use_in_breeze
+determine_python_version_to_use_in_breeze
-sanity_checks::basic_sanity_checks
+basic_sanity_checks
-start_end::script_start
+script_start
-traps::add_trap start_end::script_end EXIT
+trap script_end EXIT
-breeze::prepare_formatted_versions
+prepare_formatted_versions
-breeze::prepare_usage
+prepare_usage
set +u
-breeze::parse_arguments "${@}"
+parse_arguments "${@}"
-breeze::print_header_line
+print_header_line
-build_images::forget_last_answer
+forget_last_answer
-breeze::read_saved_environment_variables
+read_saved_environment_variables
-breeze::check_and_save_all_params
+check_and_save_all_params
-build_images::determine_docker_cache_strategy
+determine_docker_cache_strategy
-build_images::get_docker_image_names
+get_docker_image_names
-initialization::make_constants_read_only
+make_constants_read_only
-sanity_checks::sanitize_mounted_files
+sanitize_mounted_files
-breeze::prepare_command_files
+prepare_command_files
-breeze::run_build_command
+run_build_command
-breeze::print_header_line
+print_header_line
-breeze::print_badge
+print_badge
-breeze::print_cheatsheet
+print_cheatsheet
-breeze::print_setup_instructions
+print_setup_instructions
set +u # Account for an empty array
-breeze::run_breeze_command "${REMAINING_ARGS[@]}"
+run_breeze_command "${REMAINING_ARGS[@]}"
set +u # Account for an empty array
if [[ -n ${second_command_to_run} ]]; then
command_to_run=${second_command_to_run}
- breeze::run_breeze_command "${REMAINING_ARGS[@]}"
+ run_breeze_command "${REMAINING_ARGS[@]}"
fi
diff --git a/scripts/ci/backport_packages/ci_prepare_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
index c88576f..f30b4a7 100755
--- a/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
+++ b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
@@ -18,8 +18,8 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
-runs::run_prepare_backport_packages "$@"
+run_prepare_backport_packages "$@"
diff --git a/scripts/ci/backport_packages/ci_prepare_backport_readme.sh b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
index d4ddd33..8bf481c 100755
--- a/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
+++ b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
@@ -18,8 +18,8 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
-runs::run_prepare_backport_readme "$@"
+run_prepare_backport_readme "$@"
diff --git a/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
index 04d140d..190659a 100755
--- a/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
+++ b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
@@ -31,8 +31,8 @@ function run_test_package_import_all_classes() {
"--" "/opt/airflow/scripts/in_container/run_test_package_import_all_classes.sh"
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
run_test_package_import_all_classes
diff --git a/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
index a84652d..d923683 100755
--- a/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
+++ b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
@@ -26,8 +26,8 @@ function run_test_package_installation_separately() {
"--" "/opt/airflow/scripts/in_container/run_test_package_installation_separately.sh"
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
run_test_package_installation_separately
diff --git a/scripts/ci/constraints/ci_generate_constraints.sh b/scripts/ci/constraints/ci_generate_constraints.sh
index d2bd1f3..946d010 100755
--- a/scripts/ci/constraints/ci_generate_constraints.sh
+++ b/scripts/ci/constraints/ci_generate_constraints.sh
@@ -18,8 +18,8 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
-runs::run_generate_constraints
+run_generate_constraints
diff --git a/scripts/ci/docs/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
index e7273a7..0058454 100755
--- a/scripts/ci/docs/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -18,8 +18,8 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
-runs::run_docs "${@}"
+run_docs "${@}"
diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh
index 892aa1b..9a9f0f8 100755
--- a/scripts/ci/images/ci_build_dockerhub.sh
+++ b/scripts/ci/images/ci_build_dockerhub.sh
@@ -55,17 +55,17 @@ if [[ ${DOCKER_TAG} == *python*-ci ]]; then
echo "Building CI image"
echo
rm -rf "${BUILD_CACHE_DIR}"
- build_images::prepare_ci_build
- build_images::rebuild_ci_image_if_needed
- push_pull_remove_images::push_ci_images
+ prepare_ci_build
+ rebuild_ci_image_if_needed
+ push_ci_images
elif [[ ${DOCKER_TAG} == *python* ]]; then
echo
echo "Building prod image"
echo
rm -rf "${BUILD_CACHE_DIR}"
- build_images::prepare_prod_build
- build_images::build_prod_images
- push_pull_remove_images::push_prod_images
+ prepare_prod_build
+ build_prod_images
+ push_prod_images
else
echo
echo "Skipping the build in Dockerhub. The tag is not good: ${DOCKER_TAG}"
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index bd0f29a..4fa23a9 100755
--- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -21,14 +21,14 @@
# Builds or waits for the CI image in the CI environment
# Depending on "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_ci_image_on_ci() {
- build_images::prepare_ci_build
+ prepare_ci_build
rm -rf "${BUILD_CACHE_DIR}"
mkdir -pv "${BUILD_CACHE_DIR}"
if [[ ${USE_GITHUB_REGISTRY} == "true" && ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
# Pretend that the image was build. We already have image with the right sources baked in!
- md5sum::calculate_md5sum_for_all_files
+ calculate_md5sum_for_all_files
# Tries to wait for the images indefinitely
# skips further image checks - since we already have the target image
@@ -39,16 +39,16 @@ function build_ci_image_on_ci() {
fi
# first we pull base python image. We will need it to re-push it after master build
# Becoming the new "latest" image for other builds
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
+ wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
"${PYTHON_TAG_SUFFIX}" "${PYTHON_BASE_IMAGE}"
# And then the base image
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}" \
+ wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}" \
":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_CI_IMAGE}"
- md5sum::update_all_md5
+ update_all_md5_files
else
- build_images::rebuild_ci_image_if_needed
+ rebuild_ci_image_if_needed
fi
# Disable force pulling forced above this is needed for the subsequent scripts so that
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index f8cdcd2..65a4da4 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -21,7 +21,7 @@
# Builds or waits for the PROD image in the CI environment
# Depending on the "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_prod_images_on_ci() {
- build_images::prepare_prod_build
+ prepare_prod_build
rm -rf "${BUILD_CACHE_DIR}"
mkdir -pv "${BUILD_CACHE_DIR}"
@@ -31,13 +31,13 @@ function build_prod_images_on_ci() {
# Tries to wait for the image indefinitely
# skips further image checks - since we already have the target image
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
+ wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_IMAGE}"
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \
+ wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \
":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_BUILD_IMAGE}"
else
- build_images::build_prod_images
+ build_prod_images
fi
diff --git a/scripts/ci/images/ci_push_ci_images.sh b/scripts/ci/images/ci_push_ci_images.sh
index df2ae17..48d9ec5 100755
--- a/scripts/ci/images/ci_push_ci_images.sh
+++ b/scripts/ci/images/ci_push_ci_images.sh
@@ -18,6 +18,6 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::prepare_ci_build
+prepare_ci_build
-push_pull_remove_images::push_ci_images
+push_ci_images
diff --git a/scripts/ci/images/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh
index 31216c3..5b6f800 100755
--- a/scripts/ci/images/ci_push_production_images.sh
+++ b/scripts/ci/images/ci_push_production_images.sh
@@ -18,6 +18,6 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::prepare_prod_build
+prepare_prod_build
-push_pull_remove_images::push_prod_images
+push_prod_images
diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh
index 7e86977..a1dd705 100755
--- a/scripts/ci/images/ci_wait_for_all_ci_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_ci_images.sh
@@ -47,10 +47,10 @@ echo
# shellcheck source=scripts/ci/libraries/_all_libs.sh
source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-initialization::initialize_common_environment
+initialize_common_environment
for PYTHON_MAJOR_MINOR_VERSION in "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}"
do
export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
- push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ wait_for_github_registry_image "${AIRFLOW_CI_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
done
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh
index f8cf2c0..9cb4b34 100755
--- a/scripts/ci/images/ci_wait_for_all_prod_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -47,12 +47,12 @@ echo
# shellcheck source=scripts/ci/libraries/_all_libs.sh
source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-initialization::initialize_common_environment
+initialize_common_environment
for PYTHON_MAJOR_MINOR_VERSION in "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}"
do
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
export AIRFLOW_PROD_BUILD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-build"
- push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- push_pull_remove_images::wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ wait_for_github_registry_image "${AIRFLOW_PROD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ wait_for_github_registry_image "${AIRFLOW_PROD_BUILD_IMAGE_NAME}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
done
diff --git a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
index 2a7455a..f8393f2 100755
--- a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
+++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
@@ -19,14 +19,14 @@
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
set -euo pipefail
-traps::add_trap "kind::dump_kind_logs" EXIT HUP INT TERM
+add_trap "dump_kind_logs" EXIT HUP INT TERM
-kind::make_sure_kubernetes_tools_are_installed
-kind::get_kind_cluster_name
-build_images::prepare_prod_build
-build_images::build_prod_images
-kind::build_image_for_kubernetes_tests
-kind::load_image_to_kind_cluster
-kind::deploy_airflow_with_helm
-kind::forward_port_to_kind_webserver
-kind::deploy_test_kubernetes_resources
+make_sure_kubernetes_tools_are_installed
+get_kind_cluster_name
+prepare_prod_build
+build_prod_images
+build_image_for_kubernetes_tests
+load_image_to_kind_cluster
+deploy_airflow_with_helm
+forward_port_to_kind_webserver
+deploy_test_kubernetes_resources
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index 48698f5..e5e7615 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -18,10 +18,10 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-kind::make_sure_kubernetes_tools_are_installed
-kind::get_kind_cluster_name
+make_sure_kubernetes_tools_are_installed
+get_kind_cluster_name
-traps::add_trap kind::dump_kind_logs EXIT HUP INT TERM
+add_trap dump_kind_logs EXIT HUP INT TERM
INTERACTIVE="false"
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index d1beedf..7439f03 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -19,7 +19,7 @@
# For remote installation of airflow (from GitHub or Pypi) when building the image, you need to
# pass build flags depending on the version and method of the installation (for example to
# get proper requirement constraint files)
-function build_images::add_build_args_for_remote_install() {
+function add_build_args_for_remote_install() {
# entrypoint is used as AIRFLOW_SOURCES_FROM/TO in order to avoid costly copying of all sources of
# Airflow - those are not needed for remote install at all. Entrypoint is later overwritten by
EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
@@ -54,7 +54,7 @@ function build_images::add_build_args_for_remote_install() {
# Retrieves version of airflow stored in the production image (used to display the actual
# Version we use if it was build from PyPI or GitHub
-function build_images::get_airflow_version_from_production_image() {
+function get_airflow_version_from_production_image() {
VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
}
@@ -71,18 +71,18 @@ function build_images::get_airflow_version_from_production_image() {
# "no" answer given previously. This happens until you run any of the breeze commands or run all
# pre-commits `pre-commit run` - then the "LAST_FORCE_ANSWER_FILE" will be removed and you will
# be asked again.
-function build_images::forget_last_answer() {
+function forget_last_answer() {
if [[ ${REMEMBER_LAST_ANSWER:="false"} != "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Forgetting last answer from ${LAST_FORCE_ANSWER_FILE}:"
- verbosity::print_info
+ print_info
+ print_info "Forgetting last answer from ${LAST_FORCE_ANSWER_FILE}:"
+ print_info
rm -f "${LAST_FORCE_ANSWER_FILE}"
else
if [[ -f "${LAST_FORCE_ANSWER_FILE}" ]]; then
- verbosity::print_info
- verbosity::print_info "Still remember last answer from ${LAST_FORCE_ANSWER_FILE}:"
- verbosity::print_info "$(cat "${LAST_FORCE_ANSWER_FILE}")"
- verbosity::print_info
+ print_info
+ print_info "Still remember last answer from ${LAST_FORCE_ANSWER_FILE}:"
+ print_info "$(cat "${LAST_FORCE_ANSWER_FILE}")"
+ print_info
fi
fi
}
@@ -92,7 +92,7 @@ function build_images::forget_last_answer() {
# is needed and that the rebuild is not already forced. It asks the user using available terminals
# So that the script works also from within pre-commit run via git hooks - where stdin is not
# available - it tries to find usable terminal and ask the user via this terminal.
-function build_images::confirm_image_rebuild() {
+function confirm_image_rebuild() {
ACTION="rebuild"
if [[ ${FORCE_PULL_IMAGES:=} == "true" ]]; then
ACTION="pull and rebuild"
@@ -106,14 +106,14 @@ function build_images::confirm_image_rebuild() {
set +e
local RES
if [[ ${CI:="false"} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "CI environment - forcing rebuild for image ${THE_IMAGE_TYPE}."
- verbosity::print_info
+ print_info
+ print_info "CI environment - forcing rebuild for image ${THE_IMAGE_TYPE}."
+ print_info
RES="0"
elif [[ -n "${FORCE_ANSWER_TO_QUESTIONS=}" ]]; then
- verbosity::print_info
- verbosity::print_info "Forcing answer '${FORCE_ANSWER_TO_QUESTIONS}'"
- verbosity::print_info
+ print_info
+ print_info "Forcing answer '${FORCE_ANSWER_TO_QUESTIONS}'"
+ print_info
case "${FORCE_ANSWER_TO_QUESTIONS}" in
[yY][eE][sS] | [yY])
RES="0"
@@ -146,17 +146,17 @@ function build_images::confirm_image_rebuild() {
<"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}"
RES=$?
else
- verbosity::print_info
- verbosity::print_info "No terminal, no stdin - quitting"
- verbosity::print_info
+ print_info
+ print_info "No terminal, no stdin - quitting"
+ print_info
# No terminal, no stdin, no force answer - quitting!
RES="2"
fi
set -e
if [[ ${RES} == "1" ]]; then
- verbosity::print_info
- verbosity::print_info "Skipping rebuilding the image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}"
- verbosity::print_info
+ print_info
+ print_info "Skipping rebuilding the image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ print_info
export SKIP_REBUILD="true"
# Force "no" also to subsequent questions so that if you answer it once, you are not asked
# For all other pre-commits and you will continue using the images you already have
@@ -182,7 +182,7 @@ function build_images::confirm_image_rebuild() {
# It contains only one .json file - result of docker inspect - describing the image
# We cannot use docker registry APIs as they are available only with authorisation
# But this image can be pulled without authentication
-function build_images::build_ci_image_manifest() {
+function build_ci_image_manifest() {
docker inspect "${AIRFLOW_CI_IMAGE}" >"manifests/${AIRFLOW_CI_BASE_TAG}.json"
docker build \
--build-arg AIRFLOW_CI_BASE_TAG="${AIRFLOW_CI_BASE_TAG}" \
@@ -201,7 +201,7 @@ EOF
# Retrieves information about layers in the local IMAGE
# it stores list of SHAs of image layers in the file pointed at by TMP_MANIFEST_LOCAL_SHA
#
-function build_images::get_local_image_info() {
+function get_local_image_info() {
TMP_MANIFEST_LOCAL_JSON=$(mktemp)
TMP_MANIFEST_LOCAL_SHA=$(mktemp)
set +e
@@ -230,7 +230,7 @@ function build_images::get_local_image_info() {
# which is built together with the main image. This special manifest image is prepared during
# building of the main image and contains single JSON file being result of docker inspect on that image
# This image is from scratch so it is very tiny
-function build_images::get_remote_image_info() {
+function get_remote_image_info() {
set +e
# Pull remote manifest image
if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"; then
@@ -279,7 +279,7 @@ MAGIC_CUT_OFF_NUMBER_OF_LAYERS=36
# Compares layers from both remote and local image and set FORCE_PULL_IMAGES to true in case
# More than the last NN layers are different.
-function build_images::compare_layers() {
+function compare_layers() {
NUM_DIFF=$(diff -y --suppress-common-lines "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_SHA}" |
wc -l || true)
rm -f "${TMP_MANIFEST_REMOTE_JSON}" "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_JSON}" "${TMP_MANIFEST_LOCAL_SHA}"
@@ -305,13 +305,13 @@ function build_images::compare_layers() {
}
# Prints summary of the build parameters
-function build_images::print_build_info() {
- verbosity::print_info
- verbosity::print_info "Airflow ${AIRFLOW_VERSION} Python: ${PYTHON_MAJOR_MINOR_VERSION}. Image description: ${IMAGE_DESCRIPTION}"
- verbosity::print_info
+function print_build_info() {
+ print_info
+ print_info "Airflow ${AIRFLOW_VERSION} Python: ${PYTHON_MAJOR_MINOR_VERSION}. Image description: ${IMAGE_DESCRIPTION}"
+ print_info
}
-function build_images::get_docker_image_names() {
+function get_docker_image_names() {
# python image version to use
export PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}}
@@ -348,7 +348,7 @@ function build_images::get_docker_image_names() {
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
# DockerHub user etc. the variables are set so that other functions can use those variables.
-function build_images::prepare_ci_build() {
+function prepare_ci_build() {
export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
@@ -377,8 +377,8 @@ function build_images::prepare_ci_build() {
export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
readonly AIRFLOW_IMAGE
- sanity_checks::go_to_airflow_sources
- permissions::fix_group_permissions
+ go_to_airflow_sources
+ fix_group_permissions
}
# Only rebuilds CI image if needed. It checks if the docker image build is needed
@@ -386,29 +386,29 @@ function build_images::prepare_ci_build() {
# changed or in any of the edge cases (docker image removed, .build cache removed etc.
# In case rebuild is needed, it determines (by comparing layers in local and remote image)
# Whether pull is needed before rebuild.
-function build_images::rebuild_ci_image_if_needed() {
+function rebuild_ci_image_if_needed() {
if [[ -f "${BUILT_CI_IMAGE_FLAG_FILE}" ]]; then
- verbosity::print_info
- verbosity::print_info "${THE_IMAGE_TYPE} image already built locally."
- verbosity::print_info
+ print_info
+ print_info "${THE_IMAGE_TYPE} image already built locally."
+ print_info
else
- verbosity::print_info
- verbosity::print_info "${THE_IMAGE_TYPE} image not built locally: pulling and building"
- verbosity::print_info
+ print_info
+ print_info "${THE_IMAGE_TYPE} image not built locally: pulling and building"
+ print_info
export FORCE_PULL_IMAGES="true"
export FORCE_BUILD_IMAGES="true"
fi
if [[ ${CHECK_IMAGE_FOR_REBUILD} == "false" ]]; then
- verbosity::print_info
- verbosity::print_info "Skip checking for rebuilds of the CI image but checking if it needs to be pulled"
- verbosity::print_info
- push_pull_remove_images::pull_ci_images_if_needed
+ print_info
+ print_info "Skip checking for rebuilds of the CI image but checking if it needs to be pulled"
+ print_info
+ pull_ci_images_if_needed
return
fi
local needs_docker_build="false"
- md5sum::check_if_docker_build_is_needed
+ check_if_docker_build_is_needed
if [[ ${needs_docker_build} == "true" ]]; then
if [[ ${SKIP_CHECK_REMOTE_IMAGE:=} != "true" && ${DOCKER_CACHE} == "pulled" ]]; then
# Check if remote image is different enough to force pull
@@ -418,11 +418,11 @@ function build_images::rebuild_ci_image_if_needed() {
echo
echo "Checking if the remote image needs to be pulled"
echo
- build_images::get_remote_image_info
+ get_remote_image_info
if [[ ${REMOTE_DOCKER_REGISTRY_UNREACHABLE:=} != "true" ]]; then
- build_images::get_local_image_info
+ get_local_image_info
if [[ ${LOCAL_MANIFEST_IMAGE_UNAVAILABLE:=} != "true" ]]; then
- build_images::compare_layers
+ compare_layers
else
FORCE_PULL_IMAGES="true"
fi
@@ -430,7 +430,7 @@ function build_images::rebuild_ci_image_if_needed() {
fi
SKIP_REBUILD="false"
if [[ ${CI:=} != "true" && "${FORCE_BUILD:=}" != "true" ]]; then
- build_images::confirm_image_rebuild
+ confirm_image_rebuild
fi
if [[ ${SKIP_REBUILD} != "true" ]]; then
SYSTEM=$(uname -s)
@@ -440,40 +440,40 @@ function build_images::rebuild_ci_image_if_needed() {
./scripts/ci/tools/ci_fix_ownership.sh
fi
fi
- verbosity::print_info
- verbosity::print_info "Build start: ${THE_IMAGE_TYPE} image."
- verbosity::print_info
- build_images::build_ci_image
- md5sum::update_all_md5
- build_images::build_ci_image_manifest
- verbosity::print_info
- verbosity::print_info "Build completed: ${THE_IMAGE_TYPE} image."
- verbosity::print_info
+ print_info
+ print_info "Build start: ${THE_IMAGE_TYPE} image."
+ print_info
+ build_ci_image
+ update_all_md5_files
+ build_ci_image_manifest
+ print_info
+ print_info "Build completed: ${THE_IMAGE_TYPE} image."
+ print_info
fi
else
- verbosity::print_info
- verbosity::print_info "No need to build - none of the important files changed: ${FILES_FOR_REBUILD_CHECK[*]}"
- verbosity::print_info
+ print_info
+ print_info "No need to build - none of the important files changed: ${FILES_FOR_REBUILD_CHECK[*]}"
+ print_info
fi
}
# Interactive version of confirming the ci image that is used in pre-commits
# it displays additional information - what the user should do in order to bring the local images
# back to state that pre-commit will be happy with
-function build_images::rebuild_ci_image_if_needed_and_confirmed() {
+function rebuild_ci_image_if_needed_and_confirmed() {
local needs_docker_build="false"
THE_IMAGE_TYPE="CI"
- md5sum::check_if_docker_build_is_needed
+ check_if_docker_build_is_needed
if [[ ${needs_docker_build} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Docker image build is needed!"
- verbosity::print_info
+ print_info
+ print_info "Docker image build is needed!"
+ print_info
else
- verbosity::print_info
- verbosity::print_info "Docker image build is not needed!"
- verbosity::print_info
+ print_info
+ print_info "Docker image build is not needed!"
+ print_info
fi
if [[ "${needs_docker_build}" == "true" ]]; then
@@ -491,10 +491,10 @@ function build_images::rebuild_ci_image_if_needed_and_confirmed() {
echo " The second command you use if you want to completely refresh your images from dockerhub."
echo
SKIP_REBUILD="false"
- build_images::confirm_image_rebuild
+ confirm_image_rebuild
if [[ ${SKIP_REBUILD} != "true" ]]; then
- build_images::rebuild_ci_image_if_needed
+ rebuild_ci_image_if_needed
fi
fi
}
@@ -503,17 +503,17 @@ function build_images::rebuild_ci_image_if_needed_and_confirmed() {
# passes the necessary docker build flags via DOCKER_CACHE_CI_DIRECTIVE array
# it also passes the right Build args depending on the configuration of the build
# selected by Breeze flags or environment variables.
-function build_images::build_ci_image() {
- build_images::print_build_info
+function build_ci_image() {
+ print_build_info
if [[ -n ${DETECTED_TERMINAL=} ]]; then
echo -n "Preparing ${AIRFLOW_CI_IMAGE}.
" >"${DETECTED_TERMINAL}"
- spinner::spin "${OUTPUT_LOG}" &
+ spin "${OUTPUT_LOG}" &
SPIN_PID=$!
# shellcheck disable=SC2064
- traps::add_trap "kill ${SPIN_PID}" INT TERM HUP EXIT
+ add_trap "kill ${SPIN_PID}" INT TERM HUP EXIT
fi
- push_pull_remove_images::pull_ci_images_if_needed
+ pull_ci_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
export DOCKER_CACHE_CI_DIRECTIVE=("--no-cache")
elif [[ "${DOCKER_CACHE}" == "local" ]]; then
@@ -536,10 +536,10 @@ function build_images::build_ci_image() {
if [[ -n ${DETECTED_TERMINAL=} ]]; then
echo -n "Preparing ${AIRFLOW_CI_IMAGE}.
" >"${DETECTED_TERMINAL}"
- spinner::spin "${OUTPUT_LOG}" &
+ spin "${OUTPUT_LOG}" &
SPIN_PID=$!
# shellcheck disable=SC2064
- traps::add_trap "kill ${SPIN_PID}" EXIT HUP INT TERM
+ add_trap "kill ${SPIN_PID}" EXIT HUP INT TERM
fi
if [[ -n ${DETECTED_TERMINAL=} ]]; then
echo -n "
@@ -577,14 +577,14 @@ Docker building ${AIRFLOW_CI_IMAGE}.
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
# DockerHub user etc. the variables are set so that other functions can use those variables.
-function build_images::prepare_prod_build() {
+function prepare_prod_build() {
if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
# When --install-airflow-reference is used then the image is build from github tag
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
"--build-arg" "AIRFLOW_INSTALL_SOURCES=https://github.com/apache/airflow/archive/${INSTALL_AIRFLOW_REFERENCE}.tar.gz#egg=apache-airflow"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_REFERENCE}"
- build_images::add_build_args_for_remote_install
+ add_build_args_for_remote_install
elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
# When --install-airflow-version is used then the image is build from PIP package
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
@@ -592,7 +592,7 @@ function build_images::prepare_prod_build() {
"--build-arg" "AIRFLOW_INSTALL_VERSION===${INSTALL_AIRFLOW_VERSION}"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}"
- build_images::add_build_args_for_remote_install
+ add_build_args_for_remote_install
else
# When no airflow version/reference is specified, production image is built from local sources
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
@@ -627,7 +627,7 @@ function build_images::prepare_prod_build() {
fi
AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="${BRANCH_NAME}"
- sanity_checks::go_to_airflow_sources
+ go_to_airflow_sources
}
# Builds PROD image - depending on the caching strategy (pulled, local, disabled) it
@@ -635,17 +635,17 @@ function build_images::prepare_prod_build() {
# DOCKER_CACHE_PROD_BUILD_DIRECTIVE (separate caching options are needed for "build" segment of the image)
# it also passes the right Build args depending on the configuration of the build
# selected by Breeze flags or environment variables.
-function build_images::build_prod_images() {
- build_images::print_build_info
+function build_prod_images() {
+ print_build_info
if [[ ${SKIP_BUILDING_PROD_IMAGE} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Skip building production image. Assume the one we have is good!"
- verbosity::print_info
+ print_info
+ print_info "Skip building production image. Assume the one we have is good!"
+ print_info
return
fi
- push_pull_remove_images::pull_prod_images_if_needed
+ pull_prod_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
export DOCKER_CACHE_PROD_DIRECTIVE=("--cache-from" "${AIRFLOW_PROD_BUILD_IMAGE}")
@@ -712,7 +712,7 @@ function build_images::build_prod_images() {
# $1 - image name to wait for
# $2 - suffix of the image to wait for
# $3, $4, ... - target tags to tag the image with
-function build_images::wait_for_image_tag() {
+function wait_for_image_tag() {
IMAGE_NAME="${1}"
IMAGE_SUFFIX=${2}
shift 2
@@ -749,7 +749,7 @@ function build_images::wait_for_image_tag() {
# We use pulled docker image cache by default for CI images to speed up the builds
# and local to speed up iteration on kerberos tests
-function build_images::determine_docker_cache_strategy() {
+function determine_docker_cache_strategy() {
if [[ -z "${DOCKER_CACHE=}" ]]; then
if [[ "${PRODUCTION_IMAGE}" == "true" ]]; then
export DOCKER_CACHE="local"
@@ -758,7 +758,7 @@ function build_images::determine_docker_cache_strategy() {
fi
fi
readonly DOCKER_CACHE
- verbosity::print_info
- verbosity::print_info "Using ${DOCKER_CACHE} cache strategy for the build."
- verbosity::print_info
+ print_info
+ print_info "Using ${DOCKER_CACHE} cache strategy for the build."
+ print_info
}
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 3b9538c..6e6bc95 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -21,7 +21,7 @@
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS=()
# Creates directories for Breeze
-function initialization::create_directories() {
+function create_directories() {
# This folder is mounted to inside the container in /files folder. This is the way how
# We can exchange DAGs, scripts, packages etc with the container environment
export FILES_DIR="${AIRFLOW_SOURCES}/files"
@@ -51,7 +51,7 @@ function initialization::create_directories() {
readonly CACHE_TMP_FILE_DIR
if [[ ${SKIP_CACHE_DELETION=} != "true" ]]; then
- traps::add_trap "rm -rf -- '${CACHE_TMP_FILE_DIR}'" EXIT HUP INT TERM
+ add_trap "rm -rf -- '${CACHE_TMP_FILE_DIR}'" EXIT HUP INT TERM
fi
OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log"
@@ -60,7 +60,7 @@ function initialization::create_directories() {
}
# Very basic variables that MUST be set
-function initialization::initialize_base_variables() {
+function initialize_base_variables() {
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
@@ -100,7 +100,7 @@ function initialization::initialize_base_variables() {
}
# Determine current branch
-function initialization::initialize_branch_variables() {
+function initialize_branch_variables() {
# Default branch used - this will be different in different branches
export DEFAULT_BRANCH="master"
export DEFAULT_CONSTRAINTS_BRANCH="constraints-master"
@@ -112,7 +112,7 @@ function initialization::initialize_branch_variables() {
}
# Determine dockerhub user/repo used for push/pull
-function initialization::initialize_dockerhub_variables() {
+function initialize_dockerhub_variables() {
# You can override DOCKERHUB_USER to use your own DockerHub account and play with your
# own docker images. In this case you can build images locally and push them
export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
@@ -123,7 +123,7 @@ function initialization::initialize_dockerhub_variables() {
}
# Determine available integrations
-function initialization::initialize_available_integrations() {
+function initialize_available_integrations() {
export AVAILABLE_INTEGRATIONS="cassandra kerberos mongo openldap presto rabbitmq redis"
}
@@ -132,7 +132,7 @@ function initialization::initialize_available_integrations() {
FILES_FOR_REBUILD_CHECK=()
# Determine which files trigger rebuild check
-function initialization::initialize_files_for_rebuild_check() {
+function initialize_files_for_rebuild_check() {
FILES_FOR_REBUILD_CHECK+=(
"setup.py"
"setup.cfg"
@@ -161,7 +161,7 @@ FILES_TO_CLEANUP_ON_EXIT=()
EXTRA_DOCKER_FLAGS=()
# Determine behaviour of mounting sources to the container
-function initialization::initialize_mount_variables() {
+function initialize_mount_variables() {
# Whether necessary for airflow run local sources are mounted to docker
export MOUNT_LOCAL_SOURCES=${MOUNT_LOCAL_SOURCES:="true"}
@@ -170,20 +170,20 @@ function initialization::initialize_mount_variables() {
export MOUNT_FILES=${MOUNT_FILES:="true"}
if [[ ${MOUNT_LOCAL_SOURCES} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Mounting necessary host volumes to Docker"
- verbosity::print_info
- read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)"
+ print_info
+ print_info "Mounting necessary host volumes to Docker"
+ print_info
+ read -r -a EXTRA_DOCKER_FLAGS <<<"$(convert_local_mounts_to_docker_params)"
else
- verbosity::print_info
- verbosity::print_info "Skip mounting host volumes to Docker"
- verbosity::print_info
+ print_info
+ print_info "Skip mounting host volumes to Docker"
+ print_info
fi
if [[ ${MOUNT_FILES} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Mounting files folder to Docker"
- verbosity::print_info
+ print_info
+ print_info "Mounting files folder to Docker"
+ print_info
EXTRA_DOCKER_FLAGS+=(
"-v" "${AIRFLOW_SOURCES}/files:/files"
)
@@ -197,7 +197,7 @@ function initialization::initialize_mount_variables() {
}
# Determine values of force settings
-function initialization::initialize_force_variables() {
+function initialize_force_variables() {
# Whether necessary for airflow run local sources are mounted to docker
export FORCE_PULL_IMAGES=${FORCE_PULL_IMAGES:="false"}
@@ -219,7 +219,7 @@ function initialization::initialize_force_variables() {
}
# Determine information about the host
-function initialization::initialize_host_variables() {
+function initialize_host_variables() {
# Set host user id to current user. This is used to set the ownership properly when exiting
# The container on Linux - all files created inside docker are created with root user
# but they should be restored back to the host user
@@ -252,7 +252,7 @@ function initialization::initialize_host_variables() {
}
# Determine image augmentation parameters
-function initialization::initialize_image_build_variables() {
+function initialize_image_build_variables() {
# Default extras used for building CI image
export DEFAULT_CI_EXTRAS="devel_ci"
@@ -292,7 +292,7 @@ function initialization::initialize_image_build_variables() {
}
# Determine version suffixes used to build backport packages
-function initialization::initialize_version_suffixes_for_package_building() {
+function initialize_version_suffixes_for_package_building() {
# Version suffix for PyPI packaging
export VERSION_SUFFIX_FOR_PYPI=""
# Artifact name suffix for SVN packaging
@@ -300,7 +300,7 @@ function initialization::initialize_version_suffixes_for_package_building() {
}
# Determine versions of kubernetes cluster and tools used
-function initialization::initialize_kubernetes_variables() {
+function initialize_kubernetes_variables() {
# By default we assume the kubernetes cluster is not being started
export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
# Default Kubernetes version
@@ -331,13 +331,13 @@ function initialization::initialize_kubernetes_variables() {
readonly KUBECTL_BINARY_PATH
}
-function initialization::initialize_git_variables() {
+function initialize_git_variables() {
# SHA of the commit for the current sources
COMMIT_SHA="$(git rev-parse HEAD 2>/dev/null || echo "Unknown")"
export COMMIT_SHA
}
-function initialization::initialize_github_variables() {
+function initialize_github_variables() {
# Defaults for interacting with GitHub
export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
@@ -358,24 +358,24 @@ function initialization::initialize_github_variables() {
}
# Common environment that is initialized by both Breeze and CI scripts
-function initialization::initialize_common_environment() {
- initialization::create_directories
- initialization::initialize_base_variables
- initialization::initialize_branch_variables
- initialization::initialize_available_integrations
- initialization::initialize_files_for_rebuild_check
- initialization::initialize_dockerhub_variables
- initialization::initialize_mount_variables
- initialization::initialize_force_variables
- initialization::initialize_host_variables
- initialization::initialize_image_build_variables
- initialization::initialize_version_suffixes_for_package_building
- initialization::initialize_kubernetes_variables
- initialization::initialize_git_variables
- initialization::initialize_github_variables
+function initialize_common_environment() {
+ create_directories
+ initialize_base_variables
+ initialize_branch_variables
+ initialize_available_integrations
+ initialize_files_for_rebuild_check
+ initialize_dockerhub_variables
+ initialize_mount_variables
+ initialize_force_variables
+ initialize_host_variables
+ initialize_image_build_variables
+ initialize_version_suffixes_for_package_building
+ initialize_kubernetes_variables
+ initialize_git_variables
+ initialize_github_variables
}
-function initialization::set_default_python_version_if_empty() {
+function set_default_python_version_if_empty() {
# default version of python used to tag the "master" and "latest" images in DockerHub
export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6
@@ -384,7 +384,7 @@ function initialization::set_default_python_version_if_empty() {
}
-function initialization::summarize_ci_environment() {
+function summarize_ci_environment() {
cat <<EOF
Configured build variables:
@@ -476,7 +476,7 @@ EOF
# (This makes it easy to move between different CI systems)
# This function maps CI-specific variables into a generic ones (prefixed with CI_) that
# we used in other scripts
-function initialization::get_environment_for_builds_on_ci() {
+function get_environment_for_builds_on_ci() {
if [[ ${GITHUB_ACTIONS:=} == "true" ]]; then
export CI_TARGET_REPO="${GITHUB_REPOSITORY}"
export CI_TARGET_BRANCH="${GITHUB_BASE_REF:="master"}"
@@ -494,12 +494,12 @@ function initialization::get_environment_for_builds_on_ci() {
BRANCH_EXISTS=$(git ls-remote --heads \
"https://github.com/${CI_SOURCE_REPO}.git" "${CI_SOURCE_BRANCH}" || true)
if [[ -z ${BRANCH_EXISTS=} ]]; then
- verbosity::print_info
- verbosity::print_info "https://github.com/${CI_SOURCE_REPO}.git Branch ${CI_SOURCE_BRANCH} does not exist"
- verbosity::print_info
- verbosity::print_info
- verbosity::print_info "Fallback to https://github.com/${CI_TARGET_REPO}.git Branch ${CI_TARGET_BRANCH}"
- verbosity::print_info
+ print_info
+ print_info "https://github.com/${CI_SOURCE_REPO}.git Branch ${CI_SOURCE_BRANCH} does not exist"
+ print_info
+ print_info
+ print_info "Fallback to https://github.com/${CI_TARGET_REPO}.git Branch ${CI_TARGET_BRANCH}"
+ print_info
# Fallback to the target repository if the repo does not exist
export CI_SOURCE_REPO="${CI_TARGET_REPO}"
export CI_SOURCE_BRANCH="${CI_TARGET_BRANCH}"
@@ -522,7 +522,7 @@ function initialization::get_environment_for_builds_on_ci() {
fi
if [[ ${VERBOSE} == "true" && ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then
- initialization::summarize_ci_environment
+ summarize_ci_environment
fi
}
@@ -530,7 +530,7 @@ function initialization::get_environment_for_builds_on_ci() {
# By the time this method is run, nearly all constants have been already set to the final values
# so we can set them as readonly.
-function initialization::make_constants_read_only() {
+function make_constants_read_only() {
# Set the arguments as read-only
readonly PYTHON_MAJOR_MINOR_VERSION
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index eff8c56..aeb540a 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -17,7 +17,7 @@
# under the License.
-function kind::get_kind_cluster_name(){
+function get_kind_cluster_name(){
# Name of the KinD cluster to connect to
export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
readonly KIND_CLUSTER_NAME
@@ -26,7 +26,7 @@ function kind::get_kind_cluster_name(){
readonly KUBECTL_CLUSTER_NAME
}
-function kind::dump_kind_logs() {
+function dump_kind_logs() {
echo "###########################################################################################"
echo " Dumping logs from KIND"
echo "###########################################################################################"
@@ -39,7 +39,7 @@ function kind::dump_kind_logs() {
kind --name "${KIND_CLUSTER_NAME}" export logs "${DUMP_DIR}"
}
-function kind::make_sure_kubernetes_tools_are_installed() {
+function make_sure_kubernetes_tools_are_installed() {
SYSTEM=$(uname -s| tr '[:upper:]' '[:lower:]')
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${SYSTEM}-amd64"
@@ -91,7 +91,7 @@ function kind::make_sure_kubernetes_tools_are_installed() {
PATH=${PATH}:${BUILD_CACHE_DIR}/bin
}
-function kind::create_cluster() {
+function create_cluster() {
if [[ "${TRAVIS:="false"}" == "true" ]]; then
# Travis CI does not handle the nice output of Kind well, so we need to capture it
# And display only if kind fails to start
@@ -115,7 +115,7 @@ function kind::create_cluster() {
echo
}
-function kind::delete_cluster() {
+function delete_cluster() {
kind delete cluster --name "${KIND_CLUSTER_NAME}"
echo
echo "Deleted cluster ${KIND_CLUSTER_NAME}"
@@ -123,7 +123,7 @@ function kind::delete_cluster() {
rm -rf "${HOME}/.kube/*"
}
-function kind::perform_kind_cluster_operation() {
+function perform_kind_cluster_operation() {
ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell recreate ]"
set +u
@@ -146,7 +146,7 @@ function kind::perform_kind_cluster_operation() {
echo
echo "Cluster name: ${KIND_CLUSTER_NAME}"
echo
- kind::check_cluster_ready_for_airflow
+ kind get nodes --name "${KIND_CLUSTER_NAME}"
echo
exit
else
@@ -166,23 +166,23 @@ function kind::perform_kind_cluster_operation() {
echo
echo "Recreating cluster"
echo
- kind::delete_cluster
- kind::create_cluster
+ delete_cluster
+ create_cluster
elif [[ ${OPERATION} == "stop" ]]; then
echo
echo "Deleting cluster"
echo
- kind::delete_cluster
+ delete_cluster
exit
elif [[ ${OPERATION} == "deploy" ]]; then
echo
echo "Deploying Airflow to KinD"
echo
- kind::build_image_for_kubernetes_tests
- kind::load_image_to_kind_cluster
- kind::deploy_airflow_with_helm
- kind::forward_port_to_kind_webserver
- kind::deploy_test_kubernetes_resources
+ build_image_for_kubernetes_tests
+ load_image_to_kind_cluster
+ deploy_airflow_with_helm
+ forward_port_to_kind_webserver
+ deploy_test_kubernetes_resources
elif [[ ${OPERATION} == "test" ]]; then
echo
echo "Testing with KinD"
@@ -204,13 +204,13 @@ function kind::perform_kind_cluster_operation() {
echo
echo "Creating cluster"
echo
- kind::create_cluster
+ create_cluster
elif [[ ${OPERATION} == "recreate" ]]; then
echo
echo "Cluster ${KIND_CLUSTER_NAME} does not exist. Creating rather than recreating"
echo "Creating cluster"
echo
- kind::create_cluster
+ create_cluster
elif [[ ${OPERATION} == "stop" || ${OPERATION} == "deploy" || \
${OPERATION} == "test" || ${OPERATION} == "shell" ]]; then
echo >&2
@@ -226,7 +226,7 @@ function kind::perform_kind_cluster_operation() {
fi
}
-function kind::check_cluster_ready_for_airflow() {
+function check_cluster_ready_for_airflow() {
kubectl cluster-info --cluster "${KUBECTL_CLUSTER_NAME}"
kubectl get nodes --cluster "${KUBECTL_CLUSTER_NAME}"
echo
@@ -244,7 +244,7 @@ function kind::check_cluster_ready_for_airflow() {
}
-function kind::build_image_for_kubernetes_tests() {
+function build_image_for_kubernetes_tests() {
cd "${AIRFLOW_SOURCES}" || exit 1
docker build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF
FROM ${AIRFLOW_PROD_IMAGE}
@@ -259,14 +259,14 @@ EOF
echo "The ${AIRFLOW_PROD_IMAGE_KUBERNETES} is prepared for test kubernetes deployment."
}
-function kind::load_image_to_kind_cluster() {
+function load_image_to_kind_cluster() {
echo
echo "Loading ${AIRFLOW_PROD_IMAGE_KUBERNETES} to ${KIND_CLUSTER_NAME}"
echo
kind load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE_KUBERNETES}"
}
-function kind::forward_port_to_kind_webserver() {
+function forward_port_to_kind_webserver() {
num_tries=0
set +e
while ! curl http://localhost:8080/health -s | grep -q healthy; do
@@ -287,7 +287,7 @@ function kind::forward_port_to_kind_webserver() {
set -e
}
-function kind::deploy_airflow_with_helm() {
+function deploy_airflow_with_helm() {
echo
echo "Deploying Airflow with Helm"
echo
@@ -310,7 +310,7 @@ function kind::deploy_airflow_with_helm() {
}
-function kind::deploy_test_kubernetes_resources() {
+function deploy_test_kubernetes_resources() {
echo
echo "Deploying Custom kubernetes resources"
echo
@@ -318,7 +318,7 @@ function kind::deploy_test_kubernetes_resources() {
}
-function kind::dump_kubernetes_logs() {
+function dump_kubernetes_logs() {
POD=$(kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}' \
--cluster "${KUBECTL_CLUSTER_NAME}" | grep airflow | head -1)
echo "------- pod description -------"
diff --git a/scripts/ci/libraries/_local_mounts.sh b/scripts/ci/libraries/_local_mounts.sh
index 59ef37c..4300af9 100644
--- a/scripts/ci/libraries/_local_mounts.sh
+++ b/scripts/ci/libraries/_local_mounts.sh
@@ -20,7 +20,7 @@
# By default not the whole airflow sources directory is mounted because there are often
# artifacts created there (for example .egg-info files) that are breaking the capability
# of running different python versions in Breeze. So we only mount what is needed by default.
-function local_mounts::generate_local_mounts_list {
+function generate_local_mounts_list {
local prefix="$1"
LOCAL_MOUNTS=(
"$prefix".bash_aliases:/root/.bash_aliases:cached
@@ -59,8 +59,8 @@ function local_mounts::generate_local_mounts_list {
# Converts the local mounts that we defined above to the right set of -v
# volume mappings in docker-compose file. This is needed so that we only
# maintain the volumes in one place (above)
-function local_mounts::convert_local_mounts_to_docker_params() {
- local_mounts::generate_local_mounts_list "${AIRFLOW_SOURCES}/"
+function convert_local_mounts_to_docker_params() {
+ generate_local_mounts_list "${AIRFLOW_SOURCES}/"
# Bash can't "return" arrays, so we need to quote any special characters
printf -- '-v %q ' "${LOCAL_MOUNTS[@]}"
}
diff --git a/scripts/ci/libraries/_md5sum.sh b/scripts/ci/libraries/_md5sum.sh
index be41244..0ef2d79 100644
--- a/scripts/ci/libraries/_md5sum.sh
+++ b/scripts/ci/libraries/_md5sum.sh
@@ -21,7 +21,7 @@
# The md5sum files are stored in .build directory - you can delete this directory
# If you want to rebuild everything from the scratch
#
-function md5sum::calculate_file_md5sum {
+function calculate_file_md5sum {
local FILE="${1}"
local MD5SUM
local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${BRANCH_NAME}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}"
@@ -34,13 +34,13 @@ function md5sum::calculate_file_md5sum {
echo "${MD5SUM}" > "${MD5SUM_FILE_NEW}"
local RET_CODE=0
if [[ ! -f "${MD5SUM_FILE}" ]]; then
- verbosity::print_info "Missing md5sum for ${FILE#${AIRFLOW_SOURCES}} (${MD5SUM_FILE#${AIRFLOW_SOURCES}})"
+ print_info "Missing md5sum for ${FILE#${AIRFLOW_SOURCES}} (${MD5SUM_FILE#${AIRFLOW_SOURCES}})"
RET_CODE=1
else
diff "${MD5SUM_FILE_NEW}" "${MD5SUM_FILE}" >/dev/null
RES=$?
if [[ "${RES}" != "0" ]]; then
- verbosity::print_info "The md5sum changed for ${FILE}"
+ print_info "The md5sum changed for ${FILE}"
RET_CODE=1
fi
fi
@@ -51,7 +51,7 @@ function md5sum::calculate_file_md5sum {
# Moves md5sum file from it's temporary location in CACHE_TMP_FILE_DIR to
# BUILD_CACHE_DIR - thus updating stored MD5 sum fo the file
#
-function md5sum::move_file_md5sum {
+function move_file_md5sum {
local FILE="${1}"
local MD5SUM_FILE
local MD5SUM_CACHE_DIR="${BUILD_CACHE_DIR}/${BRANCH_NAME}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}"
@@ -61,7 +61,7 @@ function md5sum::move_file_md5sum {
MD5SUM_FILE_NEW=${CACHE_TMP_FILE_DIR}/$(basename "${FILE}").md5sum.new
if [[ -f "${MD5SUM_FILE_NEW}" ]]; then
mv "${MD5SUM_FILE_NEW}" "${MD5SUM_FILE}"
- verbosity::print_info "Updated md5sum file ${MD5SUM_FILE} for ${FILE}."
+ print_info "Updated md5sum file ${MD5SUM_FILE} for ${FILE}."
fi
}
@@ -70,25 +70,25 @@ function md5sum::move_file_md5sum {
# records that we built the images locally so that next time we use
# it from the local docker cache rather than pull (unless forced)
#
-function md5sum::update_all_md5() {
- verbosity::print_info
- verbosity::print_info "Updating md5sum files"
- verbosity::print_info
+function update_all_md5_files() {
+ print_info
+ print_info "Updating md5sum files"
+ print_info
for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
do
- md5sum::move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
+ move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
done
mkdir -pv "${BUILD_CACHE_DIR}/${BRANCH_NAME}"
touch "${BUILT_CI_IMAGE_FLAG_FILE}"
}
-function md5sum::calculate_md5sum_for_all_files() {
+function calculate_md5sum_for_all_files() {
FILES_MODIFIED="false"
set +e
for FILE in "${FILES_FOR_REBUILD_CHECK[@]}"
do
- if ! md5sum::calculate_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"; then
+ if ! calculate_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"; then
FILES_MODIFIED="true"
fi
done
@@ -114,24 +114,24 @@ function md5sum::calculate_md5sum_for_all_files() {
#
# As result of this check - most of the static checks will start pretty much immediately.
#
-function md5sum::check_if_docker_build_is_needed() {
- verbosity::print_info
- verbosity::print_info "Checking if image build is needed for ${THE_IMAGE_TYPE} image."
- verbosity::print_info
+function check_if_docker_build_is_needed() {
+ print_info
+ print_info "Checking if image build is needed for ${THE_IMAGE_TYPE} image."
+ print_info
if [[ ${FORCE_BUILD_IMAGES:=""} == "true" ]]; then
- verbosity::print_info "Docker image build is forced for ${THE_IMAGE_TYPE} image"
- md5sum::calculate_md5sum_for_all_files
+ print_info "Docker image build is forced for ${THE_IMAGE_TYPE} image"
+ calculate_md5sum_for_all_files
needs_docker_build="true"
else
- md5sum::calculate_md5sum_for_all_files
+ calculate_md5sum_for_all_files
if [[ ${FILES_MODIFIED} == "true" ]]; then
needs_docker_build="true"
fi
if [[ ${needs_docker_build} == "true" ]]; then
- verbosity::print_info "Docker image build is needed for ${THE_IMAGE_TYPE} image!"
+ print_info "Docker image build is needed for ${THE_IMAGE_TYPE} image!"
else
- verbosity::print_info "Docker image build is not needed for ${THE_IMAGE_TYPE} image!"
+ print_info "Docker image build is not needed for ${THE_IMAGE_TYPE} image!"
fi
fi
- verbosity::print_info
+ print_info
}
diff --git a/scripts/ci/libraries/_parameters.sh b/scripts/ci/libraries/_parameters.sh
index 13996c0..7828bb5 100644
--- a/scripts/ci/libraries/_parameters.sh
+++ b/scripts/ci/libraries/_parameters.sh
@@ -17,12 +17,12 @@
# under the License.
# Reads environment variable passed as first parameter from the .build cache file
-function parameters::read_from_file {
+function read_from_file {
cat "${BUILD_CACHE_DIR}/.$1" 2>/dev/null || true
}
# Saves environment variable passed as first parameter to the .build cache file
-function parameters::save_to_file {
+function save_to_file {
# shellcheck disable=SC2005
echo "$(eval echo "\$$1")" > "${BUILD_CACHE_DIR}/.$1"
}
@@ -31,7 +31,7 @@ function parameters::save_to_file {
# and if it is, it saves it to .build cache file. In case the parameter is wrong, the
# saved variable is removed (so that bad value is not used again in case it comes from there)
# and exits with an error
-function parameters::check_and_save_allowed_param {
+function check_and_save_allowed_param {
_VARIABLE_NAME="${1}"
_VARIABLE_DESCRIPTIVE_NAME="${2}"
_FLAG="${3}"
@@ -54,5 +54,5 @@ function parameters::check_and_save_allowed_param {
fi
exit 1
fi
- parameters::save_to_file "${_VARIABLE_NAME}"
+ save_to_file "${_VARIABLE_NAME}"
}
diff --git a/scripts/ci/libraries/_permissions.sh b/scripts/ci/libraries/_permissions.sh
index 9d13b7c..6decd70 100644
--- a/scripts/ci/libraries/_permissions.sh
+++ b/scripts/ci/libraries/_permissions.sh
@@ -32,28 +32,28 @@
# Since we can't (easily) tell what dockerignore would restrict, we'll just to
# it to "all" files in the git repo, making sure to exclude the www/static/docs
# symlink which is broken until the docs are built.
-function permissions::filterout_deleted_files {
+function filterout_deleted_files {
# Take NUL-separated stdin, return only files that exist on stdout NUL-separated
# This is to cope with users deleting files or folders locally but not doing `git rm`
xargs -0 "$STAT_BIN" --printf '%n\0' 2>/dev/null || true;
}
-# Fixes permissions for groups for all the files that are quickly filtered using the permissions::filterout_deleted_files
-function permissions::fix_group_permissions() {
+# Fixes permissions for groups for all the files that are quickly filtered using the filterout_deleted_files
+function fix_group_permissions() {
if [[ ${PERMISSIONS_FIXED:=} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Permissions already fixed"
- verbosity::print_info
+ print_info
+ print_info "Permissions already fixed"
+ print_info
return
fi
- verbosity::print_info
+ print_info
pushd "${AIRFLOW_SOURCES}" >/dev/null || exit 1
# This deals with files
- git ls-files -z -- ./ | permissions::filterout_deleted_files | xargs -0 chmod og-w
+ git ls-files -z -- ./ | filterout_deleted_files | xargs -0 chmod og-w
# and this deals with directories
- git ls-tree -z -r -d --name-only HEAD | permissions::filterout_deleted_files | xargs -0 chmod og-w,og+x
+ git ls-tree -z -r -d --name-only HEAD | filterout_deleted_files | xargs -0 chmod og-w,og+x
popd >/dev/null || exit 1
- verbosity::print_info "Fixed group permissions"
- verbosity::print_info
+ print_info "Fixed group permissions"
+ print_info
export PERMISSIONS_FIXED="true"
}
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 2ef699f..6e8b1a1 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -20,7 +20,7 @@
# Should be run with set +e
# Parameters:
# $1 -> image to pull
-function push_pull_remove_images::pull_image_if_not_present_or_forced() {
+function pull_image_if_not_present_or_forced() {
local IMAGE_TO_PULL="${1}"
local IMAGE_HASH
IMAGE_HASH=$(docker images -q "${IMAGE_TO_PULL}" 2> /dev/null || true)
@@ -46,22 +46,22 @@ function push_pull_remove_images::pull_image_if_not_present_or_forced() {
# Parameters:
# $1 -> DockerHub image to pull
# $2 -> GitHub image to try to pull first
-function push_pull_remove_images::pull_image_github_dockerhub() {
+function pull_image_github_dockerhub() {
local DOCKERHUB_IMAGE="${1}"
local GITHUB_IMAGE="${2}"
set +e
- if push_pull_remove_images::pull_image_if_not_present_or_forced "${GITHUB_IMAGE}"; then
+ if pull_image_if_not_present_or_forced "${GITHUB_IMAGE}"; then
# Tag the image to be the DockerHub one
docker tag "${GITHUB_IMAGE}" "${DOCKERHUB_IMAGE}"
else
- push_pull_remove_images::pull_image_if_not_present_or_forced "${DOCKERHUB_IMAGE}"
+ pull_image_if_not_present_or_forced "${DOCKERHUB_IMAGE}"
fi
set -e
}
# Pulls CI image in case caching strategy is "pulled" and the image needs to be pulled
-function push_pull_remove_images::pull_ci_images_if_needed() {
+function pull_ci_images_if_needed() {
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
@@ -78,23 +78,23 @@ Docker pulling ${PYTHON_BASE_IMAGE}.
if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then
PYTHON_TAG_SUFFIX="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
- push_pull_remove_images::pull_image_github_dockerhub "${PYTHON_BASE_IMAGE}" "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
+ pull_image_github_dockerhub "${PYTHON_BASE_IMAGE}" "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
else
docker pull "${PYTHON_BASE_IMAGE}"
fi
echo
fi
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
else
- push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_CI_IMAGE}"
+ pull_image_if_not_present_or_forced "${AIRFLOW_CI_IMAGE}"
fi
fi
}
# Pulls PROD image in case caching strategy is "pulled" and the image needs to be pulled
-function push_pull_remove_images::pull_prod_images_if_needed() {
+function pull_prod_images_if_needed() {
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
echo
@@ -105,7 +105,7 @@ function push_pull_remove_images::pull_prod_images_if_needed() {
if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then
PYTHON_TAG_SUFFIX="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
- push_pull_remove_images::pull_image_github_dockerhub "${PYTHON_BASE_IMAGE}" "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
+ pull_image_github_dockerhub "${PYTHON_BASE_IMAGE}" "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${PYTHON_TAG_SUFFIX}"
else
docker pull "${PYTHON_BASE_IMAGE}"
fi
@@ -113,18 +113,18 @@ function push_pull_remove_images::pull_prod_images_if_needed() {
fi
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
# "Build" segment of production image
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_BUILD_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ pull_image_github_dockerhub "${AIRFLOW_PROD_BUILD_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
# "Main" segment of production image
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
else
- push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_PROD_BUILD_IMAGE}"
- push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_PROD_IMAGE}"
+ pull_image_if_not_present_or_forced "${AIRFLOW_PROD_BUILD_IMAGE}"
+ pull_image_if_not_present_or_forced "${AIRFLOW_PROD_IMAGE}"
fi
fi
}
# Pushes Ci images and the manifest to the registry in DockerHub.
-function push_pull_remove_images::push_ci_images_to_dockerhub() {
+function push_ci_images_to_dockerhub() {
docker push "${AIRFLOW_CI_IMAGE}"
docker tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
docker push "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
@@ -135,7 +135,7 @@ function push_pull_remove_images::push_ci_images_to_dockerhub() {
}
# Pushes Ci images and their tags to registry in GitHub
-function push_pull_remove_images::push_ci_images_to_github() {
+function push_ci_images_to_github() {
# Push image to GitHub registry with chosen push tag
# the PUSH tag might be:
# "${GITHUB_RUN_ID}" - in case of pull-request triggered 'workflow_run' builds
@@ -159,16 +159,16 @@ function push_pull_remove_images::push_ci_images_to_github() {
# Pushes Ci image and it's manifest to the registry.
-function push_pull_remove_images::push_ci_images() {
+function push_ci_images() {
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- push_pull_remove_images::push_ci_images_to_github
+ push_ci_images_to_github
else
- push_pull_remove_images::push_ci_images_to_dockerhub
+ push_ci_images_to_dockerhub
fi
}
# Pushes PROD image to registry in DockerHub
-function push_pull_remove_images::push_prod_images_to_dockerhub () {
+function push_prod_images_to_dockerhub () {
# Prod image
docker push "${AIRFLOW_PROD_IMAGE}"
if [[ -n ${DEFAULT_PROD_IMAGE=} ]]; then
@@ -181,7 +181,7 @@ function push_pull_remove_images::push_prod_images_to_dockerhub () {
# Pushes PROD image to and their tags to registry in GitHub
-function push_pull_remove_images::push_prod_images_to_github () {
+function push_prod_images_to_github () {
# Push image to GitHub registry with chosen push tag
# the PUSH tag might be:
# "${GITHUB_RUN_ID}" - in case of pull-request triggered 'workflow_run' builds
@@ -204,16 +204,16 @@ function push_pull_remove_images::push_prod_images_to_github () {
# Pushes PROD image to the registry. In case the image was taken from cache registry
# it is also pushed to the cache, not to the main registry
-function push_pull_remove_images::push_prod_images() {
+function push_prod_images() {
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- push_pull_remove_images::push_prod_images_to_github
+ push_prod_images_to_github
else
- push_pull_remove_images::push_prod_images_to_dockerhub
+ push_prod_images_to_dockerhub
fi
}
# Removes airflow CI and base images
-function push_pull_remove_images::remove_all_images() {
+function remove_all_images() {
echo
"${AIRFLOW_SOURCES}/confirm" "Removing all local images ."
echo
@@ -229,7 +229,7 @@ function push_pull_remove_images::remove_all_images() {
}
# waits for an image to be available in the github registry
-function push_pull_remove_images::wait_for_github_registry_image() {
+function wait_for_github_registry_image() {
GITHUB_API_ENDPOINT="https://${GITHUB_REGISTRY}/v2/${GITHUB_REPOSITORY_LOWERCASE}"
IMAGE_NAME="${1}"
IMAGE_TAG=${2}
diff --git a/scripts/ci/libraries/_pylint.sh b/scripts/ci/libraries/_pylint.sh
index 3e08278..df659ea 100644
--- a/scripts/ci/libraries/_pylint.sh
+++ b/scripts/ci/libraries/_pylint.sh
@@ -17,7 +17,7 @@
# under the License.
# In case of the pylint checks we filter out some files which are still in pylint_todo.txt list
-function pylint::filter_out_files_from_pylint_todo_list() {
+function filter_out_files_from_pylint_todo_list() {
FILTERED_FILES=()
set +e
for FILE in "$@"
diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh
index c762be9..67be677 100644
--- a/scripts/ci/libraries/_runs.sh
+++ b/scripts/ci/libraries/_runs.sh
@@ -17,7 +17,7 @@
# under the License.
# Docker command to build documentation
-function runs::run_docs() {
+function run_docs() {
docker run "${EXTRA_DOCKER_FLAGS[@]}" -t \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
@@ -26,7 +26,7 @@ function runs::run_docs() {
# Docker command to generate constraint files.
-function runs::run_generate_constraints() {
+function run_generate_constraints() {
docker run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
"${AIRFLOW_CI_IMAGE}" \
@@ -34,7 +34,7 @@ function runs::run_generate_constraints() {
}
# Docker command to prepare backport packages
-function runs::run_prepare_backport_packages() {
+function run_prepare_backport_packages() {
docker run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
-t \
@@ -44,7 +44,7 @@ function runs::run_prepare_backport_packages() {
}
# Docker command to generate release notes for backport packages
-function runs::run_prepare_backport_readme() {
+function run_prepare_backport_readme() {
docker run "${EXTRA_DOCKER_FLAGS[@]}" \
--entrypoint "/usr/local/bin/dumb-init" \
-t \
diff --git a/scripts/ci/libraries/_sanity_checks.sh b/scripts/ci/libraries/_sanity_checks.sh
index ed4117b..b250cf2 100644
--- a/scripts/ci/libraries/_sanity_checks.sh
+++ b/scripts/ci/libraries/_sanity_checks.sh
@@ -19,7 +19,7 @@
# Fixes a file that is expected to be a file. If - for whatever reason - the local file is not created
# When mounting it to container, docker assumes it is a missing directory and creates it. Such mistakenly
# Created directories should be removed and replaced with files
-function sanity_checks::sanitize_file() {
+function sanitize_file() {
if [[ -d "${1}" ]]; then
rm -rf "${1}"
fi
@@ -30,10 +30,10 @@ function sanity_checks::sanitize_file() {
# Those files are mounted into container when run locally
# .bash_history is preserved and you can modify .bash_aliases and .inputrc
# according to your liking
-function sanity_checks::sanitize_mounted_files() {
- sanity_checks::sanitize_file "${AIRFLOW_SOURCES}/.bash_history"
- sanity_checks::sanitize_file "${AIRFLOW_SOURCES}/.bash_aliases"
- sanity_checks::sanitize_file "${AIRFLOW_SOURCES}/.inputrc"
+function sanitize_mounted_files() {
+ sanitize_file "${AIRFLOW_SOURCES}/.bash_history"
+ sanitize_file "${AIRFLOW_SOURCES}/.bash_aliases"
+ sanitize_file "${AIRFLOW_SOURCES}/.inputrc"
# When KinD cluster is created, the folder keeps authentication information
# across sessions
@@ -52,7 +52,7 @@ function sanity_checks::sanitize_mounted_files() {
#
# Checks if core utils required in the host system are installed and explain what needs to be done if not
#
-function sanity_checks::check_if_coreutils_installed() {
+function check_if_coreutils_installed() {
set +e
getopt -T >/dev/null
GETOPT_RETVAL=$?
@@ -76,7 +76,7 @@ function sanity_checks::check_if_coreutils_installed() {
#################### Parsing options/arguments
if [[ ${GETOPT_RETVAL} != 4 || "${STAT_PRESENT}" != "0" || "${MD5SUM_PRESENT}" != "0" ]]; then
- verbosity::print_info
+ print_info
if [[ $(uname -s) == 'Darwin' ]] ; then
echo >&2 "You are running ${CMDNAME} in OSX environment"
echo >&2 "And you need to install gnu commands"
@@ -105,7 +105,7 @@ function sanity_checks::check_if_coreutils_installed() {
echo >&2 "Please install latest/GNU version of getopt and coreutils."
echo >&2 "This can usually be done with 'apt install util-linux coreutils'"
fi
- verbosity::print_info
+ print_info
exit 1
fi
}
@@ -113,7 +113,7 @@ function sanity_checks::check_if_coreutils_installed() {
#
# Asserts that we are not inside of the container
#
-function sanity_checks::assert_not_in_container() {
+function assert_not_in_container() {
if [[ ${SKIP_IN_CONTAINER_CHECK:=} == "true" ]]; then
return
fi
@@ -129,21 +129,21 @@ function sanity_checks::assert_not_in_container() {
}
# Changes directory to local sources
-function sanity_checks::go_to_airflow_sources {
- verbosity::print_info
+function go_to_airflow_sources {
+ print_info
pushd "${AIRFLOW_SOURCES}" &>/dev/null || exit 1
- verbosity::print_info
- verbosity::print_info "Running in host in $(pwd)"
- verbosity::print_info
+ print_info
+ print_info "Running in host in $(pwd)"
+ print_info
}
#
# Performs basic sanity checks common for most of the scripts in this directory
#
-function sanity_checks::basic_sanity_checks() {
- sanity_checks::assert_not_in_container
- initialization::set_default_python_version_if_empty
- sanity_checks::go_to_airflow_sources
- sanity_checks::check_if_coreutils_installed
- sanity_checks::sanitize_mounted_files
+function basic_sanity_checks() {
+ assert_not_in_container
+ set_default_python_version_if_empty
+ go_to_airflow_sources
+ check_if_coreutils_installed
+ sanitize_mounted_files
}
diff --git a/scripts/ci/libraries/_script_init.sh b/scripts/ci/libraries/_script_init.sh
index 8e9db27..4e11e27 100755
--- a/scripts/ci/libraries/_script_init.sh
+++ b/scripts/ci/libraries/_script_init.sh
@@ -24,18 +24,18 @@ readonly AIRFLOW_SOURCES
# shellcheck source=scripts/ci/libraries/_all_libs.sh
. "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-initialization::initialize_common_environment
+initialize_common_environment
-sanity_checks::basic_sanity_checks
+basic_sanity_checks
-start_end::script_start
+script_start
-build_images::determine_docker_cache_strategy
+determine_docker_cache_strategy
-initialization::get_environment_for_builds_on_ci
+get_environment_for_builds_on_ci
-build_images::get_docker_image_names
+get_docker_image_names
-initialization::make_constants_read_only
+make_constants_read_only
-traps::add_trap start_end::script_end EXIT HUP INT TERM
+add_trap script_end EXIT HUP INT TERM
diff --git a/scripts/ci/libraries/_spinner.sh b/scripts/ci/libraries/_spinner.sh
index 363b31e..c4ce390 100644
--- a/scripts/ci/libraries/_spinner.sh
+++ b/scripts/ci/libraries/_spinner.sh
@@ -19,7 +19,7 @@
# Function to spin ASCII spinner during pull and build in pre-commits to give the user indication that
# Pull/Build is happening. It only spins if the output log changes, so if pull/build is stalled
# The spinner will not move.
-function spinner::spin() {
+function spin() {
local FILE_TO_MONITOR=${1}
local SPIN=("-" "\\" "|" "/")
echo -n "
diff --git a/scripts/ci/libraries/_start_end.sh b/scripts/ci/libraries/_start_end.sh
index 203cfb9..635ad88 100644
--- a/scripts/ci/libraries/_start_end.sh
+++ b/scripts/ci/libraries/_start_end.sh
@@ -21,24 +21,24 @@
# If VERBOSE_COMMANDS variable is set to true, it enables verbose output of commands executed
# Also prints some useful diagnostics information at start of the script if VERBOSE is set to true
#
-function start_end::script_start {
- verbosity::print_info
- verbosity::print_info "Running $(basename "$0")"
- verbosity::print_info
- verbosity::print_info "Log is redirected to ${OUTPUT_LOG}"
- verbosity::print_info
+function script_start {
+ print_info
+ print_info "Running $(basename "$0")"
+ print_info
+ print_info "Log is redirected to ${OUTPUT_LOG}"
+ print_info
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Variable VERBOSE_COMMANDS Set to \"true\""
- verbosity::print_info "You will see a lot of output"
- verbosity::print_info
+ print_info
+ print_info "Variable VERBOSE_COMMANDS Set to \"true\""
+ print_info "You will see a lot of output"
+ print_info
set -x
else
- verbosity::print_info "You can increase verbosity by running 'export VERBOSE_COMMANDS=\"true\""
+ print_info "You can increase verbosity by running 'export VERBOSE_COMMANDS=\"true\""
if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then
- verbosity::print_info "And skip deleting the output file with 'export SKIP_CACHE_DELETION=\"true\""
+ print_info "And skip deleting the output file with 'export SKIP_CACHE_DELETION=\"true\""
fi
- verbosity::print_info
+ print_info
set +x
fi
START_SCRIPT_TIME=$(date +%s)
@@ -50,7 +50,7 @@ function start_end::script_start {
# command verbosity and in case the script was not run from Breeze (so via ci scripts) it displays
# total time spent in the script so that we can easily see it.
#
-function start_end::script_end {
+function script_end {
#shellcheck disable=2181
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
@@ -58,9 +58,9 @@ function start_end::script_end {
if [[ -f "${OUTPUT_LOG}" ]]; then
cat "${OUTPUT_LOG}"
fi
- verbosity::print_info "###########################################################################################"
- verbosity::print_info " EXITING WITH STATUS CODE ${EXIT_CODE}"
- verbosity::print_info "###########################################################################################"
+ print_info "###########################################################################################"
+ print_info " EXITING WITH STATUS CODE ${EXIT_CODE}"
+ print_info "###########################################################################################"
fi
if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
set +x
@@ -73,10 +73,10 @@ function start_end::script_end {
END_SCRIPT_TIME=$(date +%s)
RUN_SCRIPT_TIME=$((END_SCRIPT_TIME-START_SCRIPT_TIME))
if [[ ${BREEZE:=} != "true" ]]; then
- verbosity::print_info
- verbosity::print_info "Finished the script $(basename "$0")"
- verbosity::print_info "Elapsed time spent in the script: ${RUN_SCRIPT_TIME} seconds"
- verbosity::print_info "Exit code ${EXIT_CODE}"
- verbosity::print_info
+ print_info
+ print_info "Finished the script $(basename "$0")"
+ print_info "Elapsed time spent in the script: ${RUN_SCRIPT_TIME} seconds"
+ print_info "Exit code ${EXIT_CODE}"
+ print_info
fi
}
diff --git a/scripts/ci/libraries/_traps.sh b/scripts/ci/libraries/_traps.sh
index db65187..be2e31b 100644
--- a/scripts/ci/libraries/_traps.sh
+++ b/scripts/ci/libraries/_traps.sh
@@ -24,7 +24,7 @@
# trap to set
# .... list of signals to handle
#######################################################################################################
-function traps::add_trap() {
+function add_trap() {
trap="${1}"
shift
for signal in "${@}"
diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh
index 7420dc1..aa4f096 100644
--- a/scripts/ci/libraries/_verbosity.sh
+++ b/scripts/ci/libraries/_verbosity.sh
@@ -16,6 +16,13 @@
# specific language governing permissions and limitations
# under the License.
# In case "VERBOSE_COMMANDS" is set to "true" set -x is used to enable debugging
+function check_verbose_setup {
+ if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
+ set -x
+ else
+ set +x
+ fi
+}
DOCKER_BINARY_PATH="${DOCKER_BINARY_PATH:=$(command -v docker || echo "/bin/docker")}"
export DOCKER_BINARY_PATH
@@ -101,13 +108,13 @@ function kind {
}
# Prints verbose information in case VERBOSE variable is set
-function verbosity::print_info() {
+function print_info() {
if [[ ${VERBOSE:="false"} == "true" && ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then
echo "$@"
fi
}
-function verbosity::set_verbosity() {
+function set_verbosity() {
# whether verbose output should be produced
export VERBOSE=${VERBOSE:="false"}
@@ -118,4 +125,4 @@ function verbosity::set_verbosity() {
export PRINT_INFO_FROM_SCRIPTS=${PRINT_INFO_FROM_SCRIPTS:="true"}
}
-verbosity::set_verbosity
+set_verbosity
diff --git a/scripts/ci/pre_commit/pre_commit_ci_build.sh b/scripts/ci/pre_commit/pre_commit_ci_build.sh
index d92f5cb..646377e 100755
--- a/scripts/ci/pre_commit/pre_commit_ci_build.sh
+++ b/scripts/ci/pre_commit/pre_commit_ci_build.sh
@@ -23,8 +23,8 @@ export SKIP_CHECK_REMOTE_IMAGE="true"
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-build_images::forget_last_answer
+forget_last_answer
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed_and_confirmed
+rebuild_ci_image_if_needed_and_confirmed
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
index 6e8b0a7..c24ee29 100755
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -24,16 +24,16 @@ export SKIP_CHECK_REMOTE_IMAGE="true"
TMP_OUTPUT=$(mktemp)
# Remove temp file if it's hanging around
-traps::add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM
+add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM
LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml"
LEAD=' # START automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh'
TAIL=' # END automatically generated volumes from LOCAL_MOUNTS in _local_mounts.sh'
-local_mounts::generate_local_mounts_list " - ../../../"
+generate_local_mounts_list " - ../../../"
-sed "/$LEAD/q" "${LOCAL_YML_FILE}" > "${TMP_OUTPUT}"
+sed -ne "0,/$LEAD/ p" "${LOCAL_YML_FILE}" > "${TMP_OUTPUT}"
printf '%s\n' "${LOCAL_MOUNTS[@]}" >> "${TMP_OUTPUT}"
sed -ne "/$TAIL/,\$ p" "${LOCAL_YML_FILE}" >> "${TMP_OUTPUT}"
diff --git a/scripts/ci/pre_commit/pre_commit_mermaid.sh b/scripts/ci/pre_commit/pre_commit_mermaid.sh
index 4485cce..b364511 100755
--- a/scripts/ci/pre_commit/pre_commit_mermaid.sh
+++ b/scripts/ci/pre_commit/pre_commit_mermaid.sh
@@ -43,7 +43,7 @@ else
fi
# shellcheck disable=SC2064
-traps::add_trap "rm -rf '${tmp_file}'" EXIT HUP INT TERM
+add_trap "rm -rf '${tmp_file}'" EXIT HUP INT TERM
for file in "${@}"
do
diff --git a/scripts/ci/static_checks/flake8.sh b/scripts/ci/static_checks/flake8.sh
index 37a030a..e14924e 100755
--- a/scripts/ci/static_checks/flake8.sh
+++ b/scripts/ci/static_checks/flake8.sh
@@ -34,8 +34,8 @@ function run_flake8() {
fi
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
run_flake8 "$@"
diff --git a/scripts/ci/static_checks/mypy.sh b/scripts/ci/static_checks/mypy.sh
index 8941851..47770c4 100755
--- a/scripts/ci/static_checks/mypy.sh
+++ b/scripts/ci/static_checks/mypy.sh
@@ -31,8 +31,8 @@ function run_mypy() {
"--" "/opt/airflow/scripts/in_container/run_mypy.sh" "${FILES[@]}"
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
run_mypy "$@"
diff --git a/scripts/ci/static_checks/pylint.sh b/scripts/ci/static_checks/pylint.sh
index d677617..6d30f5c 100755
--- a/scripts/ci/static_checks/pylint.sh
+++ b/scripts/ci/static_checks/pylint.sh
@@ -33,12 +33,12 @@ function run_pylint() {
fi
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
if [[ "${#@}" != "0" ]]; then
- pylint::filter_out_files_from_pylint_todo_list "$@"
+ filter_out_files_from_pylint_todo_list "$@"
if [[ "${#FILTERED_FILES[@]}" == "0" ]]; then
echo "Filtered out all files. Skipping pylint."
diff --git a/scripts/ci/static_checks/refresh_pylint_todo.sh b/scripts/ci/static_checks/refresh_pylint_todo.sh
index 7c4c35c..0473410 100755
--- a/scripts/ci/static_checks/refresh_pylint_todo.sh
+++ b/scripts/ci/static_checks/refresh_pylint_todo.sh
@@ -26,8 +26,8 @@ function refresh_pylint_todo() {
/opt/airflow/scripts/in_container/refresh_pylint_todo.sh
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
refresh_pylint_todo
diff --git a/scripts/ci/static_checks/run_static_checks.sh b/scripts/ci/static_checks/run_static_checks.sh
index 544a22b..6400c8d 100755
--- a/scripts/ci/static_checks/run_static_checks.sh
+++ b/scripts/ci/static_checks/run_static_checks.sh
@@ -25,9 +25,9 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
exit
fi
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
python -m pip install pre-commit \
--constraint "https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index a9770fe..9ef3c23 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -73,9 +73,9 @@ function run_airflow_testing_in_docker() {
return "${EXIT_CODE}"
}
-build_images::prepare_ci_build
+prepare_ci_build
-build_images::rebuild_ci_image_if_needed
+rebuild_ci_image_if_needed
# Test environment
export BACKEND=${BACKEND:="sqlite"}
diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash
index 9014382..2df8786 100644
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/bats_utils.bash
@@ -28,6 +28,6 @@ export SKIP_IN_CONTAINER_CHECK="true"
# shellcheck source=scripts/ci/libraries/_all_libs.sh
source "scripts/ci/libraries/_all_libs.sh"
-initialization::initialize_common_environment
+initialize_common_environment
-sanity_checks::basic_sanity_checks
+basic_sanity_checks
diff --git a/tests/bats/test_breeze_params.bats b/tests/bats/test_breeze_params.bats
index 02e8255..0795ee2 100644
--- a/tests/bats/test_breeze_params.bats
+++ b/tests/bats/test_breeze_params.bats
@@ -30,7 +30,7 @@ teardown() {
@test "Test missing value for a parameter" {
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
- run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
+ run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") - <<EOF
ERROR: Allowed Test Param: [ a b c ]. Is: ''.
@@ -44,7 +44,7 @@ EOF
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=x
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
- run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
+ run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") - <<EOF
ERROR: Allowed Test Param: [ a b c ]. Is: 'x'.
@@ -60,7 +60,7 @@ EOF
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=x
echo "x" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
- run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
+ run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") - <<EOF
ERROR: Allowed Test Param: [ a b c ]. Is: 'x'.
@@ -77,7 +77,7 @@ EOF
@test "Test correct value for a parameter" {
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=a
- run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
+ run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") <(echo "")
[ -f "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" ]
diff <(echo "a") <(cat "${AIRFLOW_SOURCES}/.build/.TEST_PARAM")
@@ -93,7 +93,7 @@ EOF
)
export _BREEZE_ALLOWED_TEST_PARAMS
export TEST_PARAM=a
- run parameters::check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
+ run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") <(echo "")
[ -f "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" ]
diff <(echo "a") <(cat "${AIRFLOW_SOURCES}/.build/.TEST_PARAM")
@@ -102,7 +102,7 @@ EOF
@test "Test read_parameter from missing file" {
- run parameters::read_from_file TEST_PARAM
+ run read_from_file TEST_PARAM
[ -z "${TEST_FILE}" ]
diff <(echo "${output}") <(echo "")
[ ! -f "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" ]
@@ -111,7 +111,7 @@ EOF
@test "Test read_parameter from file" {
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
- run parameters::read_from_file TEST_PARAM
+ run read_from_file TEST_PARAM
diff <(echo "${output}") <(echo "a")
[ -f "${AIRFLOW_SOURCES}/.build/.TEST_PARAM" ]
[ "${status}" == "0" ]
diff --git a/tests/bats/test_local_mounts.bats b/tests/bats/test_local_mounts.bats
index 807b68e..aef961f 100644
--- a/tests/bats/test_local_mounts.bats
+++ b/tests/bats/test_local_mounts.bats
@@ -21,7 +21,7 @@
@test "convert volume list to docker params" {
load bats_utils
- read -r -a RES <<< "$(local_mounts::convert_local_mounts_to_docker_params)"
+ read -r -a RES <<< "$(convert_local_mounts_to_docker_params)"
[[ ${#RES[@]} -gt 0 ]] # Array should be non-zero length
[[ $((${#RES[@]} % 2)) == 0 ]] # Array should be even length