You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/09/02 16:02:42 UTC
[airflow] 02/02: Revert "Implement Google Shell Conventions for breeze script … (#10651)"
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch revert-breeze
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit beceab5ac0dcc92be7a0b2dd6604b6ca53b52391
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Wed Sep 2 17:01:36 2020 +0100
Revert "Implement Google Shell Conventions for breeze script … (#10651)"
This reverts commit 46c8d6714c981746cc114b8b1af5cb27aa0018e2.
---
BREEZE.rst | 8 +-
breeze | 2571 ++++++++------------
breeze-complete | 20 +-
confirm | 2 +-
.../ci_prepare_and_test_backport_packages.sh | 2 +
.../ci_prepare_backport_packages.sh | 4 +
.../ci_prepare_backport_readme.sh | 4 +
...ci_test_backport_packages_import_all_classes.sh | 4 +
...ci_test_backport_packages_install_separately.sh | 4 +
scripts/ci/constraints/ci_branch_constraints.sh | 4 +
scripts/ci/constraints/ci_commit_constraints.sh | 4 +
scripts/ci/constraints/ci_generate_constraints.sh | 4 +
scripts/ci/docs/ci_docs.sh | 4 +
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 1 +
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 1 +
scripts/ci/images/ci_push_ci_images.sh | 2 +
scripts/ci/images/ci_push_production_images.sh | 2 +
scripts/ci/images/ci_wait_for_all_ci_images.sh | 26 +-
scripts/ci/images/ci_wait_for_all_prod_images.sh | 29 +-
.../ci/kubernetes/ci_deploy_app_to_kubernetes.sh | 12 +-
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 11 +-
scripts/ci/libraries/_all_libs.sh | 10 +-
scripts/ci/libraries/_build_images.sh | 210 +-
scripts/ci/libraries/_initialization.sh | 251 +-
scripts/ci/libraries/_kind.sh | 56 +-
scripts/ci/libraries/_md5sum.sh | 8 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 37 +-
scripts/ci/libraries/_sanity_checks.sh | 25 +-
scripts/ci/libraries/_script_init.sh | 23 +-
scripts/ci/libraries/_start_end.sh | 1 +
scripts/ci/libraries/_traps.sh | 38 -
scripts/ci/libraries/_verbosity.sh | 48 +-
scripts/ci/openapi/client_codegen_diff.sh | 1 +
.../ci/pre_commit/pre_commit_breeze_cmd_line.sh | 16 -
.../ci/pre_commit/pre_commit_check_integrations.sh | 5 +-
scripts/ci/pre_commit/pre_commit_ci_build.sh | 1 +
.../ci/pre_commit/pre_commit_local_yml_mounts.sh | 4 +-
scripts/ci/pre_commit/pre_commit_mermaid.sh | 9 +-
scripts/ci/static_checks/bat_tests.sh | 4 +
scripts/ci/static_checks/flake8.sh | 4 +
scripts/ci/static_checks/mypy.sh | 4 +
scripts/ci/static_checks/pylint.sh | 4 +
scripts/ci/static_checks/refresh_pylint_todo.sh | 4 +-
scripts/ci/static_checks/run_static_checks.sh | 4 +
scripts/ci/testing/ci_run_airflow_testing.sh | 7 +-
scripts/ci/tools/ci_clear_tmp.sh | 16 +
scripts/ci/tools/ci_count_changed_files.sh | 3 +
scripts/ci/tools/ci_fix_ownership.sh | 16 +
scripts/in_container/_in_container_script_init.sh | 2 +-
scripts/in_container/_in_container_utils.sh | 25 +-
scripts/in_container/check_environment.sh | 4 +-
scripts/in_container/entrypoint_ci.sh | 6 +-
scripts/in_container/prod/entrypoint_prod.sh | 4 +-
scripts/in_container/run_cli_tool.sh | 10 +-
scripts/in_container/run_docs_build.sh | 5 +-
scripts/in_container/run_generate_constraints.sh | 5 +-
.../in_container/run_prepare_backport_packages.sh | 8 +-
.../in_container/run_prepare_backport_readme.sh | 5 +-
tests/bats/bats_utils.bash | 17 +-
tests/bats/test_breeze_params.bats | 26 +
tests/bats/test_empty_test.bats | 2 +
tests/bats/test_local_mounts.bats | 2 +
62 files changed, 1467 insertions(+), 2182 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 5d87345..2ffa186 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1048,10 +1048,8 @@ This is the current syntax for `./breeze <./breeze>`_:
Usage: breeze [FLAGS] [COMMAND] -- <EXTRA_ARGS>
- By default the script enters the CI container and drops you to bash shell, but you can choose
- one of the commands to run specific actions instead.
-
- Add --help after each command to see details:
+ By default the script enters IT environment and drops you to bash shell, but you can choose one
+ of the commands to run specific actions instead. Add --help after each command to see details:
Commands without arguments:
@@ -1801,7 +1799,7 @@ This is the current syntax for `./breeze <./breeze>`_:
Detailed usage for command: static-check
- breeze static-check [FLAGS] static_check [-- <EXTRA_ARGS>]
+ breeze static-check [FLAGS] STATIC_CHECK [-- <EXTRA_ARGS>]
Run selected static checks for currently changed files. You should specify static check that
you would like to run or 'all' to run all checks. One of:
diff --git a/breeze b/breeze
index d03e556..1e4a939 100755
--- a/breeze
+++ b/breeze
@@ -18,81 +18,83 @@
# under the License.
set -euo pipefail
-AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-export AIRFLOW_SOURCES
-readonly AIRFLOW_SOURCES
+AIRFLOW_SOURCES="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Bash arrays need to be defined outside of functions unfortunately :(
-# Because on Mac OS Bash 3.4 defining arrays inside functions does not work
# Array with extra options for Docker compose
declare -a EXTRA_DC_OPTIONS
-export EXTRA_DC_OPTIONS
-
# Array with selected integrations
declare -a INTEGRATIONS
-export INTEGRATIONS
-
# This is where remaining args are passed
declare -a REMAINING_ARGS
-export REMAINING_ARGS
-
# This is where static check options are defined
declare -a EXTRA_STATIC_CHECK_OPTIONS
-export EXTRA_STATIC_CHECK_OPTIONS
-#######################################################################################################
-# Sets up all the default variables for Breeze. They are needed by all other functions
-# All those variables are exported. They are not set to read-only because those
-# defaults can be modified later on when command line arguments are parsed
-# and variables stored in .build directory (stored in the previous run) are read
-#
-# Used globals:
-# FORCE_SCREEN_WIDTH
-#
-# Modified globals (constants or candidates for constants after we override them via appropriate flags):
-#
-# BREEZE
-# SUPPRESS_CHEATSHEET_FILE
-# SUPPRESS_ASCIIART_FILE
-# MAX_SCREEN_WIDTH
-# SCREEN_WIDTH
-# MOUNT_LOCAL_SOURCES
-# MOUNT_FILES
-# FORCE_PULL_IMAGES
-# ENABLE_KIND_CLUSTER
-# FORWARD_CREDENTIALS
-# DB_RESET
-# INSTALL_AIRFLOW_VERSION
-# INSTALL_AIRFLOW_REFERENCE
-# FORCE_BUILD_IMAGES
-# PRODUCTION_IMAGE
-# PYTHON_MAJOR_MINOR_VERSION
-#
-# Global variables:
-#
-# command_to_run
-# second_command_to_run
-# docker_compose_command
-#
-# Also it sets the variables and globals set by common initialization functions from
-# scripts/ci/libraries/_initialization.sh and breeze-complete script (which sets-up auto-complete).
-#
-#######################################################################################################
-function setup_default_breeze_constants() {
+# Sets up all the default variables for Breeze
+# They are needed by all other functions
+function setup_default_breeze_variables() {
+ # Default command to run - entering breeze environment
+ COMMAND_TO_RUN="enter_breeze"
+ # In some cases we also want to run two commands in a row (for example when we restart the environment)
+ SECOND_COMMAND_TO_RUN=""
# Indicates that we are inside Breeze environment
export BREEZE=true
- readonly BREEZE
+ # Maximum screen width to print the lines spanning the whole terminal width
+ export MAX_SCREEN_WIDTH=100
+ # By default we build CI images but when we specify `--production-image` we switch to production image
+ export PRODUCTION_IMAGE="false"
+ # Directory where all CI scripts are located
+ export SCRIPTS_CI_DIR="${AIRFLOW_SOURCES}/scripts/ci"
+
+ # Directory where all the build cache is stored - we keep there status of all the docker images
+ # As well as hashes of the important files, but also we generate build scripts there that are
+ # Used to execute the commands for breeze
+ export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
+ # This folder is mounted to inside the container in /files folder. This is the way how
+ # We can exchange DAGs, scripts, packages etc with the container environment
+ export FILES_DIR="${AIRFLOW_SOURCES}/files"
+ # Temporary dir used well ... temporarily
+ export TMP_DIR="${AIRFLOW_SOURCES}/tmp"
+
+ # Create those folders above in case they do not exist
+ mkdir -p "${BUILD_CACHE_DIR}" >/dev/null
+ mkdir -p "${TMP_DIR}" >/dev/null
+ mkdir -p "${FILES_DIR}" >/dev/null
- # If those files are present, the ASCII-art/cheat-sheet are suppressed
- SUPPRESS_CHEATSHEET_FILE="${AIRFLOW_SOURCES}/.build/.suppress_cheatsheet"
- readonly SUPPRESS_CHEATSHEET_FILE
+ # load all the common functions here - those are the functions that are shared between Breeze
+ # and CI scripts (CI scripts do not use Breeze as execution environment)
+ # shellcheck source=scripts/ci/libraries/_all_libs.sh
+ . "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
- SUPPRESS_ASCIIART_FILE="${AIRFLOW_SOURCES}/.build/.suppress_asciiart"
- readonly SUPPRESS_ASCIIART_FILE
+ # We have different versions of images depending on the python version used. We keep up with the
+ # Latest patch-level changes in Python (this is done automatically during CI builds) so we have
+ # To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor
+ # version of python in X.Y format (3.6, 3.7, 3.8 etc).
+ export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
- # Maximum screen indented_screen_width to print the lines spanning the whole terminal indented_screen_width
- export MAX_SCREEN_WIDTH=100
- readonly MAX_SCREEN_WIDTH
+ # When we generate documentation for README files, we want to force the width of terminal so that
+ # No matter who is running the documentation generation gets the same output
+ if [[ ${FORCE_SCREEN_WIDTH:="false"} != "true" ]]; then
+ # Sets width of the screen from terminal
+ SCREEN_WIDTH="$(tput cols)"
+ if [[ -z ${SCREEN_WIDTH} ]]; then
+ SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
+ fi
+ if (( SCREEN_WIDTH > MAX_SCREEN_WIDTH )); then
+ SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
+ fi
+ else
+ SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
+ fi
+
+ # Name of the script is kept in this variable
+ CMDNAME="$(basename -- "$0")"
+ export CMDNAME
+
+ # Update short and long options in the breeze-complete script
+ # This way autocomplete will work automatically with all options available
+ # shellcheck source=breeze-complete
+ . "${AIRFLOW_SOURCES}/breeze-complete"
# By default we mount local Airflow sources
export MOUNT_LOCAL_SOURCES="true"
@@ -100,6 +102,19 @@ function setup_default_breeze_constants() {
# By default we mount files folder
export MOUNT_FILES="true"
+ # Holds last sub-command used - this is used by --help flag to print help for the command entered
+ export LAST_SUBCOMMAND=""
+
+ # Determines if help should be run (set to true by --help flag)
+ export RUN_HELP="false"
+
+ # Holds docker compose command if the `docker-compose` command is used.
+ export DOCKER_COMPOSE_COMMAND=""
+
+ # If set to true, the docker images are rebuilt locally. By default we assume we do not need to
+ # rebuild the image but if we find we do, this variable will be set to "true"
+ export NEEDS_DOCKER_BUILD="false"
+
# By default we only pull images if we do not have them locally.
# This can be overridden by '--force-pull-images' flag
export FORCE_PULL_IMAGES="false"
@@ -115,89 +130,55 @@ function setup_default_breeze_constants() {
# If it set is set to specified version, then the source version of Airflow
# is removed and the specified version of Airflow is installed from PyPi
- export INSTALL_AIRFLOW_VERSION=""
+ export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""}
# If it is set to specified reference (tag/branch), then the source version
# of Airflow is removed and the specified version of Airflow is installed from GitHub
- export INSTALL_AIRFLOW_REFERENCE=""
+ export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""}
# Determines whether to force build without checking if it is needed
# Can be overridden by '--force-build-images' flag.
- export FORCE_BUILD_IMAGES="false"
+ export FORCE_BUILD_IMAGES=${FORCE_BUILD_IMAGES:="false"}
- # load all the common functions here - those are the functions that are shared between Breeze
- # and CI scripts. The CI scripts do not use Breeze as driving script - they read all configuration
- # from the environment variables. That's why we keep all the common initialization in those libs
- # shellcheck source=scripts/ci/libraries/_all_libs.sh
- . "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
- # When we generate documentation for README files, we want to force the indented_screen_width of terminal so that
- # No matter who is running the documentation generation gets the same output
- if [[ ${FORCE_SCREEN_WIDTH:="false"} != "true" ]]; then
- # Sets indented_screen_width of the screen from terminal
- SCREEN_WIDTH="$(tput cols)"
- if [[ -z ${SCREEN_WIDTH=} ]]; then
- SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
- fi
- if ((SCREEN_WIDTH > MAX_SCREEN_WIDTH)); then
- SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
- fi
- else
- SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
- fi
-
- export SCREEN_WIDTH
- readonly SCREEN_WIDTH
-
- # Update short and long options in the breeze-complete script
- # This way autocomplete will work automatically with all options available
- # shellcheck source=breeze-complete
- . "${AIRFLOW_SOURCES}/breeze-complete"
-
- # Default command to run - entering breeze environment
- command_to_run="enter_breeze"
-
- # In some cases we also want to run two commands in a row (for example when we restart the environment)
- second_command_to_run=""
-
- # Determines if help should be run (set to true by --help flag)
- run_help="false"
-
- # Holds docker compose command if the `docker-compose` command is used.
- docker_compose_command=""
+ # If those files are present, the ASCII-art/cheat-sheet are suppressed
+ SUPPRESS_CHEATSHEET_FILE="${AIRFLOW_SOURCES}/.suppress_cheatsheet"
+ SUPPRESS_ASCIIART_FILE="${AIRFLOW_SOURCES}/.suppress_asciiart"
+
+ # Default values for the flags used
+
+ _BREEZE_DEFAULT_BACKEND="sqlite"
+ _BREEZE_DEFAULT_KUBERNETES_MODE="image"
+ _BREEZE_DEFAULT_KUBERNETES_VERSION="v1.18.6"
+ _BREEZE_DEFAULT_KIND_VERSION="v0.8.0"
+ _BREEZE_DEFAULT_HELM_VERSION="v3.2.4"
+ _BREEZE_DEFAULT_POSTGRES_VERSION="9.6"
+ _BREEZE_DEFAULT_POSTGRES_VERSION="9.6"
+ _BREEZE_DEFAULT_MYSQL_VERSION="5.7"
+ STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION=3.6
}
-#######################################################################################################
-#
# Initializes development-friendly virtualenv if you are already in such env. It installs all the necessary
# packages from PyPI and it case of problems it provides useful hints on what prerequisites should be
# installed. It also removes and resets the existing AIRFLOW_HOME installation to make sure that you
# have it synchronized with the version of airflow installed. It resets the airflow's sqlite database to
# a clean state. You can use this function if your virtualenv is broken, to clean it up
-#
-# Used globals:
-# PYTHON_MAJOR_MINOR_VERSION
-# AIRFLOW_HOME_DIR
-# AIRFLOW_SOURCES
-# DEFAULT_CONSTRAINTS_BRANCH
-# OSTYPE
-#
-#######################################################################################################
function initialize_virtualenv() {
- # Check if we are inside virtualenv
- set +e
- echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" |
- python"${PYTHON_MAJOR_MINOR_VERSION}"
- local res=$?
- set -e
- if [[ ${res} != "0" ]]; then
+ # Check if we are inside virtualenv
+ set +e
+ echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n sys.exit(1)" | "python${PYTHON_MAJOR_MINOR_VERSION}"
+ RES=$?
+ set -e
+ if [[ ${RES} != "0" ]]; then
echo >&2
echo >&2 "ERROR: Initializing local virtualenv only works when you have virtualenv activated"
echo >&2
echo >&2 "Please enter your local virtualenv before (for example using 'pyenv activate' or 'workon') "
echo >&2
exit 1
- else
+ else
+ # If no Airflow Home defined - fallback to ${HOME}/airflow
+ AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow}
+ export CASS_DRIVER_NO_CYTHON="1"
echo
echo "Initializing the virtualenv: $(command -v python)!"
echo
@@ -207,18 +188,18 @@ function initialize_virtualenv() {
echo
pushd "${AIRFLOW_SOURCES}"
set +e
- # We need to export this one to speed up Cassandra driver installation in virtualenv
- CASS_DRIVER_NO_CYTHON="1" pip install -e ".[devel]" \
+ pip install -e ".[devel]" \
--constraint "https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
- res=$?
+ RES=$?
set -e
popd
- if [[ ${res} != "0" ]]; then
+ if [[ ${RES} != "0" ]]; then
echo "#######################################################################"
echo " You had some troubles installing the venv !!!!!"
echo " Try running the command below and rerun virtualenv installation"
echo
- if [[ ${OSTYPE} == "darwin"* ]]; then
+ SYSTEM=$(uname -s)
+ if [[ ${SYSTEM} == "Darwin" ]]; then
echo " brew install sqlite mysql postgresql openssl"
else
echo " sudo apt install build-essentials python3.6-dev python3.7-dev python3.8-dev python-dev openssl \\"
@@ -226,7 +207,7 @@ function initialize_virtualenv() {
fi
echo
echo "#######################################################################"
- exit ${res}
+ exit ${RES}
fi
echo
echo "Wiping and recreating ${AIRFLOW_HOME_DIR}"
@@ -236,26 +217,18 @@ function initialize_virtualenv() {
echo
echo "Resetting AIRFLOW sqlite database"
echo
- AIRFLOW__CORE__UNIT_TEST_MODE="False" airflow db reset -y
+ unset AIRFLOW__CORE__UNIT_TEST_MODE
+ airflow db reset -y
echo
echo "Resetting AIRFLOW sqlite unit test database"
echo
- AIRFLOW__CORE__UNIT_TEST_MODE="True" airflow db reset -y
+ AIRFLOW__CORE__UNIT_TEST_MODE=True airflow db reset -y
exit 0
- fi
+ fi
}
-#######################################################################################################
-#
+
# Sets up autocomplete for Breeze for both - bash and zsh
-#
-# Used globals:
-#
-# AIRFLOW_SOURCES
-# HOME
-# OSTYPE
-#
-#######################################################################################################
function setup_autocomplete() {
echo "Installing bash/zsh completion for local user"
echo
@@ -267,17 +240,17 @@ function setup_autocomplete() {
echo
echo "Breeze Bash completion is now linked to: ${AIRFLOW_SOURCES}/breeze-complete"
echo
- local breeze_comment="Added by Airflow Breeze autocomplete setup"
- if ! grep "${breeze_comment}" "${HOME}/.bashrc" >/dev/null 2>&1; then
+ local BREEZE_COMMENT="Added by Airflow Breeze autocomplete setup"
+ if ! grep "${BREEZE_COMMENT}" "${HOME}/.bashrc" >/dev/null 2>&1; then
touch ~/.bashrc
# shellcheck disable=SC2129
- echo "# START: ${breeze_comment}" >>~/.bashrc
+ echo "# START: ${BREEZE_COMMENT}" >>~/.bashrc
cat <<"EOF" >>~/.bashrc
-for bcfile in ~/.bash_completion.d/* ; do
- . ${bcfile}
+for BCFILE in ~/.bash_completion.d/* ; do
+ . ${BCFILE}
done
EOF
- echo "# END: ${breeze_comment}" >>~/.bashrc
+ echo "# END: ${BREEZE_COMMENT}" >>~/.bashrc
echo
echo "The ${HOME}/.bashrc has been modified"
echo
@@ -286,15 +259,15 @@ EOF
echo "The ${HOME}/.bashrc was already modified before. Not changing it."
echo
fi
- if ! grep "${breeze_comment}" "${HOME}/.zshrc" >/dev/null 2>&1; then
+ if ! grep "${BREEZE_COMMENT}" "${HOME}/.zshrc" >/dev/null 2>&1; then
# shellcheck disable=SC2129
- echo "# START: ${breeze_comment}" >>~/.zshrc
+ echo "# START: ${BREEZE_COMMENT}" >>~/.zshrc
cat <<"EOF" >>~/.zshrc
autoload compinit && compinit
autoload bashcompinit && bashcompinit
source ~/.bash_completion.d/breeze-complete
EOF
- echo "# END: ${breeze_comment}" >>~/.zshrc
+ echo "# END: ${BREEZE_COMMENT}" >>~/.zshrc
echo
echo "The ${HOME}/.zshrc has been modified"
echo
@@ -307,15 +280,15 @@ EOF
# For MacOS we have to handle the special case where terminal app DOES NOT run .bashrc by default
# But re-runs .bash_profile :(
# See https://scriptingosx.com/2017/04/about-bash_profile-and-bashrc-on-macos/
- if ! grep "${breeze_comment}" "${HOME}/.bash_profile"; then
+ if ! grep "${BREEZE_COMMENT}" "${HOME}/.bash_profile"; then
# shellcheck disable=SC2129
- echo "# START: ${breeze_comment}" >>~/.bash_profile
+ echo "# START: ${BREEZE_COMMENT}" >>~/.bash_profile
cat <<"EOF" >>~/.bash_profile
if [ -r ~/.bashrc ]; then
source ~/.bashrc
fi
EOF
- echo "# END: ${breeze_comment}" >>~/.bash_profile
+ echo "# END: ${BREEZE_COMMENT}" >>~/.bash_profile
echo
echo "The ${HOME}/.bash_profile has been modified"
echo
@@ -336,41 +309,17 @@ EOF
exit 0
}
-#######################################################################################################
-#
# Prints information about the current configuration of Breeze - if you enter breeze interactively
-# and you did not suppress cheatsheet or asciiart, it also prints those. It also prints values
-# of constants set by read_saved_environment_variables() function and other initialization functions.
-#
-# Used globals:
-#
-# BACKEND
-# POSTGRES_VERSION
-# MYSQL_VERSION
-# SUPPRESS_CHEATSHEET_FILE
-# SUPPRESS_ASCIIART_FILE
-# PRODUCTION_IMAGE
-# BRANCH_NAME
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-# INSTALL_AIRFLOW_VERSION
-# INSTALL_AIRFLOW_REFERENCE
-#
-# Outputs:
-# Prints the information about the build to stdout.
-#
-#######################################################################################################
-function print_badge() {
- local backend_version=""
+# and you did not suppress cheatsheet or asciiart, it also prints those
+function print_badge {
if [[ ${BACKEND} == "postgres" ]]; then
- backend_version="${POSTGRES_VERSION}"
+ BACKEND_VERSION="${POSTGRES_VERSION}"
elif [[ ${BACKEND} == "mysql" ]]; then
- backend_version="${MYSQL_VERSION}"
+ BACKEND_VERSION="${MYSQL_VERSION}"
+ else
+ BACKEND_VERSION=""
fi
- if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" && ${command_to_run} == "enter_breeze" ]]; then
+ if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" && ${COMMAND_TO_RUN} == "enter_breeze" ]]; then
cat <<EOF
@@ -437,12 +386,12 @@ EOF
Python version: ${PYTHON_MAJOR_MINOR_VERSION}
DockerHub user: ${DOCKERHUB_USER}
DockerHub repo: ${DOCKERHUB_REPO}
- Backend: ${BACKEND} ${backend_version}
+ Backend: ${BACKEND} ${BACKEND_VERSION}
EOF
- if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE=} ]]; then
+ if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
cat <<EOF
- Airflow installed: ${INSTALL_AIRFLOW_VERSION=}${INSTALL_AIRFLOW_REFERENCE=}
+ Airflow installed from: ${INSTALL_AIRFLOW_VERSION}${INSTALL_AIRFLOW_REFERENCE}
EOF
fi
@@ -470,9 +419,9 @@ EOF
Python version: ${PYTHON_MAJOR_MINOR_VERSION}
DockerHub user: ${DOCKERHUB_USER}
DockerHub repo: ${DOCKERHUB_REPO}
- Backend: ${BACKEND} ${backend_version}
+ Backend: ${BACKEND} ${BACKEND_VERSION}
EOF
- if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE=} ]]; then
+ if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
cat <<EOF
Airflow installed from: ${INSTALL_AIRFLOW_VERSION}${INSTALL_AIRFLOW_REFERENCE}
@@ -482,72 +431,36 @@ EOF
fi
}
-#######################################################################################################
-#
-# Prepares command file that can be used to easily run the docker commands outside of Breeze.
-#
+# Prepares command file that can be used to easily run the commands without the need of using Breeze
# The command file generated in cache ./build directory is a standalone script that contains
-# All the environment variables and docker-compose configuration to run the command.
-# This is because depending on configuration of Breeze we might have different compose files
-# used and different env variables set.
-#
-# Those are a convenience scripts that you might use to debug command execution although
-# In most cases they are used internally by Breeze.
-#
-# Used Globals:
-# BRANCH_NAME
-# PYTHON_MAJOR_MINOR_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-# HOST_AIRFLOW_SOURCES
-# BACKEND
-# AIRFLOW_VERSION
-# INSTALL_AIRFLOW_VERSION
-# WEBSERVER_HOST_PORT
-# POSTGRES_HOST_PORT
-# POSTGRES_VERSION
-# MYSQL_HOST_PORT
-# MYSQL_VERSION
-# AIRFLOW_SOURCES
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_PROD_IMAGE_KUBERNETES
-# AIRFLOW_PROD_BASE_TAG
-# SQLITE_URL
-#
-# Arguments:
-#
-# file to prepare
-# command to run
-# compose_file to use
-# airflow_image to use
-#
-# Outputs:
-# Creates the convenience command file that can be run to use the docker command.
-#
-#######################################################################################################
+# All the environment variables and docker-compose configuration to run the command. This is because
+# depending on configuration of Breeze we might have different compose files used and different variables
+# set. Those are a convenience scripts that you might use to debug command execution although
+# In most cases they are used internally by Breeze
function prepare_command_file() {
- local file="${1}"
- local command="${2}"
- local compose_file="${3}"
- local airflow_image="${4}"
- cat <<EOF >"${file}"
+ local FILE="${1}"
+ local CMD="${2}"
+ local COMPOSE_FILE="${3}"
+ local AIRFLOW_IMAGE="${4}"
+ cat <<EOF > "${FILE}"
#!/usr/bin/env bash
if [[ \${VERBOSE} == "true" ]]; then
echo
echo "Executing script:"
echo
- echo "${file} \${@}"
+ echo "\${BASH_SOURCE[0]} \${@}"
echo
set -x
fi
cd "\$( dirname "\${BASH_SOURCE[0]}" )" || exit
export DOCKERHUB_USER=${DOCKERHUB_USER}
export DOCKERHUB_REPO=${DOCKERHUB_REPO}
-export HOST_USER_ID=${HOST_USER_ID}
-export HOST_GROUP_ID=${HOST_GROUP_ID}
+HOST_USER_ID=\$(id -ur)
+export HOST_USER_ID
+HOST_GROUP_ID=\$(id -gr)
+export HOST_GROUP_ID
export HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}"
-export COMPOSE_FILE="${compose_file}"
+export COMPOSE_FILE="${COMPOSE_FILE}"
export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}"
export BACKEND="${BACKEND}"
export AIRFLOW_VERSION="${AIRFLOW_VERSION}"
@@ -561,103 +474,92 @@ export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}"
export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}"
export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}"
export AIRFLOW_PROD_IMAGE_KUBERNETES="${AIRFLOW_PROD_IMAGE_KUBERNETES}"
-export AIRFLOW_PROD_BASE_TAG="${AIRFLOW_PROD_BASE_TAG}"
-export AIRFLOW_IMAGE="${airflow_image}"
+export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+export AIRFLOW_IMAGE="${AIRFLOW_IMAGE}"
export SQLITE_URL="${SQLITE_URL}"
-docker-compose --log-level INFO ${command}
+docker-compose --log-level INFO ${CMD}
EOF
- chmod u+x "${file}"
+ chmod u+x "${FILE}"
}
-#######################################################################################################
-#
-# Prepare all command files that we are using. Depending on the command to execute we use two
+
+# Prepare all command files that we are using. Depending on the command to execute we use different
# convenience scripts:
-#
+# cmd_run_ci - to run CI image command (for example entering Breeze, or running a script in CI image)
+# cmd_run_prod - to run PROD image command (for example entering Prod image or running a script there)
+# test_run_ci - to run test target in CI image
# dc_ci - to run docker compose command for CI image
# dc_prod - to run docker compose command for PROD image
#
-# Global constants set:
-#
-# PYTHON_BASE_IMAGE_VERSION
-# PYTHON_BASE_IMAGE
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_BASE_TAG
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_PROD_IMAGE_KUBERNETES
-# BUILT_CI_IMAGE_FLAG_FILE
-#
-#######################################################################################################
function prepare_command_files() {
- local main_ci_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
- local main_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
- local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
- local local_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local.yml
- local files_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/files.yml
- local local_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local-prod.yml
- local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
- local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
-
- local compose_ci_file=${main_ci_docker_compose_file}:${backend_docker_compose_file}
- local compose_prod_file=${main_prod_docker_compose_file}:${backend_docker_compose_file}
+ MAIN_CI_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/base.yml
+ MAIN_PROD_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/base.yml
+ BACKEND_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
+ LOCAL_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/local.yml
+ FILES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/files.yml
+ LOCAL_PROD_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/local-prod.yml
+ REMOVE_SOURCES_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
+ FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
+
+ COMPOSE_CI_FILE=${MAIN_CI_DOCKER_COMPOSE_FILE}:${BACKEND_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${MAIN_PROD_DOCKER_COMPOSE_FILE}:${BACKEND_DOCKER_COMPOSE_FILE}
if [[ "${MOUNT_LOCAL_SOURCES}" != "false" ]]; then
- compose_ci_file=${compose_ci_file}:${local_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${local_prod_docker_compose_file}
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${LOCAL_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${LOCAL_PROD_DOCKER_COMPOSE_FILE}
fi
if [[ "${MOUNT_FILES}" != "false" ]]; then
- compose_ci_file=${compose_ci_file}:${files_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${files_docker_compose_file}
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${FILES_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${FILES_DOCKER_COMPOSE_FILE}
fi
if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
- compose_ci_file=${compose_ci_file}:${forward_credentials_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${forward_credentials_docker_compose_file}
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE}
+ COMPOSE_PROD_FILE=${COMPOSE_PROD_FILE}:${FORWARD_CREDENTIALS_DOCKER_COMPOSE_FILE}
fi
- if [[ -n ${INSTALL_AIRFLOW_VERSION=} ]]; then
- compose_ci_file=${compose_ci_file}:${remove_sources_docker_compose_file}
+ if [[ ${INSTALL_AIRFLOW_VERSION} != "" ]]; then
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${REMOVE_SOURCES_DOCKER_COMPOSE_FILE}
fi
set +u
- local unique_integrations
# shellcheck disable=SC2207
- unique_integrations=($(echo "${INTEGRATIONS[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
+ UNIQUE_INTEGRATIONS=($(echo "${INTEGRATIONS[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))
- local integration
- for integration in "${unique_integrations[@]}"; do
- compose_ci_file=${compose_ci_file}:${SCRIPTS_CI_DIR}/docker-compose/integration-${integration}.yml
+ for _INT in "${UNIQUE_INTEGRATIONS[@]}"
+ do
+ COMPOSE_CI_FILE=${COMPOSE_CI_FILE}:${SCRIPTS_CI_DIR}/docker-compose/integration-${_INT}.yml
done
set -u
- export DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI="dc_ci"
- readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI
+ export COMPOSE_CI_FILE
+ export COMPOSE_PROD_FILE
- export DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD="dc_prod"
- readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD
+ get_base_image_version
+ # Base python image for the build
+ export PYTHON_BASE_IMAGE=python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster
+ export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
+ export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
+
+ LAST_DC_CI_FILE="dc_ci"
+ LAST_DC_PROD_FILE="dc_prod"
# Prepare script for "run docker compose CI command"
- prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" \
- "\"\${@}\"" "${compose_ci_file}" "${AIRFLOW_CI_IMAGE}"
+ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" \
+ "\"\${@}\"" "${COMPOSE_CI_FILE}" "${AIRFLOW_CI_IMAGE}"
# Prepare script for "run docker compose PROD command"
- prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}" \
- "\"\${@}\"" "${compose_prod_file}" "${AIRFLOW_PROD_IMAGE}"
+ prepare_command_file "${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}" \
+ "\"\${@}\"" "${COMPOSE_PROD_FILE}" "${AIRFLOW_PROD_IMAGE}"
}
-#######################################################################################################
-#
# Prints detailed help for all commands and flags. Used to generate documentation added to BREEZE.rst
# automatically.
-#
-# Used globals:
-# ALL_BREEZE_COMMANDS
-#
-# Outputs:
-# Prints detailed help for all commands to stdout.
-#
-#######################################################################################################
function do_help_all() {
echo
print_line
@@ -669,695 +571,548 @@ function do_help_all() {
echo
print_line
echo
- local subcommand
- for subcommand in ${ALL_BREEZE_COMMANDS}; do
- detailed_usage "${subcommand}"
- print_line
- echo
+ for SUBCOMMAND in ${ALL_BREEZE_COMMANDS}
+ do
+ detailed_usage "${SUBCOMMAND}"
+ print_line
+ echo
done
echo
flags
}
-#######################################################################################################
-#
# Parses all arguments that can be passed to Breeze command - that includes command to run and flags.
-#
-# Updated global constants:
-# By the end of this function, all the constants from `make_constants_read_only` function are set
-# and they are set as read-only.
-#
-#######################################################################################################
function parse_arguments() {
set -u
- local params
- if ! params=$(getopt \
+ if ! PARAMS=$(getopt \
-o "${_BREEZE_GETOPT_SHORT_OPTIONS:=}" \
-l "${_BREEZE_GETOPT_LONG_OPTIONS:=}" \
- --name "$CMDNAME" -- "$@"); then
+ --name "$CMDNAME" -- "$@")
+ then
flags
exit 1
fi
- eval set -- "${params}"
- unset params
+ eval set -- "${PARAMS}"
+ unset PARAMS
# Parse Flags.
# Please update short and long options in the breeze-complete script
# This way autocomplete will work out-of-the-box
- while true; do
- case "${1}" in
- -h | --help)
- run_help="true"
- shift
- ;;
- -p | --python)
- export PYTHON_MAJOR_MINOR_VERSION="${2}"
- echo "Python version: ${PYTHON_MAJOR_MINOR_VERSION}"
- echo
- shift 2
- ;;
- -b | --backend)
- export BACKEND="${2}"
- echo "Backend: ${BACKEND}"
- echo
- shift 2
- ;;
- -i | --integration)
- local integration=${2}
- check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
- echo "Integration: ${integration}"
- if [[ ${integration} == "all" ]]; then
- for integration in ${_BREEZE_ALLOWED_INTEGRATIONS}; do
- if [[ ${integration} != "all" ]]; then
- echo "${integration}"
- INTEGRATIONS+=("${integration}")
- fi
- done
- else
- INTEGRATIONS+=("${INTEGRATION}")
- fi
- echo
- shift 2
- ;;
- -K | --kubernetes-mode)
- export KUBERNETES_MODE="${2}"
- echo "Kubernetes mode: ${KUBERNETES_MODE}"
- echo
- shift 2
- ;;
- -V | --kubernetes-version)
- export KUBERNETES_VERSION="${2}"
- echo "Kubernetes version: ${KUBERNETES_VERSION}"
- echo
- shift 2
- ;;
+ while true
+ do
+ case "${1}" in
+ -h|--help)
+ RUN_HELP="true"
+ shift ;;
+ -p|--python)
+ export PYTHON_MAJOR_MINOR_VERSION="${2}";
+ echo "Python version: ${PYTHON_MAJOR_MINOR_VERSION}"
+ echo
+ shift 2 ;;
+ -b|--backend)
+ export BACKEND="${2}";
+ echo "Backend: ${BACKEND}"
+ echo
+ shift 2 ;;
+ -i|--integration)
+ INTEGRATION=${2}
+ check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
+ echo "Integration: ${INTEGRATION}"
+ if [[ ${INTEGRATION} == "all" ]]; then
+ for _INT in ${_BREEZE_ALLOWED_INTEGRATIONS}
+ do
+ if [[ ${_INT} != "all" ]]; then
+ echo "${_INT}"
+ INTEGRATIONS+=("${_INT}")
+ fi
+ done
+ else
+ INTEGRATIONS+=("${INTEGRATION}");
+ fi
+ echo
+ shift 2 ;;
+ -K|--kubernetes-mode)
+ export KUBERNETES_MODE="${2}";
+ echo "Kubernetes mode: ${KUBERNETES_MODE}"
+ echo
+ shift 2 ;;
+ -V|--kubernetes-version)
+ export KUBERNETES_VERSION="${2}";
+ echo "Kubernetes version: ${KUBERNETES_VERSION}"
+ echo
+ shift 2 ;;
--kind-version)
- export KIND_VERSION="${2}"
- echo "Kind version: ${KIND_VERSION}"
- echo
- shift 2
- ;;
+ export KIND_VERSION="${2}";
+ echo "Kind version: ${KIND_VERSION}"
+ echo
+ shift 2 ;;
--helm-version)
- export HELM_VERSION="${2}"
- echo "Helm version: ${HELM_VERSION}"
- echo
- shift 2
- ;;
+ export HELM_VERSION="${2}";
+ echo "Helm version: ${HELM_VERSION}"
+ echo
+ shift 2 ;;
--postgres-version)
- export POSTGRES_VERSION="${2}"
- echo "Postgres version: ${POSTGRES_VERSION}"
- echo
- shift 2
- ;;
+ export POSTGRES_VERSION="${2}";
+ echo "Postgres version: ${POSTGRES_VERSION}"
+ echo
+ shift 2 ;;
--mysql-version)
- export MYSQL_VERSION="${2}"
- echo "MySQL version: ${MYSQL_VERSION}"
- echo
- shift 2
- ;;
- -l | --skip-mounting-local-sources)
- MOUNT_LOCAL_SOURCES="false"
- echo "Mount local sources: ${MOUNT_LOCAL_SOURCES}"
- echo
- shift
- ;;
- -a | --install-airflow-version)
- INSTALL_AIRFLOW_VERSION="${2}"
- # Reference is mutually exclusive with version
- INSTALL_AIRFLOW_REFERENCE=""
- echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}"
- echo
- shift 2
- ;;
- -t | --install-airflow-reference)
- INSTALL_AIRFLOW_REFERENCE="${2}"
- # Reference is mutually exclusive with version
- INSTALL_AIRFLOW_VERSION=""
- echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}"
- echo
- shift 2
- ;;
- -d | --db-reset)
- echo "Resetting the DB!"
- echo
- export DB_RESET="true"
- shift 1
- ;;
- -v | --verbose)
- export VERBOSE="true"
- echo "Verbose output"
- echo
- shift
- ;;
- -y | --assume-yes)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- echo "Assuming 'yes' answer to all questions."
- echo
- shift
- ;;
- -n | --assume-no)
- export FORCE_ANSWER_TO_QUESTIONS="no"
- echo "Assuming 'no' answer to all questions."
- echo
- shift
- ;;
- -q | --assume-quit)
- export FORCE_ANSWER_TO_QUESTIONS="quit"
- echo "Assuming 'quit' answer to all questions."
- echo
- shift
- ;;
- -F | --force-build-images)
- echo "Force build images"
- echo
- export FORCE_BUILD_IMAGES="true"
- # if you want to force build an image - assume you want to build it :)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- shift
- ;;
- -C | --force-clean-images)
- echo "Clean build of images without cache"
- echo
- export DOCKER_CACHE="no-cache"
- # if not set here, docker cached is determined later, depending on type of image to be build
- readonly DOCKER_CACHE
- export FORCE_BUILD_IMAGES="true"
- shift
- ;;
- -r | --skip-rebuild-check)
- echo "Skips checking image for rebuilds"
- echo
- export CHECK_IMAGE_FOR_REBUILD="false"
- export SKIP_BUILDING_PROD_IMAGE="true"
- shift
- ;;
- -L | --build-cache-local)
- echo "Use local cache to build images"
- echo
- export DOCKER_CACHE="local"
- # if not set here, docker cached is determined later, depending on type of image to be build
- readonly DOCKER_CACHE
- shift
- ;;
- -U | --build-cache-pulled)
- echo "Use pulled cache to build images"
- echo
- export DOCKER_CACHE="pulled"
- # if not set here, docker cached is determined later, depending on type of image to be build
- readonly DOCKER_CACHE
- shift
- ;;
- -X | --build-cache-disabled)
- echo "Use disabled cache to build images"
- echo
- export DOCKER_CACHE="disabled"
- readonly DOCKER_CACHE
- # if not set here, docker cached is determined later, depending on type of image to be build
- shift
- ;;
- -P | --force-pull-images)
- echo "Force pulling images before build. Uses pulled images as cache."
- echo
- export FORCE_PULL_IMAGES="true"
- export FORCE_BUILD_IMAGES="true"
- # if you want to force build an image - assume you want to build it :)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- shift
- ;;
- -I | --production-image)
- export PRODUCTION_IMAGE="true"
- export SQLITE_URL=
- echo
- echo "*************** PRODUCTION IMAGE *************************"
- echo
- shift
- ;;
- -E | --extras)
- export AIRFLOW_EXTRAS="${2}"
- echo "Extras : ${AIRFLOW_EXTRAS}"
- shift 2
- ;;
+ export MYSQL_VERSION="${2}";
+ echo "MySQL version: ${MYSQL_VERSION}"
+ echo
+ shift 2 ;;
+ -l|--skip-mounting-local-sources)
+ MOUNT_LOCAL_SOURCES="false"
+ echo "Mount local sources: ${MOUNT_LOCAL_SOURCES}"
+ echo
+ shift ;;
+ -a|--install-airflow-version)
+ INSTALL_AIRFLOW_VERSION="${2}"
+ # Reference is exclusive with version
+ INSTALL_AIRFLOW_REFERENCE=""
+ echo "Installs version of Airflow: ${INSTALL_AIRFLOW_VERSION}"
+ echo
+ shift 2 ;;
+ -t|--install-airflow-reference)
+ INSTALL_AIRFLOW_REFERENCE="${2}"
+ # Reference is exclusive with version
+ INSTALL_AIRFLOW_VERSION=""
+ echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}"
+ echo
+ shift 2 ;;
+ -d|--db-reset)
+ echo "Resetting the DB!"
+ echo
+ export DB_RESET="true"
+ shift 1 ;;
+ -v|--verbose)
+ export VERBOSE="true"
+ echo "Verbose output"
+ echo
+ shift ;;
+ -y|--assume-yes)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ echo "Assuming 'yes' answer to all questions."
+ echo
+ shift ;;
+ -n|--assume-no)
+ export FORCE_ANSWER_TO_QUESTIONS="no"
+ echo "Assuming 'no' answer to all questions."
+ echo
+ shift ;;
+ -q|--assume-quit)
+ export FORCE_ANSWER_TO_QUESTIONS="quit"
+ echo "Assuming 'quit' answer to all questions."
+ echo
+ shift ;;
+ -F|--force-build-images)
+ echo "Force build images"
+ echo
+ export FORCE_BUILD_IMAGES="true"
+ # if you want to force build an image - assume you want to build it :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ shift ;;
+ -C|--force-clean-images)
+ echo "Clean build of images without cache"
+ echo
+ export DOCKER_CACHE="no-cache"
+ export FORCE_BUILD_IMAGES="true"
+ shift ;;
+ -r|--skip-rebuild-check)
+ echo "Skips checking image for rebuilds"
+ echo
+ export CHECK_IMAGE_FOR_REBUILD="false"
+ export SKIP_BUILDING_PROD_IMAGE="true"
+ shift ;;
+ -L|--build-cache-local)
+ echo "Use local cache to build images"
+ echo
+ export DOCKER_CACHE="local"
+ shift ;;
+ -U|--build-cache-pulled)
+ echo "Use pulled cache to build images"
+ echo
+ export DOCKER_CACHE="pulled"
+ shift ;;
+ -X|--build-cache-disabled)
+ echo "Use disabled cache to build images"
+ echo
+ export DOCKER_CACHE="disabled"
+ shift ;;
+ -P|--force-pull-images)
+ echo "Force pulling images before build. Uses pulled images as cache."
+ echo
+ export FORCE_PULL_IMAGES="true"
+ export FORCE_BUILD_IMAGES="true"
+ # if you want to force build an image - assume you want to build it :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ shift ;;
+ -I|--production-image)
+ export PRODUCTION_IMAGE="true"
+ export SQLITE_URL=
+ echo
+ echo "*************** PRODUCTION IMAGE *************************"
+ echo
+ shift ;;
+ -E|--extras)
+ export AIRFLOW_EXTRAS="${2}"
+ echo "Extras : ${AIRFLOW_EXTRAS}"
+ shift 2 ;;
--additional-extras)
- export ADDITIONAL_AIRFLOW_EXTRAS="${2}"
- echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}"
- shift 2
- ;;
+ export ADDITIONAL_AIRFLOW_EXTRAS="${2}"
+ echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}"
+ shift 2 ;;
--additional-python-deps)
- export ADDITIONAL_PYTHON_DEPS="${2}"
- echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}"
- shift 2
- ;;
+ export ADDITIONAL_PYTHON_DEPS="${2}"
+ echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}"
+ shift 2 ;;
--additional-dev-deps)
- export ADDITIONAL_DEV_DEPS="${2}"
- echo "Additional apt dev dependencies: ${ADDITIONAL_DEV_DEPS}"
- shift 2
- ;;
+ export ADDITIONAL_DEV_DEPS="${2}"
+ echo "Additional apt dev dependencies: ${ADDITIONAL_DEV_DEPS}"
+ shift 2 ;;
--additional-runtime-deps)
- export ADDITIONAL_RUNTIME_DEPS="${2}"
- echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}"
- shift 2
- ;;
- -D | --dockerhub-user)
- export DOCKERHUB_USER="${2}"
- echo "Dockerhub user ${DOCKERHUB_USER}"
- echo
- shift 2
- ;;
- -R | --dockerhub-repo)
- export DOCKERHUB_REPO="${2}"
- echo "Dockerhub repo ${DOCKERHUB_REPO}"
- echo
- shift 2
- ;;
- -f | --forward-credentials)
- echo "Forwarding credentials. Be careful as your credentials ar available in the container!"
- echo
- export FORWARD_CREDENTIALS="true"
- shift 1
- ;;
- -c | --github-registry)
- echo
- echo "Use github registry"
- echo
- export USE_GITHUB_REGISTRY="true"
- shift
- ;;
- -g | --github-repository)
- echo
- echo "GitHub repository: ${2}"
- echo
- echo "Using github registry."
- echo
- export GITHUB_REPOSITORY="${2}"
- export USE_GITHUB_REGISTRY="true"
- shift 2
- ;;
- -s | --github-image-id)
- echo
- echo "GitHub image id: ${2}}"
- echo
- echo "Force pulling the image, using github registry and skip mounting local sources."
- echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!."
- echo
- export FORCE_PULL_IMAGES="true"
- export USE_GITHUB_REGISTRY="true"
- export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}"
- export GITHUB_REGISTRY_PUSH_IMAGE_TAG="${2}"
- export CHECK_IMAGE_FOR_REBUILD="false"
- export SKIP_BUILDING_PROD_IMAGE="true"
- export MOUNT_LOCAL_SOURCES="false"
- export SKIP_CHECK_REMOTE_IMAGE="true"
- shift 2
- ;;
- -S | --version-suffix-for-pypi)
- if [[ -n ${VERSION_SUFFIX_FOR_SVN=} ]]; then
- echo
- echo "You can only set one version suffix - either for PyPI or for SVN"
- echo
- exit 1
- fi
- export VERSION_SUFFIX_FOR_PYPI="${2}"
- echo "Version suffix for PyPI ${VERSION_SUFFIX_FOR_PYPI}"
- echo
- shift 2
- ;;
- -N | --version-suffix-for-svn)
- if [[ -n ${VERSION_SUFFIX_FOR_PYPI=} ]]; then
- echo
- echo "You can only set one version suffix - either for PyPI or for SVN"
- echo
- exit 1
- fi
- export VERSION_SUFFIX_FOR_SVN="${2}"
- echo "Version suffix for SVN ${VERSION_SUFFIX_FOR_SVN}"
- echo
- shift 2
- ;;
+ export ADDITIONAL_RUNTIME_DEPS="${2}"
+ echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}"
+ shift 2 ;;
+ -D|--dockerhub-user)
+ export DOCKERHUB_USER="${2}"
+ echo "Dockerhub user ${DOCKERHUB_USER}"
+ echo
+ shift 2 ;;
+ -R|--dockerhub-repo)
+ export DOCKERHUB_REPO="${2}"
+ echo "Dockerhub repo ${DOCKERHUB_REPO}"
+ echo
+ shift 2 ;;
+ -f|--forward-credentials)
+ echo "Forwarding credentials. Be careful as your credentials ar available in the container!"
+ echo
+ export FORWARD_CREDENTIALS="true"
+ shift 1 ;;
+ -c|--github-registry)
+ echo
+ echo "Use github registry"
+ echo
+ export USE_GITHUB_REGISTRY="true"
+ shift ;;
+ -g|--github-repository)
+ echo
+ echo "GitHub repository: ${2}"
+ echo
+ echo "Using github registry."
+ echo
+ export GITHUB_REPOSITORY="${2}"
+ export USE_GITHUB_REGISTRY="true"
+ shift 2;;
+ -s|--github-image-id)
+ echo
+ echo "GitHub image id: ${2}}"
+ echo
+ echo "Force pulling the image, using github registry and skip mounting local sources."
+ echo "This is in order to get the exact same version as used in CI environment for SHA/RUN_ID!."
+ echo
+ export FORCE_PULL_IMAGES="true"
+ export USE_GITHUB_REGISTRY="true"
+ export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}"
+ export GITHUB_REGISTRY_PUSH_IMAGE_TAG="${2}"
+ export CHECK_IMAGE_FOR_REBUILD="false"
+ export SKIP_BUILDING_PROD_IMAGE="true"
+ export MOUNT_LOCAL_SOURCES="false"
+ export SKIP_CHECK_REMOTE_IMAGE="true"
+ shift 2;;
+ -S|--version-suffix-for-pypi)
+ if [[ ${VERSION_SUFFIX_FOR_SVN} != "" ]]; then
+ echo
+ echo "You can only set one version suffix - either for PyPI or for SVN"
+ echo
+ exit 1
+ fi
+ export VERSION_SUFFIX_FOR_PYPI="${2}"
+ echo "Version suffix for PyPI ${VERSION_SUFFIX_FOR_PYPI}"
+ echo
+ shift 2 ;;
+ -N|--version-suffix-for-svn)
+ if [[ ${VERSION_SUFFIX_FOR_PYPI} != "" ]]; then
+ echo
+ echo "You can only set one version suffix - either for PyPI or for SVN"
+ echo
+ exit 1
+ fi
+ export VERSION_SUFFIX_FOR_SVN="${2}"
+ echo "Version suffix for SVN ${VERSION_SUFFIX_FOR_SVN}"
+ echo
+ shift 2 ;;
--)
- shift
- break
- ;;
+ shift ;
+ break ;;
*)
- flags
- echo >&2
- echo >&2 "ERROR: Unknown flag ${1}"
- echo >&2
- exit 1
- ;;
- esac
+ flags
+ echo >&2
+ echo >&2 "ERROR: Unknown flag ${1}"
+ echo >&2
+ exit 1
+ ;;
+ esac
done
- local last_subcommand=""
# Parse commands
if [[ "$#" -ne 0 ]]; then
- case "${1}" in
+ case "${1}" in
shell)
- last_subcommand="${1}"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ shift ;;
exec)
- last_subcommand="${1}"
- command_to_run="run_exec"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="run_exec"
+ shift ;;
build-docs)
- last_subcommand="${1}"
- command_to_run="build_docs"
- shift 1
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="build_docs"
+ shift 1 ;;
build-image)
- last_subcommand="${1}"
- command_to_run="build_image"
- # if you want to build an image - assume you want to build it :)
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- # and assume you want to build it no matter if it is needed
- export FORCE_BUILD_IMAGES="true"
- echo "Build image"
- echo
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="build_image"
+ # if you want to build an image - assume you want to build it :)
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ # and assume you want to build it no matter if it is needed
+ export FORCE_BUILD_IMAGES="true"
+ echo "Build image"
+ echo
+ shift ;;
cleanup-image)
- last_subcommand="${1}"
- echo "Cleanup the image"
- echo
- command_to_run="cleanup_image"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ echo "Cleanup the image"
+ echo
+ COMMAND_TO_RUN="cleanup_image"
+ shift ;;
docker-compose)
- last_subcommand="${1}"
- if [[ $# -lt 2 ]]; then
- echo "You should specify docker compose command to run"
- shift
- run_help="true"
- else
- docker_compose_command="${2}"
- shift 2
- fi
- command_to_run="run_docker_compose"
- ;;
- generate-constraints)
- last_subcommand="${1}"
- command_to_run="perform_generate_constraints"
- export FORCE_ANSWER_TO_QUESTIONS="yes"
- export FORCE_BUILD_IMAGES="true"
- export UPGRADE_TO_LATEST_CONSTRAINTS="true"
+ LAST_SUBCOMMAND="${1}"
+ if [[ $# -lt 2 ]]; then
+ echo "You should specify docker compose command to run"
shift
- ;;
+ RUN_HELP="true"
+ else
+ DOCKER_COMPOSE_COMMAND="${2}"
+ shift 2
+ fi
+ COMMAND_TO_RUN="run_docker_compose"
+ ;;
+ generate-constraints)
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_generate_constraints"
+ export FORCE_ANSWER_TO_QUESTIONS="yes"
+ export FORCE_BUILD_IMAGES="true"
+ export UPGRADE_TO_LATEST_CONSTRAINTS="true"
+ shift ;;
prepare-backport-packages)
- last_subcommand="${1}"
- command_to_run="perform_prepare_backport_packages"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_prepare_backport_packages"
+ shift ;;
prepare-backport-readme)
- last_subcommand="${1}"
- command_to_run="perform_prepare_backport_readme"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_prepare_backport_readme"
+ shift ;;
push-image)
- last_subcommand="${1}"
- command_to_run="perform_push_image"
- export SKIP_CHECK_REMOTE_IMAGE="true"
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_push_image"
+ export SKIP_CHECK_REMOTE_IMAGE="true"
+ shift ;;
initialize-local-virtualenv)
- last_subcommand="${1}"
- echo "Initializing local virtualenv"
- echo
- command_to_run="perform_initialize_local_virtualenv"
- shift
- ;;
- kind-cluster)
- last_subcommand="${1}"
- # Switch to production image for all kind operations
- export PRODUCTION_IMAGE="true"
- command_to_run="manage_kind_cluster"
- export KIND_CLUSTER_OPERATION="${2:-}"
- if [[ -n ${KIND_CLUSTER_OPERATION=} ]]; then
- shift 2
- else
- shift
- fi
- ;;
- setup-autocomplete)
- last_subcommand="${1}"
- echo "Setting up autocomplete"
- echo
- command_to_run="perform_setup_autocomplete"
- shift
- ;;
- static-check)
- last_subcommand="${1}"
- command_to_run="perform_static_checks"
- if [[ "$#" -lt 2 ]]; then
- if [[ ${run_help} != "true" ]]; then
- echo "You should specify static check that you would like to run or 'all' to run all checks."
- echo
- echo "One of :"
- echo
- echo "${_BREEZE_ALLOWED_STATIC_CHECKS:=}"
- echo
- echo "For example:"
- echo
- echo "${CMDNAME} static-check mypy"
- echo
- exit 1
- else
- shift
- fi
+ LAST_SUBCOMMAND="${1}"
+ echo "Initializing local virtualenv"
+ echo
+ COMMAND_TO_RUN="perform_initialize_local_virtualenv"
+ shift ;;
+ kind-cluster)
+ LAST_SUBCOMMAND="${1}"
+ # Force local cache strategy for all kind-cluster operations
+ # this helps to iterate with production images
+ DOCKER_CACHE="local"
+ COMMAND_TO_RUN="manage_kind_cluster"
+ export KIND_CLUSTER_OPERATION="${2:-}"
+ if [[ ${KIND_CLUSTER_OPERATION} != "" ]]; then
+ shift 2
+ else
+ shift
+ fi
+ ;;
+ setup-autocomplete)
+ LAST_SUBCOMMAND="${1}"
+ echo "Setting up autocomplete"
+ echo
+ COMMAND_TO_RUN="perform_setup_autocomplete"
+ shift ;;
+ static-check )
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="perform_static_checks"
+ if [[ "$#" -lt 2 ]]; then
+ if [[ ${RUN_HELP} != "true" ]]; then
+ echo "You should specify static check that you would like to run or 'all' to run all checks."
+ echo
+ echo "One of :"
+ echo
+ echo "${_BREEZE_ALLOWED_STATIC_CHECKS:=}"
+ echo
+ echo "For example:"
+ echo
+ echo "${CMDNAME} static-check mypy"
+ echo
+ exit 1
else
- export PYTHON_MAJOR_MINOR_VERSION=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}
- static_check="${2:-}"
- EXTRA_STATIC_CHECK_OPTIONS+=("--show-diff-on-failure")
- shift 2
+ shift
fi
- ;;
+ else
+ export PYTHON_MAJOR_MINOR_VERSION=${STATIC_CHECK_PYTHON_MAJOR_MINOR_VERSION}
+ export STATIC_CHECK="${2:-}"
+ export STATIC_CHECK_ALL_FILES="false"
+ EXTRA_STATIC_CHECK_OPTIONS+=("--show-diff-on-failure")
+ shift 2
+ fi
+ ;;
stop)
- last_subcommand="${1}"
- command_to_run="run_docker_compose"
- docker_compose_command="down"
- EXTRA_DC_OPTIONS+=("--remove-orphans")
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="run_docker_compose"
+ DOCKER_COMPOSE_COMMAND="down"
+ EXTRA_DC_OPTIONS+=("--remove-orphans")
+ shift ;;
restart)
- last_subcommand="${1}"
- command_to_run="run_docker_compose"
- docker_compose_command="down"
- EXTRA_DC_OPTIONS+=("--remove-orphans")
- second_command_to_run="enter_breeze"
- echo "Restarts the environment. Includes emptying the databases."
- shift
- ;;
+ LAST_SUBCOMMAND="${1}"
+ COMMAND_TO_RUN="run_docker_compose"
+ DOCKER_COMPOSE_COMMAND="down"
+ EXTRA_DC_OPTIONS+=("--remove-orphans")
+ SECOND_COMMAND_TO_RUN="enter_breeze"
+ echo "Restarts the environment. Includes emptying the databases."
+ shift ;;
tests)
- last_subcommand="${1}"
- if [[ $# -lt 2 ]]; then
- run_help="true"
- else
- shift
- fi
- command_to_run="run_tests"
- ;;
- toggle-suppress-cheatsheet)
- last_subcommand="${1}"
- if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
- rm -f "${SUPPRESS_CHEATSHEET_FILE}"
- else
- touch "${SUPPRESS_CHEATSHEET_FILE}"
- fi
- echo "Toggle suppress cheatsheet"
- echo
+ LAST_SUBCOMMAND="${1}"
+ if [[ $# -lt 2 ]]; then
+ RUN_HELP="true"
+ else
shift
- command_to_run="toggle_suppress_cheatsheet"
- ;;
+ fi
+ COMMAND_TO_RUN="run_tests" ;;
+ toggle-suppress-cheatsheet)
+ LAST_SUBCOMMAND="${1}"
+ if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
+ rm -f "${SUPPRESS_CHEATSHEET_FILE}"
+ else
+ touch "${SUPPRESS_CHEATSHEET_FILE}"
+ fi
+ echo "Toggle suppress cheatsheet"
+ echo
+ shift ;;
toggle-suppress-asciiart)
- last_subcommand="${1}"
- if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
- rm -f "${SUPPRESS_ASCIIART_FILE}"
- else
- touch "${SUPPRESS_ASCIIART_FILE}"
- fi
- echo "Toggle suppress asciiart"
- echo
- shift
- command_to_run="toggle_suppress_asciiart"
- ;;
+ LAST_SUBCOMMAND="${1}"
+ if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
+ rm -f "${SUPPRESS_ASCIIART_FILE}"
+ else
+ touch "${SUPPRESS_ASCIIART_FILE}"
+ fi
+ echo "Toggle suppress asciiart"
+ echo
+ shift ;;
flags)
- flags
- exit 0
- ;;
+ flags
+ exit 0 ;;
help)
- usage
- exit 0
- ;;
+ usage
+ exit 0 ;;
help-all)
- do_help_all
- exit 0
- ;;
+ do_help_all
+ exit 0 ;;
*)
- usage
- echo >&2
- echo >&2 "ERROR: Unknown command ${1}"
- echo >&2
- exit 1
- ;;
- esac
+ usage
+ echo >&2
+ echo >&2 "ERROR: Unknown command ${1}"
+ echo >&2
+ exit 1
+ ;;
+ esac
else
- :
- # By default, start interactive terminal
+ :
+ # By default, start interactive terminal
fi
- if [[ ${run_help} == "true" ]]; then
- if [[ ${last_subcommand} == "" ]]; then
- usage
- flag_footer
- else
- detailed_usage "${last_subcommand}"
- fi
- exit 0
+ if [[ ${RUN_HELP} == "true" ]]; then
+ if [[ ${LAST_SUBCOMMAND} == "" ]]; then
+ usage
+ flag_footer
+ else
+ detailed_usage "${LAST_SUBCOMMAND}"
+ fi
+ exit 0
fi
-
- # EXTRA_DC_OPTIONS is only used by Breeze. It's value is set here as well.
- readonly EXTRA_DC_OPTIONS
-
- # Also Remaining args are set here and set as read-only - no more changes to it.
REMAINING_ARGS+=("$@")
- export REMAINING_ARGS
- readonly REMAINING_ARGS
}
-#######################################################################################################
-#
# Prepares nicely formatted versions of list of allowed and default values defined in Breeze.
# It is used in help command to print the lists in a readable format and fold the lists
-# so that they fit the screen indented_screen_width.
-#
-# Updated global constants:
-# FORMATTED_* constant variables that can be used in Breeze Help output
-#
-#######################################################################################################
+# so that they fit the screen width.
function prepare_formatted_versions() {
- local indent=15
- local list_prefix
- list_prefix=$(printf "%-${indent}s" " ")
- local indented_screen_width=$((SCREEN_WIDTH - indent))
-
- FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS=$(echo "${_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS
-
- FORMATTED_BACKENDS=$(echo "${_BREEZE_ALLOWED_BACKENDS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_BACKENDS
-
- FORMATTED_STATIC_CHECKS=$(echo "${_BREEZE_ALLOWED_STATIC_CHECKS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_STATIC_CHECKS
-
- FORMATTED_INTEGRATIONS=$(echo "${_BREEZE_ALLOWED_INTEGRATIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_INTEGRATIONS
-
- FORMATTED_KUBERNETES_MODES=$(echo "${_BREEZE_ALLOWED_KUBERNETES_MODES=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KUBERNETES_MODES
-
- FORMATTED_KUBERNETES_VERSIONS=$(echo "${_BREEZE_ALLOWED_KUBERNETES_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KUBERNETES_VERSIONS
-
- FORMATTED_KIND_VERSIONS=$(echo "${_BREEZE_ALLOWED_KIND_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KIND_VERSIONS
-
- FORMATTED_HELM_VERSIONS=$(echo "${_BREEZE_ALLOWED_HELM_VERSIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_HELM_VERSIONS
-
- FORMATTED_KIND_OPERATIONS=$(echo "${_BREEZE_ALLOWED_KIND_OPERATIONS=""}" | tr '\n' ' ' |
- fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_KIND_OPERATIONS
-
- FORMATTED_INSTALL_AIRFLOW_VERSIONS=$(echo "${_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_INSTALL_AIRFLOW_VERSIONS
-
- FORMATTED_POSTGRES_VERSIONS=$(echo "${_BREEZE_ALLOWED_POSTGRES_VERSIONS=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_POSTGRES_VERSIONS
-
- FORMATTED_MYSQL_VERSIONS=$(echo "${_BREEZE_ALLOWED_MYSQL_VERSIONS=""}" |
- tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
- readonly FORMATTED_MYSQL_VERSIONS
-
- FORMATTED_DEFAULT_CI_EXTRAS=$(echo "${DEFAULT_CI_EXTRAS=}" |
- tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
- readonly FORMATTED_DEFAULT_CI_EXTRAS
-
- FORMATTED_DEFAULT_PROD_EXTRAS=$(echo "${DEFAULT_PROD_EXTRAS=}" |
- tr ',' ' ' | fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
- readonly FORMATTED_DEFAULT_PROD_EXTRAS
+ INDENT=15
+ LIST_PREFIX=$(printf "%-${INDENT}s" " ")
+ WIDTH=$((SCREEN_WIDTH - INDENT))
+ FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS=$(echo "${_BREEZE_ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_BACKENDS=$(echo "${_BREEZE_ALLOWED_BACKENDS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_STATIC_CHECKS=$(echo "${_BREEZE_ALLOWED_STATIC_CHECKS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_INTEGRATIONS=$(echo "${_BREEZE_ALLOWED_INTEGRATIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KUBERNETES_MODES=$(echo "${_BREEZE_ALLOWED_KUBERNETES_MODES=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KUBERNETES_VERSIONS=$(echo "${_BREEZE_ALLOWED_KUBERNETES_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KIND_VERSIONS=$(echo "${_BREEZE_ALLOWED_KIND_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_HELM_VERSIONS=$(echo "${_BREEZE_ALLOWED_HELM_VERSIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_KIND_OPERATIONS=$(echo "${_BREEZE_ALLOWED_KIND_OPERATIONS=""}" | tr '\n' ' ' | \
+ fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_INSTALL_AIRFLOW_VERSIONS=$(echo "${_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=""}" | \
+ tr '\n' ' ' | fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_POSTGRES_VERSIONS=$(echo "${_BREEZE_ALLOWED_POSTGRES_VERSIONS=""}" | \
+ tr '\n' ' ' | fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_MYSQL_VERSIONS=$(echo "${_BREEZE_ALLOWED_MYSQL_VERSIONS=""}" | \
+ tr '\n' ' ' | fold -w "${WIDTH}" -s | sed "s/^/${LIST_PREFIX}/")
+ FORMATTED_DEFAULT_CI_EXTRAS=$(echo "${DEFAULT_CI_EXTRAS=}" | \
+ tr ',' ' ' | fold -w "${WIDTH}" -s | sed "s/ /,/g; s/^/${LIST_PREFIX}/")
+ FORMATTED_DEFAULT_PROD_EXTRAS=$(echo "${DEFAULT_PROD_EXTRAS=}" | \
+ tr ',' ' ' | fold -w "${WIDTH}" -s | sed "s/ /,/g; s/^/${LIST_PREFIX}/")
}
-#######################################################################################################
-#
# Prepares usage information for all the commands in Breeze.
# Those usage commands are stored in appropriate environment variables.
-#
-# Created constants:
-# USAGE_*
-# DETAILED_USAGE_*
-#
-#######################################################################################################
-
-# shellcheck disable=SC2034,SC2090,SC2089,SC2155
-
function prepare_usage() {
# Note that MacOS uses Bash 3.* and we cannot use associative arrays
export USAGE_SHELL="[Default] Enters interactive shell in the container"
- readonly USAGE_SHELL
export USAGE_EXEC="Execs into running breeze container in new terminal"
- readonly USAGE_EXEC
export USAGE_BUILD_DOCS="Builds documentation in the container"
- readonly USAGE_BUILD_DOCS
export USAGE_BUILD_IMAGE="Builds CI or Production docker image"
- readonly USAGE_BUILD_DOCS
export USAGE_CLEANUP_IMAGE="Cleans up the container image created"
- readonly USAGE_BUILD_DOCS
export USAGE_DOCKER_COMPOSE="Executes specified docker-compose command"
- readonly USAGE_DOCKER_COMPOSE
export USAGE_FLAGS="Shows all breeze's flags"
- readonly USAGE_FLAGS
export USAGE_GENERATE_CONSTRAINTS="Generates pinned constraint files"
- readonly USAGE_GENERATE_CONSTRAINTS
export USAGE_INITIALIZE_LOCAL_VIRTUALENV="Initializes local virtualenv"
- readonly USAGE_INITIALIZE_LOCAL_VIRTUALENV
export USAGE_PREPARE_BACKPORT_README="Prepares backport packages readme files"
- readonly USAGE_PREPARE_BACKPORT_README
export USAGE_PREPARE_BACKPORT_PACKAGES="Prepares backport packages"
- readonly USAGE_PREPARE_BACKPORT_PACKAGES
export USAGE_PUSH_IMAGE="Pushes images to registry"
- readonly USAGE_PUSH_IMAGE
export USAGE_KIND_CLUSTER="Manages KinD cluster on the host"
- readonly USAGE_KIND_CLUSTER
export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze"
- readonly USAGE_SETUP_AUTOCOMPLETE
export USAGE_STOP="Stops the docker-compose environment"
- readonly USAGE_STOP
export USAGE_RESTART="Stops the docker-compose environment including DB cleanup"
- readonly USAGE_RESTART
export USAGE_STATIC_CHECK="Performs selected static check for changed files"
- readonly USAGE_STATIC_CHECK
+ export USAGE_STATIC_CHECK_ALL_FILES="Performs selected static check for all files"
export USAGE_TOGGLE_SUPPRESS_CHEATSHEET="Toggles on/off cheatsheet"
- readonly USAGE_TOGGLE_SUPPRESS_CHEATSHEET
export USAGE_TOGGLE_SUPPRESS_ASCIIART="Toggles on/off asciiart"
- readonly USAGE_TOGGLE_SUPPRESS_ASCIIART
export USAGE_TESTS="Runs selected tests in the container"
- readonly USAGE_TESTS
export USAGE_HELP="Shows this help message"
- readonly USAGE_HELP
export USAGE_HELP_ALL="Shows detailed help for all commands and flags"
- readonly USAGE_HELP_ALL
- export DETAILED_USAGE_SHELL="
+
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_SHELL="
${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
This is default subcommand if no subcommand is used.
@@ -1399,7 +1154,8 @@ ${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
Flags:
$(flag_footer)
"
- readonly DETAILED_USAGE_SHELL
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_SHELL
export DETAILED_USAGE_EXEC="
${CMDNAME} exec [-- <EXTRA_ARGS>]
@@ -1419,8 +1175,8 @@ ${CMDNAME} build-docs [-- <EXTRA_ARGS>]
The possible extra args are: --docs-only, --spellcheck-only, --help
"
- readonly DETAILED_USAGE_BUILD_DOCS
- export DETAILED_USAGE_BUILD_IMAGE="
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_BUILD_IMAGE="
${CMDNAME} build-image [FLAGS]
Builds docker image (CI or production) without entering the container. You can pass
@@ -1444,8 +1200,10 @@ $(flag_build_docker_images)
$(flag_pull_push_docker_images)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_BUILD_IMAGE
- export DETAILED_USAGE_CLEANUP_IMAGE="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_BUILD_IMAGE
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_CLEANUP_IMAGE="
${CMDNAME} cleanup-image [FLAGS]
Removes the breeze-related images created in your local docker image cache. This will
@@ -1457,8 +1215,10 @@ $(flag_airflow_variants)
$(flag_production_image)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_CLEANUP_IMAGE
- export DETAILED_USAGE_DOCKER_COMPOSE="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_CLEANUP_IMAGE
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_DOCKER_COMPOSE="
${CMDNAME} docker-compose [FLAGS] COMMAND [-- <EXTRA_ARGS>]
Run docker-compose command instead of entering the environment. Use 'help' as command
@@ -1472,12 +1232,13 @@ $(flag_airflow_variants)
$(flag_backend_variants)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_DOCKER_COMPOSE
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_DOCKER_COMPOSE
export DETAILED_USAGE_FLAGS="
Explains in detail all the flags that can be used with breeze.
"
- readonly DETAILED_USAGE_FLAGS
- export DETAILED_USAGE_PREPARE_BACKPORT_README="
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PREPARE_BACKPORT_README="
${CMDNAME} prepare-backport-packages [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
Prepares README.md files for backport packages. You can provide (after --) optional version
@@ -1507,8 +1268,9 @@ ${CMDNAME} prepare-backport-packages [FLAGS] [YYYY.MM.DD] [PACKAGE_ID ...]
Flags:
$(flag_verbosity)
"
- readonly DETAILED_USAGE_PREPARE_BACKPORT_README
- export DETAILED_USAGE_GENERATE_CONSTRAINTS="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PREPARE_BACKPORT_README
+ DETAILED_USAGE_GENERATE_CONSTRAINTS="
${CMDNAME} generate-constraints [FLAGS]
Generates pinned constraint files from setup.py. Those files are generated in files folder
@@ -1523,8 +1285,9 @@ Flags:
$(flag_airflow_variants)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_GENERATE_CONSTRAINTS
- export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_GENERATE_CONSTRAINTS
+ DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV="
${CMDNAME} initialize-local-virtualenv [FLAGS]
Initializes locally created virtualenv installing all dependencies of Airflow
@@ -1536,8 +1299,10 @@ ${CMDNAME} initialize-local-virtualenv [FLAGS]
Flags:
$(flag_airflow_variants)
"
- readonly DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV
- export DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_INITIALIZE_LOCAL_VIRTUALENV
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES="
${CMDNAME} prepare-backport-packages [FLAGS] [PACKAGE_ID ...]
Prepares backport packages. You can provide (after --) optional list of packages to prepare.
@@ -1565,8 +1330,10 @@ Flags:
$(flag_version_suffix)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES
- export DETAILED_USAGE_PUSH_IMAGE="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PREPARE_BACKPORT_PACKAGES
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_PUSH_IMAGE="
${CMDNAME} push_image [FLAGS]
Pushes images to docker registry. You can push the images to DockerHub registry (default)
@@ -1598,8 +1365,9 @@ Flags:
$(flag_pull_push_docker_images)
$(flag_verbosity)
"
- readonly DETAILED_USAGE_PUSH_IMAGE
- export DETAILED_USAGE_KIND_CLUSTER="
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_PUSH_IMAGE
+ DETAILED_USAGE_KIND_CLUSTER="
${CMDNAME} kind-cluster [FLAGS] OPERATION
Manages host-side Kind Kubernetes cluster that is used to run Kubernetes integration tests.
@@ -1615,7 +1383,7 @@ Flags:
$(flag_airflow_variants)
$(flag_build_docker_images)
"
- readonly DETAILED_USAGE_KIND_CLUSTER
+ export DETAILED_USAGE_KIND_CLUSTER
export DETAILED_USAGE_SETUP_AUTOCOMPLETE="
${CMDNAME} setup-autocomplete
@@ -1623,7 +1391,6 @@ ${CMDNAME} setup-autocomplete
shell and when typing breeze command <TAB> will provide autocomplete for
parameters and values.
"
- readonly DETAILED_USAGE_SETUP_AUTOCOMPLETE
export DETAILED_USAGE_STOP="
${CMDNAME} stop
@@ -1631,8 +1398,7 @@ ${CMDNAME} stop
containers will continue running so that startup time is shorter. But they take quite a lot of
memory and CPU. This command stops all running containers from the environment.
"
- readonly DETAILED_USAGE_STOP
- export DETAILED_USAGE_RESTART="
+ DETAILED_USAGE_RESTART="
${CMDNAME} restart [FLAGS]
Restarts running docker compose environment. When you restart the environment, the docker
@@ -1642,9 +1408,9 @@ ${CMDNAME} restart [FLAGS]
Flags:
$(flag_footer)
"
- readonly DETAILED_USAGE_RESTART
+ export DETAILED_USAGE_RESTART
export DETAILED_USAGE_STATIC_CHECK="
-${CMDNAME} static-check [FLAGS] static_check [-- <EXTRA_ARGS>]
+${CMDNAME} static-check [FLAGS] STATIC_CHECK [-- <EXTRA_ARGS>]
Run selected static checks for currently changed files. You should specify static check that
you would like to run or 'all' to run all checks. One of:
@@ -1662,8 +1428,8 @@ ${FORMATTED_STATIC_CHECKS}
'${CMDNAME} static-check mypy -- --help'
"
- readonly DETAILED_USAGE_STATIC_CHECK
- export DETAILED_USAGE_TESTS="
+ # shellcheck disable=SC2089
+ DETAILED_USAGE_TESTS="
${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
Run the specified unit test target. There might be multiple
@@ -1677,167 +1443,104 @@ ${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]
Flags:
$(flag_footer)
"
- readonly DETAILED_USAGE_TESTS
+ # shellcheck disable=SC2090
+ export DETAILED_USAGE_TESTS
export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET="
${CMDNAME} toggle-suppress-cheatsheet
Toggles on/off cheatsheet displayed before starting bash shell.
"
- readonly DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET
export DETAILED_USAGE_TOGGLE_SUPPRESS_ASCIIART="
${CMDNAME} toggle-suppress-asciiart
Toggles on/off asciiart displayed before starting bash shell.
"
- readonly DETAILED_USAGE_TOGGLE_SUPPRESS_ASCIIART
export DETAILED_USAGE_HELP="
${CMDNAME} help
Shows general help message for all commands.
"
- readonly DETAILED_USAGE_HELP
export DETAILED_USAGE_HELP_ALL="
${CMDNAME} help-all
Shows detailed help for all commands and flags.
"
- readonly DETAILED_USAGE_HELP_ALL
}
-# shellcheck enable=all
-
-#######################################################################################################
-#
# Gets environment variable value converting the lowercase name of command into variable name
-# Arguments:
-# prefix for the variable to add
-# name of the variable
-# Outputs:
-# Writes the capitalized name of the variable to stdout
-#######################################################################################################
function get_variable_from_lowercase_name() {
- local prefix="${1}"
- local name="${2}"
- local suffix
- suffix="$(echo "${name}" | tr "[:lower:]-" "[:upper:]_")"
- local variable_name="${prefix}_${suffix}"
- echo "${!variable_name}"
+ PREFIX="${1}"
+ NAME="${2}"
+ SUFFIX="$(echo "${NAME}" | tr "[:lower:]-" "[:upper:]_")"
+ VARIABLE_NAME="${PREFIX}_${SUFFIX}"
+ echo "${!VARIABLE_NAME}"
}
-#######################################################################################################
-#
# Gets usage information from lowercase command
-# Arguments:
-# lowercase command name
-# Outputs:
-# Usage information for the command.
-#######################################################################################################
function get_usage() {
- get_variable_from_lowercase_name "USAGE" "${1}"
+ get_variable_from_lowercase_name USAGE "${1}"
}
-#######################################################################################################
-#
+
# Gets detailed usage information from lowercase command
-# Arguments:
-# lowercase command name
-# Outputs:
-# Detailed usage information for the command.
-#######################################################################################################
function get_detailed_usage() {
- get_variable_from_lowercase_name "DETAILED_USAGE" "${1}"
+ get_variable_from_lowercase_name DETAILED_USAGE "${1}"
}
-#######################################################################################################
-#
-# Prints general usage information for all commands.
-#
-# Globals used:
-# CMDNAME
-# BREEZE_COMMANDS
-# BREEZE_EXTRA_ARG_COMMANDS
-# BREEZE_HELP_COMMANDS
-#
-# Outputs:
-# General usage information for all commands.
-#######################################################################################################
+# Prints general usage information
function usage() {
- echo "
+ echo "
Usage: ${CMDNAME} [FLAGS] [COMMAND] -- <EXTRA_ARGS>
-By default the script enters the CI container and drops you to bash shell, but you can choose
-one of the commands to run specific actions instead.
-
-Add --help after each command to see details:
+By default the script enters IT environment and drops you to bash shell, but you can choose one
+of the commands to run specific actions instead. Add --help after each command to see details:
Commands without arguments:
"
- for subcommand in ${BREEZE_COMMANDS}; do
- printf " %-40s %s\n" "${subcommand}" "$(get_usage "${subcommand}")"
- done
- echo "
+ for SUBCOMMAND in ${BREEZE_COMMANDS}
+ do
+ printf " %-40s %s\n" "${SUBCOMMAND}" "$(get_usage "${SUBCOMMAND}")"
+ done
+ echo "
Commands with arguments:
"
- for subcommand in ${BREEZE_EXTRA_ARG_COMMANDS}; do
- printf " %-30s%-10s %s\n" "${subcommand}" "<ARG>" "$(get_usage "${subcommand}")"
- done
- echo "
+ for SUBCOMMAND in ${BREEZE_EXTRA_ARG_COMMANDS}
+ do
+ printf " %-30s%-10s %s\n" "${SUBCOMMAND}" "<ARG>" "$(get_usage "${SUBCOMMAND}")"
+ done
+ echo "
Help commands:
"
- for subcommand in ${BREEZE_HELP_COMMANDS}; do
- printf " %-40s %s\n" "${subcommand}" "$(get_usage "${subcommand}")"
- done
- echo
+ for SUBCOMMAND in ${BREEZE_HELP_COMMANDS}
+ do
+ printf " %-40s %s\n" "${SUBCOMMAND}" "$(get_usage "${SUBCOMMAND}")"
+ done
+ echo
}
-#######################################################################################################
-#
-# Prints detailed usage for command specified
-#
-# Argument:
-# Command name.
-#
-# Outputs:
-# Detailed usage information for the command
-#######################################################################################################
# Prints detailed usage for command specified
function detailed_usage() {
- subcommand=${1}
- echo "
+ SUBCOMMAND=${1}
+ echo "
-Detailed usage for command: ${subcommand}
+Detailed usage for command: ${SUBCOMMAND}
-$(get_detailed_usage "${subcommand}")
+$(get_detailed_usage "${SUBCOMMAND}")
"
}
-#######################################################################################################
-#
-# Prints flag footer - common to all commands..
-#
-# Outputs:
-# Footer common for all commands.
-#######################################################################################################
+# Prints flag footer
function flag_footer() {
- echo "
+ echo "
Run '${CMDNAME} flags' to see all applicable flags.
"
}
-#######################################################################################################
-#
# Prints flags for different variants of airflow to use
-#
-# Global constants used:
-# FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_airflow_variants() {
- echo "
+ echo "
-p, --python <PYTHON_MAJOR_MINOR_VERSION>
Python version used for the image. This is always major/minor version.
@@ -1850,20 +1553,9 @@ ${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints flags for different backend to use
-#
-# Global constants used:
-# FORMATTED_BACKENDS
-# FORMATTED_POSTGRES_VERSIONS
-# FORMATTED_MYSQL_VERSIONS
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_backend_variants() {
- echo "
+ echo "
-b, --backend <BACKEND>
Backend to use for tests - it determines which database is used.
One of:
@@ -1884,33 +1576,17 @@ ${FORMATTED_MYSQL_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints production image flags
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_production_image() {
- echo "
+ echo "
-I, --production-image
Use production image for entering the environment and builds (not for tests).
"
}
-#######################################################################################################
-#
# Prints additional breeze action flags
-#
-# Global constants used:
-# FORMATTED_INTEGRATIONS
-#
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_breeze_actions() {
- echo "
+ echo "
-d, --db-reset
Resets the database at entry to the environment. It will drop all the tables
and data and recreate the DB from scratch even if 'restart' command was not used.
@@ -1928,22 +1604,9 @@ ${FORMATTED_INTEGRATIONS}
"
}
-#######################################################################################################
-#
# Prints Kubernetes action flags
-#
-# Global constants used:
-# FORMATTED_KUBERNETES_MODE
-# FORMATTED_KUBERNETES_VERSIONS
-# FORMATTED_KIND_VERSIONS
-# FORMATTED_HELM_VERSIONS
-#
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_kubernetes_configuration() {
- echo "
+ echo "
Configuration for the KinD Kubernetes cluster and tests:
-K, --kubernetes-mode <KUBERNETES_MODE>
@@ -1981,34 +1644,18 @@ ${FORMATTED_HELM_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints flags that determine what is the source mounting scheme
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_local_file_mounting() {
- echo "
+ echo "
-l, --skip-mounting-local-sources
Skips mounting local volume with sources - you get exactly what is in the
docker image rather than your current local sources of Airflow.
"
}
-#######################################################################################################
-#
# Prints flags that allow to choose different airflow variants
-#
-# Global constants used:
-# FORMATTED_INSTALL_AIRFLOW_VERSIONS
-#
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_choose_different_airflow_version() {
- echo "
+ echo "
-a, --install-airflow-version <INSTALL_AIRFLOW_VERSION>
If specified, installs Airflow directly from PIP released version. This happens at
image building time in production image and at container entering time for CI image. One of:
@@ -2021,15 +1668,9 @@ ${FORMATTED_INSTALL_AIRFLOW_VERSIONS}
"
}
-#######################################################################################################
-#
# Prints flags that allow to set assumed answers to questions
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_assume_answers_to_questions() {
- echo "
+ echo "
-y, --assume-yes
Assume 'yes' answer to all questions.
@@ -2041,30 +1682,18 @@ function flag_assume_answers_to_questions() {
"
}
-#######################################################################################################
-#
# Prints flags that are used for credential forwarding
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_credentials() {
- echo "
+ echo "
-f, --forward-credentials
Forwards host credentials to docker container. Use with care as it will make
your credentials available to everything you install in Docker.
"
}
-#######################################################################################################
-#
# Prints flags that control verbosity
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_verbosity() {
- echo "
+ echo "
-v, --verbose
Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
debugging - when you run breeze with --verbose flags you will be able to see the commands
@@ -2075,33 +1704,16 @@ function flag_verbosity() {
"
}
-#######################################################################################################
-#
-# Prints information about help flag
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_help() {
- echo "
+ echo "
-h, --help
Shows detailed help message for the command specified.
"
}
-#######################################################################################################
-#
# Prints flags controlling docker build process
-#
-# Global constants used:
-# FORMATTED_DEFAULT_CI_EXTRAS
-# FORMATTED_DEFAULT_PROD_EXTRAS
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_build_docker_images() {
- echo "
+ echo "
-F, --force-build-images
Forces building of the local docker images. The images are rebuilt
automatically for the first time or when changes are detected in
@@ -2170,21 +1782,9 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS}
"
}
-#######################################################################################################
-#
# Prints flags controlling docker pull and push process
-#
-# Global constants used:
-# _BREEZE_DEFAULT_DOCKERHUB_USER
-# _BREEZE_DEFAULT_DOCKERHUB_REPO
-# _BREEZE_DEFAULT_GITHUB_REPOSITORY
-# _BREEZE_DEFAULT_GITHUB_IMAGE_ID
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_pull_push_docker_images() {
- echo "
+ echo "
-D, --dockerhub-user
DockerHub user used to pull, push and build images. Default: ${_BREEZE_DEFAULT_DOCKERHUB_USER:=}.
@@ -2216,15 +1816,9 @@ function flag_pull_push_docker_images() {
"
}
-#######################################################################################################
-#
# Prints flags that control version of generated packages
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flag_version_suffix() {
- echo "
+ echo "
-S, --version-suffix-for-pypi
Adds optional suffix to the version in the generated backport package. It can be used
to generate rc1/rc2 ... versions of the packages to be uploaded to PyPI.
@@ -2235,15 +1829,11 @@ function flag_version_suffix() {
"
}
-#######################################################################################################
-#
+
+
# Prints all flags
-#
-# Outputs:
-# Flag information.
-#######################################################################################################
function flags() {
- echo "
+ echo "
$(print_line)
Summary of all flags supported by Breeze:
@@ -2306,13 +1896,7 @@ $(flag_help)
"
}
-#######################################################################################################
-#
-# Prints header line filling screen indented_screen_width - only when VERBOSE is set
-#
-# Outputs:
-# Prints header line.
-#######################################################################################################
+# Prints header line filling screen width - only when VERBOSE is set
function print_header_line() {
if [ ${VERBOSE:="false"} == "true" ]; then
echo
@@ -2321,121 +1905,68 @@ function print_header_line() {
fi
}
-#######################################################################################################
-#
-# Prints separation line filling screen indented_screen_width
-#
-# Outputs:
-# Prints line.
-#######################################################################################################
-function print_line() {
+
+# Prints separation line filling screen width
+function print_line {
printf '#%.0s' $(seq "${SCREEN_WIDTH}")
}
-# Prints star line filling screen indented_screen_width
-function print_star_line() {
+# Prints star line filling screen width
+function print_star_line {
printf '*%.0s' $(seq "${SCREEN_WIDTH}")
}
-#######################################################################################################
-#
-# Reads saved environment variables. Some of the variables are stored across session so that once
-# you use them you do not have to use it next time. That makes those flags persistent.
-#
-# An example of it is `--backend` or '--kubernetes-mode' flags.
-#
-# Note that PYTHON_MAJOR_MINOR_VERSION is not read here - it is read at the
-# `setup_default_breeze_variables` method because it is needed
-# to determine the right images to use and set several variables that depend on the Python version
-#
-# Uses:
-# _BREEZE_DEFAULT_* corresponding defaults for variables it reads
-#
-# Modified and used global constants:
-#
-# BACKEND
-# KUBERNETES_MODE
-# KUBERNETES_VERSION
-# KIND_VERSION
-# HELM_VERSION
-# POSTGRES_VERSION
-# MYSQL_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-#
-#######################################################################################################
-
-function read_saved_environment_variables() {
- BACKEND="${BACKEND:=$(read_from_file BACKEND)}"
- BACKEND=${BACKEND:-${_BREEZE_DEFAULT_BACKEND}}
+# Reads save environment variables. Some of the variables are stored across session so that once
+# you use them you do not have to use it next time. That makes those flags persistent
+# An example of it is `--backend` or '--kubernetes-mode' flags. Note that PYTHON_MAJOR_MINOR_VERSION is
+# not read here - it is read at the `setup_default_breeze_variables` method because it is needed
+# t determine the right images to use and set several variables that depend on the Python version
+function read_saved_environment_variables {
+ export BACKEND="${BACKEND:=$(read_from_file BACKEND)}"
+ export BACKEND=${BACKEND:-${_BREEZE_DEFAULT_BACKEND}}
- KUBERNETES_MODE="${KUBERNETES_MODE:=$(read_from_file KUBERNETES_MODE)}"
- KUBERNETES_MODE=${KUBERNETES_MODE:=${_BREEZE_DEFAULT_KUBERNETES_MODE}}
+ export KUBERNETES_MODE="${KUBERNETES_MODE:=$(read_from_file KUBERNETES_MODE)}"
+ export KUBERNETES_MODE=${KUBERNETES_MODE:=${_BREEZE_DEFAULT_KUBERNETES_MODE}}
- KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(read_from_file KUBERNETES_VERSION)}"
- KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_BREEZE_DEFAULT_KUBERNETES_VERSION}}
+ export KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(read_from_file KUBERNETES_VERSION)}"
+ export KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_BREEZE_DEFAULT_KUBERNETES_VERSION}}
- KIND_VERSION="${KIND_VERSION:=$(read_from_file KIND_VERSION)}"
- KIND_VERSION=${KIND_VERSION:=${_BREEZE_DEFAULT_KIND_VERSION}}
+ export KIND_VERSION="${KIND_VERSION:=$(read_from_file KIND_VERSION)}"
+ export KIND_VERSION=${KIND_VERSION:=${_BREEZE_DEFAULT_KIND_VERSION}}
- HELM_VERSION="${HELM_VERSION:=$(read_from_file HELM_VERSION)}"
- HELM_VERSION=${HELM_VERSION:=${_BREEZE_DEFAULT_HELM_VERSION}}
+ export HELM_VERSION="${HELM_VERSION:=$(read_from_file HELM_VERSION)}"
+ export HELM_VERSION=${HELM_VERSION:=${_BREEZE_DEFAULT_HELM_VERSION}}
- POSTGRES_VERSION="${POSTGRES_VERSION:=$(read_from_file POSTGRES_VERSION)}"
- POSTGRES_VERSION=${POSTGRES_VERSION:=${_BREEZE_DEFAULT_POSTGRES_VERSION}}
+ export POSTGRES_VERSION="${POSTGRES_VERSION:=$(read_from_file POSTGRES_VERSION)}"
+ export POSTGRES_VERSION=${POSTGRES_VERSION:=${_BREEZE_DEFAULT_POSTGRES_VERSION}}
- MYSQL_VERSION="${MYSQL_VERSION:=$(read_from_file MYSQL_VERSION)}"
- MYSQL_VERSION=${MYSQL_VERSION:=${_BREEZE_DEFAULT_MYSQL_VERSION}}
+ export MYSQL_VERSION="${MYSQL_VERSION:=$(read_from_file MYSQL_VERSION)}"
+ export MYSQL_VERSION=${MYSQL_VERSION:=${_BREEZE_DEFAULT_MYSQL_VERSION}}
# Here you read DockerHub user/account that you use
# You can populate your own images in DockerHub this way and work with the,
# You can override it with "--dockerhub-user" option and it will be stored in .build directory
- DOCKERHUB_USER="${DOCKERHUB_USER:=$(read_from_file DOCKERHUB_USER)}"
- DOCKERHUB_USER="${DOCKERHUB_USER:=${_BREEZE_DEFAULT_DOCKERHUB_USER}}"
+ export DOCKERHUB_USER="${DOCKERHUB_USER:=$(read_from_file DOCKERHUB_USER)}"
+ export DOCKERHUB_USER="${DOCKERHUB_USER:=${_BREEZE_DEFAULT_DOCKERHUB_USER}}"
# Here you read DockerHub repo that you use
# You can populate your own images in DockerHub this way and work with them
# You can override it with "--dockerhub-repo" option and it will be stored in .build directory
- DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(read_from_file DOCKERHUB_REPO)}"
- DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_BREEZE_DEFAULT_DOCKERHUB_REPO}}"
+ export DOCKERHUB_REPO="${DOCKERHUB_REPO:=$(read_from_file DOCKERHUB_REPO)}"
+ export DOCKERHUB_REPO="${DOCKERHUB_REPO:=${_BREEZE_DEFAULT_DOCKERHUB_REPO}}"
}
-#######################################################################################################
-#
# Checks if variables are correctly set and if they are - saves them so that they can be used across
-# sessions. In case we are installing Airflow 1.10, the constants are set to match 1.10 line.
-#
-# In case the variables are matching expected values they are saved in ".build/VARIABLE_NAME" for
-# later reuse. If not, error is printed and the saved file is cleaned, so that next time
-# default value can be used.
-#
-# Used Global constants:
-# INSTALL_AIRFLOW_REFERENCE
-# INSTALL_AIRFLOW_VERSION
-# BACKEND
-# KUBERNETES_MODE
-# KUBERNETES_VERSION
-# KIND_VERSION
-# HELM_VERSION
-# POSTGRES_VERSION
-# MYSQL_VERSION
-# DOCKERHUB_USER
-# DOCKERHUB_REPO
-#
-# Updated Global constants:
-# BRANCH_NAME
-#
-# Output: saved variable files in .build,
-#######################################################################################################
+# sessions.
function check_and_save_all_params() {
- check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
+ check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
- if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
- if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *1_10* ]]; then
+ if [[ "${INSTALL_AIRFLOW_REFERENCE:=}" != "" ]]; then
+ if [[ ${INSTALL_AIRFLOW_REFERENCE} == *1_10* ]]; then
export BRANCH_NAME="v1-10-test"
fi
- elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
- if [[ ${INSTALL_AIRFLOW_VERSION=} == *1.10* ]]; then
+ elif [[ "${INSTALL_AIRFLOW_VERSION:=}" != "" ]]; then
+ if [[ ${INSTALL_AIRFLOW_VERSION} == *1.10* ]]; then
export BRANCH_NAME="v1-10-test"
fi
fi
@@ -2452,33 +1983,22 @@ function check_and_save_all_params() {
fi
fi
- check_and_save_allowed_param "BACKEND" "backend" "--backend"
- check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
- check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
- check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
- check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
- check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
- check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
+ check_and_save_allowed_param "BACKEND" "backend" "--backend"
+ check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
+ check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
+ check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
+ check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
+ check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
+ check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
# Can't verify those - they can be anything, so let's just save them
save_to_file DOCKERHUB_USER
save_to_file DOCKERHUB_REPO
}
-#######################################################################################################
-#
# Prints cheatsheet if it is not suppressed
-#
-# Used global constants:
-#
-# AIRFLOW_SOURCES
-# WEBSERVER_HOST_PORT
-# POSTGRES_HOST_PORT
-# MYSQL_HOST_PORT
-#
-#######################################################################################################
function print_cheatsheet() {
- if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${command_to_run} == "enter_breeze" ]]; then
+ if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${COMMAND_TO_RUN} == "enter_breeze" ]]; then
echo
echo
print_line
@@ -2517,18 +2037,12 @@ function print_cheatsheet() {
fi
}
-#######################################################################################################
-#
# Prints setup instruction in case we find that autocomplete is not set
# also prints how to toggle asciiart/cheatsheet
-#
-# Used global constants:
-# CMDNAME
-#######################################################################################################
-function print_setup_instructions() {
- if [[ ${command_to_run} == "enter_breeze" ]]; then
+function print_setup_instructions {
+ if [[ ${COMMAND_TO_RUN} == "enter_breeze" ]] ; then
# shellcheck disable=SC2034 # Unused variables left for comp_breeze usage
- if ! typeset -f "_comp_breeze" >/dev/null; then
+ if ! typeset -f "_comp_breeze" > /dev/null; then
print_line
echo
echo " You can setup autocomplete by running '${CMDNAME} setup-autocomplete'"
@@ -2549,360 +2063,237 @@ function print_setup_instructions() {
fi
}
-#######################################################################################################
-#
# Checks that pre-commit is installed and upgrades it if needed
-# this is used in case static check command is used.
-#
-#######################################################################################################
-function make_sure_precommit_is_installed() {
+# this is used in case static check command is used
+function make_sure_precommit_is_installed {
echo
echo "Making sure pre-commit is installed"
echo
- local pip_binary
if command -v pip3 >/dev/null; then
- pip_binary=pip3
+ PIP_BIN=pip3
elif command -v pip >/dev/null; then
- pip_binary=pip
+ PIP_BIN=pip
else
- echo >&2
- echo >&2 "ERROR: You need to have pip or pip3 in your PATH"
- echo >&2
- S
- exit 1
+ echo >&2
+ echo >&2 "ERROR: You need to have pip or pip3 in your PATH"
+ echo >&2
+ S
+ exit 1
fi
- "${pip_binary}" install --upgrade pre-commit >/dev/null 2>&1
+ "${PIP_BIN}" install --upgrade pre-commit >/dev/null 2>&1
# Add ~/.local/bin to the path in case pip is run outside of virtualenv
export PATH="${PATH}":~/.local/bin
}
-#######################################################################################################
-#
# Removes CI and PROD images and cleans up the flag that indicates that the image was already built
-#
-# Used global constants:
-# PYTHON_BASE_IMAGE
-# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_IMAGE
-# BUILT_CI_IMAGE_FLAG_FILE
-#
-#######################################################################################################
-function remove_images() {
- docker rmi "${PYTHON_BASE_IMAGE}" || true
- docker rmi "${AIRFLOW_CI_IMAGE}" || true
+function remove_images {
+ docker rmi "${PYTHON_BASE_IMAGE}" || true
+ docker rmi "${AIRFLOW_CI_IMAGE}" || true
docker rmi "${AIRFLOW_PROD_IMAGE}" || true
- rm -f "${BUILT_CI_IMAGE_FLAG_FILE}"
+ rm -f "${BUILT_IMAGE_FLAG_FILE}"
}
-#######################################################################################################
-#
# Runs chosen static checks
-#
-# Uses variables:
-# static_check
-#
-#######################################################################################################
-function run_static_checks() {
- if [[ ${static_check} == "all" ]]; then
+function run_static_checks {
+ if [[ ${STATIC_CHECK} == "all" ]]; then
echo
echo "Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
echo
pre-commit run "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
- elif [[ ${static_check} == "all-but-pylint" ]]; then
+ elif [[ ${STATIC_CHECK} == "all-but-pylint" ]]; then
echo
echo "Setting SKIP=pylint. Running: pre-commit run" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
echo
echo
SKIP=pylint pre-commit run "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
else
- if [[ ${static_check} == *"pylint"* || ${static_check} == *"mypy"* || ${static_check} == *"flake8"* ]]; then
- echo
- echo "Running build pre-commit before running the requested static check"
- echo
- pre-commit run "build"
- fi
echo
- echo "Running: pre-commit run" "${static_check}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
+ echo "Running: pre-commit run" "${STATIC_CHECK}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
echo
- pre-commit run "${static_check}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
+ pre-commit run "${STATIC_CHECK}" "${EXTRA_STATIC_CHECK_OPTIONS[@]}" "$@"
fi
}
-#######################################################################################################
-#
-# Runs Build before a command if needed - the right build command will be determined and used
-# depending on which command you are trying ton
-#
-# Used global constants:
-# PRODUCTION_IMAGE
-# KIND_CLUSTER_OPERATION
-# FORMATTED_KIND_OPERATIONS
-#
-# Used variables:
-# command_to_run
-#
-#######################################################################################################
-function run_build_command() {
- case "${command_to_run}" in
- run_tests | run_docker_compose)
- prepare_ci_build
- rebuild_ci_image_if_needed
- ;;
- enter_breeze)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- else
+# Runs Build before a command - it will check and if needed rebuild necessary image, depending on the
+# command chosen
+function run_build_command {
+ case "${COMMAND_TO_RUN}" in
+ run_tests|run_docker_compose)
prepare_ci_build
rebuild_ci_image_if_needed
- fi
- ;;
- build_docs | perform_static_checks | perform_generate_constraints | perform_prepare_backport_readme | perform_prepare_backport_packages)
- prepare_ci_build
- rebuild_ci_image_if_needed
- ;;
- perform_push_image)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- else
+ ;;
+ enter_breeze)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ else
+ prepare_ci_build
+ rebuild_ci_image_if_needed
+ fi
+ ;;
+ build_docs|perform_static_checks|perform_generate_constraints|perform_prepare_backport_readme|perform_prepare_backport_packages)
prepare_ci_build
rebuild_ci_image_if_needed
- fi
- ;;
- build_image)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- build_prod_images
- else
- prepare_ci_build
- calculate_md5sum_for_all_files
- build_ci_image
- update_all_md5_files
- build_ci_image_manifest
- fi
- ;;
- cleanup_image | run_exec)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- prepare_prod_build
- else
- prepare_ci_build
- fi
- ;;
- perform_initialize_local_virtualenv | perform_setup_autocomplete | toggle_suppress_cheatsheet | toggle_suppress_asciiart ) ;;
-
- manage_kind_cluster)
- if [[ ${KIND_CLUSTER_OPERATION} == "start" ]]; then
- echo "Starts KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]]; then
- echo "Stops KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "restart" ]]; then
- echo "Restarts KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "recreate" ]]; then
- echo "Recreates KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "status" ]]; then
- echo "Checks status of KinD cluster"
- elif [[ ${KIND_CLUSTER_OPERATION} == "deploy" ]]; then
- echo "Deploys Airflow to KinD cluster"
- prepare_prod_build
- build_prod_images
- elif [[ ${KIND_CLUSTER_OPERATION} == "test" ]]; then
- echo "Run Kubernetes tests with the KinD cluster "
- elif [[ ${KIND_CLUSTER_OPERATION} == "shell" ]]; then
- echo "Enter an interactive shell for kubernetes testing"
- elif [[ -z ${KIND_CLUSTER_OPERATION=} ]]; then
- echo
- echo "Please provide an operation to run"
- echo
- echo "Should be one of:"
- echo "${FORMATTED_KIND_OPERATIONS}"
- echo
- exit 1
- else
- echo
- echo "ERROR: Unknown Kind Kubernetes cluster operation: '${KIND_CLUSTER_OPERATION}'"
- echo
- echo "Should be one of:"
- echo "${FORMATTED_KIND_OPERATIONS}"
- echo
- exit 1
- fi
- ;;
- *)
- echo >&2
- echo >&2 "ERROR: Unknown command to run ${command_to_run}"
- echo >&2
- exit 1
- ;;
+ ;;
+ perform_push_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ else
+ prepare_ci_build
+ rebuild_ci_image_if_needed
+ fi
+ ;;
+ build_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ build_prod_images
+ else
+ prepare_ci_build
+ calculate_md5sum_for_all_files
+ build_ci_image
+ update_all_md5_files
+ build_ci_image_manifest
+ fi
+ ;;
+ cleanup_image|run_exec)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ prepare_prod_build
+ else
+ prepare_ci_build
+ fi
+ ;;
+ perform_initialize_local_virtualenv|perform_setup_autocomplete)
+ ;;
+ manage_kind_cluster)
+ if [[ ${KIND_CLUSTER_OPERATION} == "start" ]] ; then
+ echo "Starts KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]] ; then
+ echo "Stops KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "restart" ]] ; then
+ echo "Restarts KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "recreate" ]] ; then
+ echo "Recreates KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "status" ]] ; then
+ echo "Checks status of KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "deploy" ]] ; then
+ echo "Deploys Airflow to KinD cluster"
+ elif [[ ${KIND_CLUSTER_OPERATION} == "test" ]] ; then
+ echo "Run Kubernetes tests with the KinD cluster "
+ elif [[ ${KIND_CLUSTER_OPERATION} == "shell" ]] ; then
+ echo "Enter an interactive shell for kubernetes testing"
+ else
+ echo "ERROR: Unknown Kind Kubernetes cluster operation: '${KIND_CLUSTER_OPERATION}'"
+ echo
+ echo "Should be one of:"
+ echo "${FORMATTED_KIND_OPERATIONS}"
+ echo
+ exit 1
+ fi
+ ;;
+ *)
+ echo >&2
+ echo >&2 "ERROR: Unknown command to run ${COMMAND_TO_RUN}"
+ echo >&2
+ exit 1
+ ;;
esac
}
-#######################################################################################################
-#
# Runs the actual command - depending on the command chosen it will use the right
-# Convenient script and run the right command for the script.
-#
-# Used variables:
-# command_to_run
-#
-# Used global constants:
-# PRODUCTION_IMAGE
-# SCRIPTS_CI_DIR
-# BUILD_CACHE_DIR
-# KIND_CLUSTER_OPERATION
-# EXTRA_DC_OPTIONS
-#
-# Set Global variables:
-# RUN_TESTS
-#######################################################################################################
-function run_breeze_command() {
+# Convenient script and run the right command with it
+function run_breeze_command {
set +u
- local dc_run_file
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}"
- else
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
- fi
- case "${command_to_run}" in
- enter_breeze)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- "${dc_run_file}" run --service-ports --rm airflow "${@}"
- "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
- else
- "${dc_run_file}" run --service-ports --rm airflow "${@}"
- "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
- "${SCRIPTS_CI_DIR}/tools/ci_clear_tmp.sh"
- fi
- ;;
- run_exec)
- # Unfortunately `docker-compose exec` does not support exec'ing into containers started with run :(
- # so we have to find it manually
- set +e
- local airflow_testing_container
- airflow_testing_container=$("${dc_run_file}" ps | grep airflow | awk '{print $1}' 2>/dev/null)
- : "${airflow_testing_container:?"ERROR! Breeze must be running in order to exec into running container"}"
- set -e
- docker exec -it "${airflow_testing_container}" \
- "/opt/airflow/scripts/in_container/entrypoint_exec.sh" "${@}"
- ;;
- run_tests)
- export RUN_TESTS="true"
- readonly RUN_TESTS
- "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" run --service-ports --rm airflow "$@"
- ;;
- run_docker_compose)
- set +u
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}"
- else
- dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
- fi
- "${dc_run_file}" "${docker_compose_command}" "${EXTRA_DC_OPTIONS[@]}" "$@"
- set -u
- ;;
- perform_static_checks)
- make_sure_precommit_is_installed
- run_static_checks "${@}"
- ;;
- build_image) ;;
-
- cleanup_image)
- remove_images
- ;;
- perform_generate_constraints)
- run_generate_constraints
- ;;
- perform_prepare_backport_packages)
- run_prepare_backport_packages "${@}"
- ;;
- perform_prepare_backport_readme)
- run_prepare_backport_readme "${@}"
- ;;
- perform_push_image)
- if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- push_prod_images
- else
- push_ci_images
- fi
- ;;
- perform_initialize_local_virtualenv)
- initialize_virtualenv
- ;;
- perform_setup_autocomplete)
- setup_autocomplete
- ;;
- manage_kind_cluster)
- make_sure_kubernetes_tools_are_installed
- get_kind_cluster_name
- perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
- ;;
- build_docs)
- run_docs "${@}"
- ;;
- toggle_suppress_cheatsheet)
- if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
- echo
- echo "Cheatsheet disabled"
- echo
- else
- echo
- echo "Cheatsheet enabled"
- echo
- fi
- ;;
- toggle_suppress_asciiart)
- if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
- echo
- echo "ASCIIart disabled"
- echo
- else
- echo
- echo "ASCIIart enabled"
- echo
- fi
- ;;
- *)
- echo >&2
- echo >&2 "ERROR: Unknown command to run ${command_to_run}"
- echo >&2
- ;;
+ case "${COMMAND_TO_RUN}" in
+ enter_breeze)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ "${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}" run --service-ports --rm airflow "${@}"
+ "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
+ else
+ "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" run --service-ports --rm airflow "${@}"
+ "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh"
+ "${SCRIPTS_CI_DIR}/tools/ci_clear_tmp.sh"
+ fi
+ ;;
+ run_exec)
+ # Unfortunately docker-compose exec does not support exec'ing into containers started with run :(
+ set +e
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}"
+ else
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}"
+ fi
+ AIRFLOW_TESTING_CONTAINER=$("${DC_FILE}" ps | \
+ grep airflow | awk '{print $1}' 2>/dev/null)
+ : "${AIRFLOW_TESTING_CONTAINER:?"ERROR! Breeze must be running in order to exec into running container"}"
+ set -e
+ docker exec -it "${AIRFLOW_TESTING_CONTAINER}" \
+ "/opt/airflow/scripts/in_container/entrypoint_exec.sh" "${@}"
+ ;;
+ run_tests)
+ export RUN_TESTS="true"
+ "${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}" run --service-ports --rm airflow "$@"
+ ;;
+ run_docker_compose)
+ set +u
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_PROD_FILE}"
+ else
+ DC_FILE="${BUILD_CACHE_DIR}/${LAST_DC_CI_FILE}"
+ fi
+ "${DC_FILE}" "${DOCKER_COMPOSE_COMMAND}" "${EXTRA_DC_OPTIONS[@]}" "$@"
+ set -u
+ ;;
+ perform_static_checks)
+ make_sure_precommit_is_installed
+ run_static_checks "${@}"
+ ;;
+ build_image)
+ ;;
+ cleanup_image)
+ remove_images
+ ;;
+ perform_generate_constraints)
+ run_generate_constraints
+ ;;
+ perform_prepare_backport_packages)
+ run_prepare_backport_packages "${@}"
+ ;;
+ perform_prepare_backport_readme)
+ run_prepare_backport_readme "${@}"
+ ;;
+ perform_push_image)
+ if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
+ push_prod_images
+ else
+ push_ci_images
+ fi
+ ;;
+ perform_initialize_local_virtualenv)
+ initialize_virtualenv
+ ;;
+ perform_setup_autocomplete)
+ setup_autocomplete
+ ;;
+ manage_kind_cluster)
+ make_sure_kubernetes_tools_are_installed
+ initialize_kind_variables
+ perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
+ ;;
+ build_docs)
+ run_docs "${@}"
+ ;;
+ *)
+ echo >&2
+ echo >&2 "ERROR: Unknown command to run ${COMMAND_TO_RUN}"
+ echo >&2
+ ;;
esac
set -u
}
-#######################################################################################################
-#
-# We have different versions of images depending on the python version used. We keep up with the
-# Latest patch-level changes in Python (this is done automatically during CI builds) so we have
-# To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor
-# version of python in X.Y format (3.6, 3.7, 3.8 etc).
-#
-# In Breeze the precedence of setting the version is as follows:
-# 1. --python flag (if set, it will explicitly override it in the next step)
-# 2. PYTHON_MAJOR_MINOR_VERSION exported from outside
-# 3. last used version stored in ./build/PYTHON_MAJOR_MINOR_VERSION
-# 4. DEFAULT_PYTHON_MAJOR_MINOR_VERSION from scripts/ci/libraries/_initialization.sh
-#
-# Here points 2. and 3. are realised. If result is empty string , the 4. will be set in
-# the next step (basic_sanity_checks() is called and the version is still not set by then)
-# finally, if --python flag is specified, it will override whatever is set above.
-#
-# We need to run after initialize_common_environment (so that read_from_file function is present)
-# But before we set the default value for Python
-#
-# Used and modified global constants:
-# PYTHON_MAJOR_MINOR_VERSION
-#######################################################################################################
-function determine_python_version_to_use_in_breeze() {
- PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
- export PYTHON_MAJOR_MINOR_VERSION
-}
-
-
-setup_default_breeze_constants
+setup_default_breeze_variables
initialize_common_environment
-determine_python_version_to_use_in_breeze
-
basic_sanity_checks
script_start
@@ -2924,12 +2315,6 @@ read_saved_environment_variables
check_and_save_all_params
-determine_docker_cache_strategy
-
-get_docker_image_names
-
-make_constants_read_only
-
sanitize_mounted_files
prepare_command_files
@@ -2944,11 +2329,11 @@ print_cheatsheet
print_setup_instructions
-set +u # Account for an empty array
+set +u # Account for an empty array
run_breeze_command "${REMAINING_ARGS[@]}"
-set +u # Account for an empty array
-if [[ -n ${second_command_to_run} ]]; then
- command_to_run=${second_command_to_run}
+set +u # Account for an empty array
+if [[ -n ${SECOND_COMMAND_TO_RUN} ]]; then
+ COMMAND_TO_RUN=${SECOND_COMMAND_TO_RUN}
run_breeze_command "${REMAINING_ARGS[@]}"
fi
diff --git a/breeze-complete b/breeze-complete
index 336ad0f..d30beb5 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -28,23 +28,6 @@ _BREEZE_ALLOWED_MYSQL_VERSIONS="5.7 8"
_BREEZE_ALLOWED_POSTGRES_VERSIONS="9.6 10"
_BREEZE_ALLOWED_KIND_OPERATIONS="start stop restart status deploy test shell"
-# Default values for the flags used
-
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_BACKEND="sqlite"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_KUBERNETES_MODE="image"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_KUBERNETES_VERSION="v1.18.6"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_KIND_VERSION="v0.8.0"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_HELM_VERSION="v3.2.4"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_POSTGRES_VERSION="9.6"
-# shellcheck disable=SC2034
-_BREEZE_DEFAULT_MYSQL_VERSION="5.7"
-
_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS=$(cat <<-EOF
1.10.12
1.10.11
@@ -204,6 +187,9 @@ function _get_known_values_breeze() {
static-check)
_BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_STATIC_CHECKS}
;;
+ static-check-all-files)
+ _BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_STATIC_CHECKS}
+ ;;
-A | --install-airflow-version)
_BREEZE_KNOWN_VALUES=${_BREEZE_ALLOWED_INSTALL_AIRFLOW_VERSIONS}
;;
diff --git a/confirm b/confirm
index e796737..50ab15f 100755
--- a/confirm
+++ b/confirm
@@ -17,7 +17,7 @@
# under the License.
set -euo pipefail
-if [[ -n "${FORCE_ANSWER_TO_QUESTIONS=}" ]]; then
+if [[ "${FORCE_ANSWER_TO_QUESTIONS:=""}" != "" ]]; then
RESPONSE=${FORCE_ANSWER_TO_QUESTIONS}
echo
echo "Forcing response '${FORCE_ANSWER_TO_QUESTIONS}' to '${1}?'"
diff --git a/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
index 1619396..d756dac 100755
--- a/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
+++ b/scripts/ci/backport_packages/ci_prepare_and_test_backport_packages.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/backport_packages/ci_prepare_backport_packages.sh b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
index f30b4a7..e1c8bd0 100755
--- a/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
+++ b/scripts/ci/backport_packages/ci_prepare_backport_packages.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/backport_packages/ci_prepare_backport_readme.sh b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
index 8bf481c..b869128 100755
--- a/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
+++ b/scripts/ci/backport_packages/ci_prepare_backport_readme.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
index 190659a..11f009a 100755
--- a/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
+++ b/scripts/ci/backport_packages/ci_test_backport_packages_import_all_classes.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -31,6 +33,8 @@ function run_test_package_import_all_classes() {
"--" "/opt/airflow/scripts/in_container/run_test_package_import_all_classes.sh"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
index d923683..2602c2b 100755
--- a/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
+++ b/scripts/ci/backport_packages/ci_test_backport_packages_install_separately.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -26,6 +28,8 @@ function run_test_package_installation_separately() {
"--" "/opt/airflow/scripts/in_container/run_test_package_installation_separately.sh"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/constraints/ci_branch_constraints.sh b/scripts/ci/constraints/ci_branch_constraints.sh
index 4cc7227..5bcc688 100755
--- a/scripts/ci/constraints/ci_branch_constraints.sh
+++ b/scripts/ci/constraints/ci_branch_constraints.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
if [[ ${GITHUB_REF} == 'refs/heads/master' ]]; then
echo "::set-output name=branch::constraints-master"
elif [[ ${GITHUB_REF} == 'refs/heads/v1-10-test' ]]; then
diff --git a/scripts/ci/constraints/ci_commit_constraints.sh b/scripts/ci/constraints/ci_commit_constraints.sh
index f950eaf..f20e834 100755
--- a/scripts/ci/constraints/ci_commit_constraints.sh
+++ b/scripts/ci/constraints/ci_commit_constraints.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
cp -v ./artifacts/constraints-*/constraints*.txt repo/
cd repo || exit 1
git config --local user.email "dev@airflow.apache.org"
diff --git a/scripts/ci/constraints/ci_generate_constraints.sh b/scripts/ci/constraints/ci_generate_constraints.sh
index 946d010..1b7ee4f 100755
--- a/scripts/ci/constraints/ci_generate_constraints.sh
+++ b/scripts/ci/constraints/ci_generate_constraints.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/docs/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
index 0058454..13925ad 100755
--- a/scripts/ci/docs/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -15,9 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index 4fa23a9..aba2dcb 100755
--- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -21,6 +21,7 @@
# Builds or waits for the CI image in the CI environment
# Depending on "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_ci_image_on_ci() {
+ get_environment_for_builds_on_ci
prepare_ci_build
rm -rf "${BUILD_CACHE_DIR}"
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index 65a4da4..9f16bf8 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -21,6 +21,7 @@
# Builds or waits for the PROD image in the CI environment
# Depending on the "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_prod_images_on_ci() {
+ get_environment_for_builds_on_ci
prepare_prod_build
rm -rf "${BUILD_CACHE_DIR}"
diff --git a/scripts/ci/images/ci_push_ci_images.sh b/scripts/ci/images/ci_push_ci_images.sh
index 48d9ec5..2d69d9e 100755
--- a/scripts/ci/images/ci_push_ci_images.sh
+++ b/scripts/ci/images/ci_push_ci_images.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/images/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh
index 5b6f800..db439d8 100755
--- a/scripts/ci/images/ci_push_production_images.sh
+++ b/scripts/ci/images/ci_push_production_images.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/images/ci_wait_for_all_ci_images.sh b/scripts/ci/images/ci_wait_for_all_ci_images.sh
index a1dd705..74bb3ef 100755
--- a/scripts/ci/images/ci_wait_for_all_ci_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_ci_images.sh
@@ -15,18 +15,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
-echo
-echo "Airflow sources: ${AIRFLOW_SOURCES}"
-echo
+
+# This is hook build used by DockerHub. We are also using it
+# on CI to potentially rebuild (and refresh layers that
+# are not cached) Docker images that are used to run CI jobs
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
+ echo "This script should not be called!"
+ echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY} and GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
echo
exit 1
fi
@@ -40,14 +39,7 @@ echo "Check if jq is installed"
echo
command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
-echo
-echo "The jq version $(jq --version)"
-echo
-
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
-initialize_common_environment
+jq --version
for PYTHON_MAJOR_MINOR_VERSION in "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}"
do
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh
index 9cb4b34..51e1ca9 100755
--- a/scripts/ci/images/ci_wait_for_all_prod_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -15,18 +15,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
-echo
-echo "Airflow sources: ${AIRFLOW_SOURCES}"
-echo
+
+# This is hook build used by DockerHub. We are also using it
+# on CI to potentially rebuild (and refresh layers that
+# are not cached) Docker images that are used to run CI jobs
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
+ echo "This script should not be called!"
+ echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY} and GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
echo
exit 1
fi
@@ -38,17 +37,9 @@ echo
echo
echo "Check if jq is installed"
echo
-command -v jq >/dev/null || (echo "ERROR! You must have 'jq' tool installed!" && exit 1)
-
-echo
-echo "The jq version $(jq --version)"
-echo
-
-# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
-
-initialize_common_environment
+command -v jq
+jq --version
for PYTHON_MAJOR_MINOR_VERSION in "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}"
do
export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
diff --git a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
index f8393f2..d6d653d 100755
--- a/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
+++ b/scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
@@ -19,10 +19,18 @@
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
set -euo pipefail
-add_trap "dump_kind_logs" EXIT HUP INT TERM
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
+export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
+export KUBERNETES_MODE=${KUBERNETES_MODE:="image"}
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
+
+get_environment_for_builds_on_ci
+initialize_kind_variables
make_sure_kubernetes_tools_are_installed
-get_kind_cluster_name
prepare_prod_build
build_prod_images
build_image_for_kubernetes_tests
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index e5e7615..78a61f8 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -18,10 +18,10 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-make_sure_kubernetes_tools_are_installed
-get_kind_cluster_name
-
-add_trap dump_kind_logs EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}dump_kind_logs" EXIT
INTERACTIVE="false"
@@ -70,6 +70,9 @@ else
fi
+get_environment_for_builds_on_ci
+initialize_kind_variables
+
cd "${AIRFLOW_SOURCES}" || exit 1
VIRTUALENV_PATH="${BUILD_CACHE_DIR}/.kubernetes_venv"
diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
index 3688c03..dca8dbd 100755
--- a/scripts/ci/libraries/_all_libs.sh
+++ b/scripts/ci/libraries/_all_libs.sh
@@ -17,17 +17,11 @@
# under the License.
LIBRARIES_DIR=$(dirname "${BASH_SOURCE[0]}")
-export LIBRARIES_DIR
-readonly LIBRARIES_DIR
-SCRIPTS_CI_DIR=$(cd "${LIBRARIES_DIR}/.." && pwd)
-export SCRIPTS_CI_DIR
-readonly SCRIPTS_CI_DIR
-
-# shellcheck source=scripts/ci/libraries/_traps.sh
-. "${LIBRARIES_DIR}"/_traps.sh
# shellcheck source=scripts/ci/libraries/_initialization.sh
. "${LIBRARIES_DIR}"/_initialization.sh
+
+
# shellcheck source=scripts/ci/libraries/_sanity_checks.sh
. "${LIBRARIES_DIR}"/_sanity_checks.sh
# shellcheck source=scripts/ci/libraries/_build_images.sh
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 7439f03..e94b920 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
+
# For remote installation of airflow (from GitHub or Pypi) when building the image, you need to
# pass build flags depending on the version and method of the installation (for example to
# get proper requirement constraint files)
@@ -55,7 +56,7 @@ function add_build_args_for_remote_install() {
# Retrieves version of airflow stored in the production image (used to display the actual
# Version we use if it was build from PyPI or GitHub
function get_airflow_version_from_production_image() {
- VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
+ VERBOSE="false" docker run --entrypoint /bin/bash "${AIRFLOW_PROD_IMAGE}" -c 'echo "${AIRFLOW_VERSION}"'
}
# Removes the "Forced answer" (yes/no/quit) given previously, unless you specifically want to remember it.
@@ -110,20 +111,17 @@ function confirm_image_rebuild() {
print_info "CI environment - forcing rebuild for image ${THE_IMAGE_TYPE}."
print_info
RES="0"
- elif [[ -n "${FORCE_ANSWER_TO_QUESTIONS=}" ]]; then
+ elif [[ -n "${FORCE_ANSWER_TO_QUESTIONS:=""}" ]]; then
print_info
print_info "Forcing answer '${FORCE_ANSWER_TO_QUESTIONS}'"
print_info
case "${FORCE_ANSWER_TO_QUESTIONS}" in
- [yY][eE][sS] | [yY])
- RES="0"
- ;;
- [qQ][uU][iI][tT] | [qQ])
- RES="2"
- ;;
- *)
- RES="1"
- ;;
+ [yY][eE][sS]|[yY])
+ RES="0" ;;
+ [qQ][uU][iI][tT]|[qQ])
+ RES="2" ;;
+ *)
+ RES="1" ;;
esac
elif [[ -t 0 ]]; then
# Check if this script is run interactively with stdin open and terminal attached
@@ -161,7 +159,7 @@ function confirm_image_rebuild() {
# Force "no" also to subsequent questions so that if you answer it once, you are not asked
# For all other pre-commits and you will continue using the images you already have
export FORCE_ANSWER_TO_QUESTIONS="no"
- echo 'export FORCE_ANSWER_TO_QUESTIONS="no"' >"${LAST_FORCE_ANSWER_FILE}"
+ echo 'export FORCE_ANSWER_TO_QUESTIONS="no"' > "${LAST_FORCE_ANSWER_FILE}"
elif [[ ${RES} == "2" ]]; then
echo >&2
echo >&2 "ERROR: The ${THE_IMAGE_TYPE} needs to be rebuilt - it is outdated. "
@@ -183,11 +181,11 @@ function confirm_image_rebuild() {
# We cannot use docker registry APIs as they are available only with authorisation
# But this image can be pulled without authentication
function build_ci_image_manifest() {
- docker inspect "${AIRFLOW_CI_IMAGE}" >"manifests/${AIRFLOW_CI_BASE_TAG}.json"
+ docker inspect "${AIRFLOW_CI_IMAGE}" > "manifests/${AIRFLOW_CI_BASE_TAG}.json"
docker build \
- --build-arg AIRFLOW_CI_BASE_TAG="${AIRFLOW_CI_BASE_TAG}" \
- --tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \
- -f- . <<EOF
+ --build-arg AIRFLOW_CI_BASE_TAG="${AIRFLOW_CI_BASE_TAG}" \
+ --tag="${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" \
+ -f- . <<EOF
ARG AIRFLOW_CI_BASE_TAG
FROM scratch
@@ -216,7 +214,7 @@ function get_local_image_info() {
return
fi
set -e
- # Create manifest from the local manifest image
+ # Create manifest from the local manifest image
docker cp "local-airflow-manifest:${AIRFLOW_CI_BASE_TAG}.json" "${TMP_MANIFEST_LOCAL_JSON}"
sed 's/ *//g' "${TMP_MANIFEST_LOCAL_JSON}" | grep '^"sha256:' >"${TMP_MANIFEST_LOCAL_SHA}"
docker rm --force "local-airflow-manifest"
@@ -233,10 +231,10 @@ function get_local_image_info() {
function get_remote_image_info() {
set +e
# Pull remote manifest image
- if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"; then
- echo >&2
- echo >&2 "Remote docker registry unreachable"
- echo >&2
+ if ! docker pull "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"; then
+ >&2 echo
+ >&2 echo "Remote docker registry unreachable"
+ >&2 echo
REMOTE_DOCKER_REGISTRY_UNREACHABLE="true"
return
fi
@@ -257,7 +255,7 @@ function get_remote_image_info() {
"${TMP_MANIFEST_REMOTE_JSON}"
# Filter everything except SHAs of image layers
sed 's/ *//g' "${TMP_MANIFEST_REMOTE_JSON}" | grep '^"sha256:' >"${TMP_MANIFEST_REMOTE_SHA}"
- docker rm --force "$(cat "${TMP_CONTAINER_ID}")"
+ docker rm --force "$( cat "${TMP_CONTAINER_ID}")"
}
# The Number determines the cut-off between local building time and pull + build time.
@@ -280,7 +278,7 @@ MAGIC_CUT_OFF_NUMBER_OF_LAYERS=36
# Compares layers from both remote and local image and set FORCE_PULL_IMAGES to true in case
# More than the last NN layers are different.
function compare_layers() {
- NUM_DIFF=$(diff -y --suppress-common-lines "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_SHA}" |
+ NUM_DIFF=$(diff -y --suppress-common-lines "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_SHA}" | \
wc -l || true)
rm -f "${TMP_MANIFEST_REMOTE_JSON}" "${TMP_MANIFEST_REMOTE_SHA}" "${TMP_MANIFEST_LOCAL_JSON}" "${TMP_MANIFEST_LOCAL_SHA}"
echo
@@ -288,7 +286,7 @@ function compare_layers() {
echo
# This is where setup py is rebuilt - it will usually take a looooot of time to build it, so it is
# Better to pull here
- if ((NUM_DIFF >= MAGIC_CUT_OFF_NUMBER_OF_LAYERS)); then
+ if (( NUM_DIFF >= MAGIC_CUT_OFF_NUMBER_OF_LAYERS )); then
echo
echo
echo "WARNING! Your image and the dockerhub image differ significantly"
@@ -304,6 +302,7 @@ function compare_layers() {
fi
}
+
# Prints summary of the build parameters
function print_build_info() {
print_info
@@ -311,48 +310,28 @@ function print_build_info() {
print_info
}
-function get_docker_image_names() {
+function get_base_image_version() {
# python image version to use
- export PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}}
-
- # Python base image to use
- export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
-
- # CI image base tag
- export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
-
- # CI image to build
- export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
-
- # Default CI image
- export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-ci"
-
- # Base production image tag - used to build kubernetes tag as well
- export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
-
- # PROD image to build
- export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}"
-
- # PROD build segment
- export AIRFLOW_PROD_BUILD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-build"
-
- # PROD Kubernetes image to build
- export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}}
+}
- # PROD default image
- export AIRFLOW_PROD_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}"
- # File that is touched when the CI image is built for the first time locally
- export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
-}
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
# DockerHub user etc. the variables are set so that other functions can use those variables.
function prepare_ci_build() {
+ get_base_image_version
+ # We use pulled docker image cache by default for CI images to speed up the builds
+ export DOCKER_CACHE=${DOCKER_CACHE:="pulled"}
+ print_info
+ print_info "Using ${DOCKER_CACHE} cache strategy for the build."
+ print_info
+ export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
export AIRFLOW_CI_LOCAL_MANIFEST_IMAGE="local/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
export AIRFLOW_CI_REMOTE_MANIFEST_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}-manifest"
+ export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- if [[ -n ${GITHUB_TOKEN=} ]]; then
+ if [[ ${GITHUB_TOKEN:=} != "" ]]; then
echo "${GITHUB_TOKEN}" | docker login \
--username "${GITHUB_USERNAME}" \
--password-stdin \
@@ -362,6 +341,10 @@ function prepare_ci_build() {
export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${GITHUB_REGISTRY}/${GITHUB_REPOSITORY_LOWERCASE}/${AIRFLOW_CI_BASE_TAG}"
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${GITHUB_REGISTRY}/${GITHUB_REPOSITORY_LOWERCASE}/python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
fi
+ export AIRFLOW_BUILD_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}/${AIRFLOW_CI_BASE_TAG}"
+ export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-ci"
+ export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
+ export BUILT_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then
export DEFAULT_PROD_IMAGE="${AIRFLOW_CI_IMAGE_DEFAULT}"
else
@@ -369,25 +352,24 @@ function prepare_ci_build() {
fi
export THE_IMAGE_TYPE="CI"
export IMAGE_DESCRIPTION="Airflow CI"
-
- # Those constants depend on the type of image run so they are only made constants here
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}"
- readonly AIRFLOW_EXTRAS
-
+ export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
+ export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
+ export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}"
+ export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
- readonly AIRFLOW_IMAGE
-
go_to_airflow_sources
fix_group_permissions
}
+
# Only rebuilds CI image if needed. It checks if the docker image build is needed
# because any of the important source files (from scripts/ci/libraries/_initialization.sh) has
# changed or in any of the edge cases (docker image removed, .build cache removed etc.
# In case rebuild is needed, it determines (by comparing layers in local and remote image)
# Whether pull is needed before rebuild.
function rebuild_ci_image_if_needed() {
- if [[ -f "${BUILT_CI_IMAGE_FLAG_FILE}" ]]; then
+ if [[ -f "${BUILT_IMAGE_FLAG_FILE}" ]]; then
print_info
print_info "${THE_IMAGE_TYPE} image already built locally."
print_info
@@ -399,7 +381,7 @@ function rebuild_ci_image_if_needed() {
export FORCE_BUILD_IMAGES="true"
fi
- if [[ ${CHECK_IMAGE_FOR_REBUILD} == "false" ]]; then
+ if [[ ${CHECK_IMAGE_FOR_REBUILD:="true"} == "false" ]]; then
print_info
print_info "Skip checking for rebuilds of the CI image but checking if it needs to be pulled"
print_info
@@ -407,9 +389,9 @@ function rebuild_ci_image_if_needed() {
return
fi
- local needs_docker_build="false"
+ NEEDS_DOCKER_BUILD="false"
check_if_docker_build_is_needed
- if [[ ${needs_docker_build} == "true" ]]; then
+ if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
if [[ ${SKIP_CHECK_REMOTE_IMAGE:=} != "true" && ${DOCKER_CACHE} == "pulled" ]]; then
# Check if remote image is different enough to force pull
# This is an optimisation pull vs. build time. When there
@@ -457,16 +439,17 @@ function rebuild_ci_image_if_needed() {
fi
}
+
# Interactive version of confirming the ci image that is used in pre-commits
# it displays additional information - what the user should do in order to bring the local images
# back to state that pre-commit will be happy with
function rebuild_ci_image_if_needed_and_confirmed() {
- local needs_docker_build="false"
+ NEEDS_DOCKER_BUILD="false"
THE_IMAGE_TYPE="CI"
check_if_docker_build_is_needed
- if [[ ${needs_docker_build} == "true" ]]; then
+ if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
print_info
print_info "Docker image build is needed!"
print_info
@@ -476,7 +459,7 @@ function rebuild_ci_image_if_needed_and_confirmed() {
print_info
fi
- if [[ "${needs_docker_build}" == "true" ]]; then
+ if [[ "${NEEDS_DOCKER_BUILD}" == "true" ]]; then
echo
echo "Some of your images need to be rebuild because important files (like package list) has changed."
echo
@@ -505,13 +488,13 @@ function rebuild_ci_image_if_needed_and_confirmed() {
# selected by Breeze flags or environment variables.
function build_ci_image() {
print_build_info
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "Preparing ${AIRFLOW_CI_IMAGE}.
- " >"${DETECTED_TERMINAL}"
+ " > "${DETECTED_TERMINAL}"
spin "${OUTPUT_LOG}" &
SPIN_PID=$!
# shellcheck disable=SC2064
- add_trap "kill ${SPIN_PID}" INT TERM HUP EXIT
+ trap "kill ${SPIN_PID}" SIGINT SIGTERM
fi
pull_ci_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
@@ -528,29 +511,29 @@ function build_ci_image() {
echo >&2
exit 1
fi
- if [[ -n ${SPIN_PID=} ]]; then
+ if [[ -n ${SPIN_PID:=""} ]]; then
kill -HUP "${SPIN_PID}" || true
wait "${SPIN_PID}" || true
- echo >"${DETECTED_TERMINAL}"
+ echo > "${DETECTED_TERMINAL}"
fi
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "Preparing ${AIRFLOW_CI_IMAGE}.
- " >"${DETECTED_TERMINAL}"
+ " > "${DETECTED_TERMINAL}"
spin "${OUTPUT_LOG}" &
SPIN_PID=$!
# shellcheck disable=SC2064
- add_trap "kill ${SPIN_PID}" EXIT HUP INT TERM
+ trap "kill ${SPIN_PID}" SIGINT SIGTERM
fi
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "
Docker building ${AIRFLOW_CI_IMAGE}.
-" >"${DETECTED_TERMINAL}"
+" > "${DETECTED_TERMINAL}"
fi
set +u
docker build \
--build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
--build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
- --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
+ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
--build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
@@ -565,27 +548,33 @@ Docker building ${AIRFLOW_CI_IMAGE}.
--target "main" \
. -f Dockerfile.ci
set -u
- if [[ -n "${DEFAULT_CI_IMAGE=}" ]]; then
+ if [[ -n "${DEFAULT_CI_IMAGE:=}" ]]; then
docker tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}"
fi
- if [[ -n ${SPIN_PID=} ]]; then
+ if [[ -n ${SPIN_PID:=""} ]]; then
kill -HUP "${SPIN_PID}" || true
wait "${SPIN_PID}" || true
- echo >"${DETECTED_TERMINAL}"
+ echo > "${DETECTED_TERMINAL}"
fi
}
# Prepares all variables needed by the CI build. Depending on the configuration used (python version
# DockerHub user etc. the variables are set so that other functions can use those variables.
function prepare_prod_build() {
- if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
+ get_base_image_version
+ # We use local docker image cache by default for Production images
+ export DOCKER_CACHE=${DOCKER_CACHE:="local"}
+ print_info
+ print_info "Using ${DOCKER_CACHE} cache strategy for the build."
+ print_info
+ if [[ "${INSTALL_AIRFLOW_REFERENCE:=}" != "" ]]; then
# When --install-airflow-reference is used then the image is build from github tag
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
"--build-arg" "AIRFLOW_INSTALL_SOURCES=https://github.com/apache/airflow/archive/${INSTALL_AIRFLOW_REFERENCE}.tar.gz#egg=apache-airflow"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_REFERENCE}"
add_build_args_for_remote_install
- elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
+ elif [[ "${INSTALL_AIRFLOW_VERSION:=}" != "" ]]; then
# When --install-airflow-version is used then the image is build from PIP package
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
"--build-arg" "AIRFLOW_INSTALL_SOURCES=apache-airflow"
@@ -598,6 +587,13 @@ function prepare_prod_build() {
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
)
fi
+
+ export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
+ export AIRFLOW_PROD_BUILD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-build"
+ export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}"
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
+ export AIRFLOW_PROD_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}"
+ export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then
export DEFAULT_CI_IMAGE="${AIRFLOW_PROD_IMAGE_DEFAULT}"
else
@@ -605,16 +601,15 @@ function prepare_prod_build() {
fi
export THE_IMAGE_TYPE="PROD"
export IMAGE_DESCRIPTION="Airflow production"
-
- # Those constants depend on the type of image run so they are only made constants here
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_PROD_EXTRAS}"}"
- readonly AIRFLOW_EXTRAS
-
+ export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
+ export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
+ export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}"
+ export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
- readonly AIRFLOW_IMAGE
if [[ ${USE_GITHUB_REGISTRY="false"} == "true" ]]; then
- if [[ -n ${GITHUB_TOKEN=} ]]; then
+ if [[ ${GITHUB_TOKEN:=} != "" ]]; then
echo "${GITHUB_TOKEN}" | docker login \
--username "${GITHUB_USERNAME}" \
--password-stdin \
@@ -638,13 +633,14 @@ function prepare_prod_build() {
function build_prod_images() {
print_build_info
- if [[ ${SKIP_BUILDING_PROD_IMAGE} == "true" ]]; then
+ if [[ ${SKIP_BUILDING_PROD_IMAGE:="false"} == "true" ]]; then
print_info
print_info "Skip building production image. Assume the one we have is good!"
print_info
return
fi
+
pull_prod_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then
@@ -662,9 +658,9 @@ function build_prod_images() {
"--cache-from" "${AIRFLOW_PROD_BUILD_IMAGE}"
)
else
- echo >&2
- echo >&2 "Error - thee ${DOCKER_CACHE} cache is unknown!"
- echo >&2
+ >&2 echo
+ >&2 echo "Error - thee ${DOCKER_CACHE} cache is unknown!"
+ >&2 echo
exit 1
fi
set +u
@@ -712,7 +708,7 @@ function build_prod_images() {
# $1 - image name to wait for
# $2 - suffix of the image to wait for
# $3, $4, ... - target tags to tag the image with
-function wait_for_image_tag() {
+function wait_for_image_tag {
IMAGE_NAME="${1}"
IMAGE_SUFFIX=${2}
shift 2
@@ -723,7 +719,7 @@ function wait_for_image_tag() {
echo
while true; do
docker pull "${IMAGE_TO_WAIT_FOR}" || true
- if [[ -z "$(docker images -q "${IMAGE_TO_WAIT_FOR}" 2>/dev/null || true)" ]]; then
+ if [[ "$(docker images -q "${IMAGE_TO_WAIT_FOR}" 2> /dev/null)" == "" ]]; then
echo
echo "The image ${IMAGE_TO_WAIT_FOR} is not yet available. Waiting"
echo
@@ -735,30 +731,14 @@ function wait_for_image_tag() {
echo
echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${IMAGE_NAME}."
echo
- docker tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}"
+ docker tag "${IMAGE_TO_WAIT_FOR}" "${IMAGE_NAME}"
for TARGET_TAG in "${@}"; do
echo
echo "Tagging ${IMAGE_TO_WAIT_FOR} as ${TARGET_TAG}."
echo
- docker tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}"
+ docker tag "${IMAGE_TO_WAIT_FOR}" "${TARGET_TAG}"
done
break
fi
done
}
-
-# We use pulled docker image cache by default for CI images to speed up the builds
-# and local to speed up iteration on kerberos tests
-function determine_docker_cache_strategy() {
- if [[ -z "${DOCKER_CACHE=}" ]]; then
- if [[ "${PRODUCTION_IMAGE}" == "true" ]]; then
- export DOCKER_CACHE="local"
- else
- export DOCKER_CACHE="pulled"
- fi
- fi
- readonly DOCKER_CACHE
- print_info
- print_info "Using ${DOCKER_CACHE} cache strategy for the build."
- print_info
-}
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 6e6bc95..e2d4d97 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -17,50 +17,26 @@
# under the License.
# Needs to be declared outside function in MacOS
-# shellcheck disable=SC2034
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS=()
-# Creates directories for Breeze
-function create_directories() {
+# Very basic variables that MUST be set
+function initialize_base_variables() {
# This folder is mounted to inside the container in /files folder. This is the way how
# We can exchange DAGs, scripts, packages etc with the container environment
export FILES_DIR="${AIRFLOW_SOURCES}/files"
- readonly FILES_DIR
-
- # Directory where all the build cache is stored - we keep there status of all the docker images
- # As well as hashes of the important files, but also we generate build scripts there that are
- # Used to execute the commands for breeze
- export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
- export BUILD_CACHE_DIR
- readonly BUILD_CACHE_DIR
-
- # Create those folders above in case they do not exist
- mkdir -p "${BUILD_CACHE_DIR}" >/dev/null
- mkdir -p "${FILES_DIR}" >/dev/null
+ # Temporary dir used well ... temporarily
+ export TMP_DIR="${AIRFLOW_SOURCES}/tmp"
# By default we are not in CI environment GitHub Actions sets CI to "true"
export CI="${CI="false"}"
# Create useful directories if not yet created
+ mkdir -p "${TMP_DIR}"
+ mkdir -p "${FILES_DIR}"
mkdir -p "${AIRFLOW_SOURCES}/.mypy_cache"
mkdir -p "${AIRFLOW_SOURCES}/logs"
mkdir -p "${AIRFLOW_SOURCES}/dist"
- CACHE_TMP_FILE_DIR=$(mktemp -d)
- export CACHE_TMP_FILE_DIR
- readonly CACHE_TMP_FILE_DIR
-
- if [[ ${SKIP_CACHE_DELETION=} != "true" ]]; then
- add_trap "rm -rf -- '${CACHE_TMP_FILE_DIR}'" EXIT HUP INT TERM
- fi
-
- OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log"
- export OUTPUT_LOG
- readonly OUTPUT_LOG
-}
-
-# Very basic variables that MUST be set
-function initialize_base_variables() {
# Default port numbers for forwarded ports
export WEBSERVER_HOST_PORT=${WEBSERVER_HOST_PORT:="28080"}
export POSTGRES_HOST_PORT=${POSTGRES_HOST_PORT:="25433"}
@@ -77,13 +53,17 @@ function initialize_base_variables() {
# Python version and avoids problems with root-owned .pyc files in host
export PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE:="true"}
- # By default we build CI images but we can switch to production image with PRODUCTION_IMAGE="true"
- export PRODUCTION_IMAGE="false"
+ # Sets to the build cache directory - status of build and convenience scripts are stored there
+ BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
+ export BUILD_CACHE_DIR
# Currently supported major/minor versions of python
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS+=("3.6" "3.7" "3.8")
export CURRENT_PYTHON_MAJOR_MINOR_VERSIONS
+ # default python Major/Minor version
+ PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:="3.6"}
+
# If set to true, the database will be reset at entry. Works for Postgres and MySQL
export DB_RESET=${DB_RESET:="false"}
@@ -91,12 +71,8 @@ function initialize_base_variables() {
AIRFLOW_VERSION=$(grep version "${AIRFLOW_SOURCES}/airflow/version.py" | awk '{print $3}' | sed "s/['+]//g")
export AIRFLOW_VERSION
- # Whether credentials should be forwarded to inside the docker container
- export FORWARD_CREDENTIALS=${FORWARD_CREDENTIALS:="false"}
-
- # If no Airflow Home defined - fallback to ${HOME}/airflow
- AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow}
- export AIRFLOW_HOME_DIR
+ # default version of python used to tag the "master" and "latest" images in DockerHub
+ export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6
}
# Determine current branch
@@ -104,7 +80,6 @@ function initialize_branch_variables() {
# Default branch used - this will be different in different branches
export DEFAULT_BRANCH="master"
export DEFAULT_CONSTRAINTS_BRANCH="constraints-master"
- readonly DEFAULT_CONSTRAINTS_BRANCH
# Default branch name for triggered builds is the one configured in default branch
# We need to read it here as it comes from _common_values.sh
@@ -256,7 +231,6 @@ function initialize_image_build_variables() {
# Default extras used for building CI image
export DEFAULT_CI_EXTRAS="devel_ci"
- # Default build id
export CI_BUILD_ID="${CI_BUILD_ID:="0"}"
# Default extras used for building Production image. The master of this information is in the Dockerfile
@@ -277,24 +251,13 @@ function initialize_image_build_variables() {
# Checks if the image should be rebuilt
export CHECK_IMAGE_FOR_REBUILD="${CHECK_IMAGE_FOR_REBUILD:="true"}"
- # Skips building production images altogether (assume they are already built)
- export SKIP_BUILDING_PROD_IMAGE="${SKIP_BUILDING_PROD_IMAGE:="false"}"
-
- # Additional airflow extras on top of the default ones
- export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
- # Additional python dependencies on top of the default ones
- export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
- # additional development apt dependencies on top of the default ones
- export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}"
- # additional runtime apt dependencies on top of the default ones
- export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}"
-
}
# Determine version suffixes used to build backport packages
function initialize_version_suffixes_for_package_building() {
# Version suffix for PyPI packaging
export VERSION_SUFFIX_FOR_PYPI=""
+
# Artifact name suffix for SVN packaging
export VERSION_SUFFIX_FOR_SVN=""
}
@@ -303,63 +266,46 @@ function initialize_version_suffixes_for_package_building() {
function initialize_kubernetes_variables() {
# By default we assume the kubernetes cluster is not being started
export ENABLE_KIND_CLUSTER=${ENABLE_KIND_CLUSTER:="false"}
+
# Default Kubernetes version
export DEFAULT_KUBERNETES_VERSION="v1.18.6"
- readonly DEFAULT_KUBERNETES_VERSION
+
# Default KinD version
export DEFAULT_KIND_VERSION="v0.8.0"
- readonly DEFAULT_KIND_VERSION
+
# Default Helm version
export DEFAULT_HELM_VERSION="v3.2.4"
- readonly DEFAULT_HELM_VERSION
+
# Namespace where airflow is installed via helm
export HELM_AIRFLOW_NAMESPACE="airflow"
- # Kind version
- export KIND_VERSION=${KIND_VERSION:=${DEFAULT_KIND_VERSION}}
- # Helm version
- export HELM_VERSION=${HELM_VERSION:=${DEFAULT_HELM_VERSION}}
- # Kubectl version
- export KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}}
- # Local Kind path
- export KIND_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kind"
- readonly KIND_BINARY_PATH
- # Local Helm path
- export HELM_BINARY_PATH="${BUILD_CACHE_DIR}/bin/helm"
- readonly HELM_BINARY_PATH
- # local Kubectl path
- export KUBECTL_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kubectl"
- readonly KUBECTL_BINARY_PATH
+
+ COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")"
+ export COMMIT_SHA
+
+ export CI_BUILD_ID="0"
}
function initialize_git_variables() {
# SHA of the commit for the current sources
- COMMIT_SHA="$(git rev-parse HEAD 2>/dev/null || echo "Unknown")"
+ COMMIT_SHA="$(git rev-parse HEAD || echo "Unknown")"
export COMMIT_SHA
}
function initialize_github_variables() {
# Defaults for interacting with GitHub
- export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
-
+ export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+ GITHUB_REPOSITORY_LOWERCASE="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
+ export GITHUB_REPOSITORY_LOWERCASE
export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"}
+ export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
- export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
- GITHUB_REPOSITORY_LOWERCASE="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
- export GITHUB_REPOSITORY_LOWERCASE
-
- # Used only in CI environment
- export GITHUB_TOKEN="${GITHUB_TOKEN=""}"
- export GITHUB_USERNAME="${GITHUB_USERNAME=""}"
-
-
}
# Common environment that is initialized by both Breeze and CI scripts
function initialize_common_environment() {
- create_directories
initialize_base_variables
initialize_branch_variables
initialize_available_integrations
@@ -375,15 +321,6 @@ function initialize_common_environment() {
initialize_github_variables
}
-function set_default_python_version_if_empty() {
- # default version of python used to tag the "master" and "latest" images in DockerHub
- export DEFAULT_PYTHON_MAJOR_MINOR_VERSION=3.6
-
- # default python Major/Minor version
- export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}}
-
-}
-
function summarize_ci_environment() {
cat <<EOF
@@ -493,7 +430,7 @@ function get_environment_for_builds_on_ci() {
export CI_SOURCE_BRANCH="${GITHUB_HEAD_REF}"
BRANCH_EXISTS=$(git ls-remote --heads \
"https://github.com/${CI_SOURCE_REPO}.git" "${CI_SOURCE_BRANCH}" || true)
- if [[ -z ${BRANCH_EXISTS=} ]]; then
+ if [[ ${BRANCH_EXISTS} == "" ]]; then
print_info
print_info "https://github.com/${CI_SOURCE_REPO}.git Branch ${CI_SOURCE_BRANCH} does not exist"
print_info
@@ -510,114 +447,32 @@ function get_environment_for_builds_on_ci() {
fi
else
# CI PR settings
- export CI_TARGET_REPO="${CI_TARGET_REPO="apache/airflow"}"
- export CI_TARGET_BRANCH="${DEFAULT_BRANCH="master"}"
- export CI_BUILD_ID="${CI_BUILD_ID="0"}"
- export CI_JOB_ID="${CI_JOB_ID="0"}"
- export CI_EVENT_TYPE="${CI_EVENT_TYPE="pull_request"}"
- export CI_REF="${CI_REF="refs/head/master"}"
-
- export CI_SOURCE_REPO="${CI_SOURCE_REPO="apache/airflow"}"
- export CI_SOURCE_BRANCH="${DEFAULT_BRANCH="master"}"
+ export CI_TARGET_REPO="${CI_TARGET_REPO:="apache/airflow"}"
+ export CI_TARGET_BRANCH="${DEFAULT_BRANCH:="master"}"
+ export CI_BUILD_ID="${CI_BUILD_ID:="0"}"
+ export CI_JOB_ID="${CI_JOB_ID:="0"}"
+ export CI_EVENT_TYPE="${CI_EVENT_TYPE:="pull_request"}"
+ export CI_REF="${CI_REF:="refs/head/master"}"
+
+ export CI_SOURCE_REPO="${CI_SOURCE_REPO:="apache/airflow"}"
+ export CI_SOURCE_BRANCH="${DEFAULT_BRANCH:="master"}"
+
+ # Github Registry settings (in Github Actions they come from .github/workflow/ci.yml)
+ export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="true"}
+ export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"}
+ export GITHUB_REPOSITORY="${GITHUB_REPOSITORY:="apache/airflow"}"
+ export GITHUB_USERNAME="${GITHUB_USERNAME=""}"
+ export GITHUB_TOKEN="${GITHUB_TOKEN=""}"
+ export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
+ export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
+ export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
+
+ # DockerHub settings
+ export DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
+ export DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
fi
if [[ ${VERBOSE} == "true" && ${PRINT_INFO_FROM_SCRIPTS} == "true" ]]; then
summarize_ci_environment
fi
}
-
-# shellcheck disable=SC2034
-
-# By the time this method is run, nearly all constants have been already set to the final values
-# so we can set them as readonly.
-function make_constants_read_only() {
- # Set the arguments as read-only
- readonly PYTHON_MAJOR_MINOR_VERSION
-
- readonly WEBSERVER_HOST_PORT
- readonly POSTGRES_HOST_PORT
- readonly MYSQL_HOST_PORT
-
-
- readonly HOST_USER_ID
- readonly HOST_GROUP_ID
- readonly HOST_AIRFLOW_SOURCES
- readonly HOST_HOME
- readonly HOST_OS
-
- readonly INTEGRATIONS
-
- readonly ENABLE_KIND_CLUSTER
- readonly KUBERNETES_MODE
- readonly KUBERNETES_VERSION
- readonly KIND_VERSION
- readonly HELM_VERSION
- readonly KUBECTL_VERSION
-
- readonly BACKEND
- readonly POSTGRES_VERSION
- readonly MYSQL_VERSION
-
- readonly MOUNT_LOCAL_SOURCES
-
- readonly INSTALL_AIRFLOW_VERSION
- readonly INSTALL_AIRFLOW_REFERENCE
-
- readonly DB_RESET
- readonly VERBOSE
-
- readonly PRODUCTION_IMAGE
-
- # The FORCE_* variables are missing here because they are not constant - they are just exported variables.
- # Their value might change during the script execution - for example when during the
- # pre-commit the answer is "no", we set the FORCE_ANSWER_TO_QUESTIONS to "no"
- # for all subsequent questions. Also in CI environment we first force pulling and building
- # the images but then we disable it so that in subsequent steps the image is reused.
- # similarly CHECK_IMAGE_FOR_REBUILD variable.
-
- readonly SKIP_BUILDING_PROD_IMAGE
- readonly CI_BUILD_ID
- readonly CI_JOB_ID
-
- # AIRFLOW_EXTRAS are made readonly by the time the image is built (either PROD or CI)
- readonly ADDITIONAL_AIRFLOW_EXTRAS
- readonly ADDITIONAL_PYTHON_DEPS
- readonly ADDITIONAL_DEV_DEPS
- readonly ADDITIONAL_RUNTIME_DEPS
-
- readonly DOCKERHUB_USER
- readonly DOCKERHUB_REPO
- readonly DOCKER_CACHE
-
- readonly USE_GITHUB_REGISTRY
- readonly GITHUB_REGISTRY
- readonly GITHUB_REGISTRY_WAIT_FOR_IMAGE
- readonly GITHUB_REGISTRY_PULL_IMAGE_TAG
- readonly GITHUB_REGISTRY_PUSH_IMAGE_TAG
-
- readonly GITHUB_REPOSITORY
- readonly GITHUB_REPOSITORY_LOWERCASE
- readonly GITHUB_TOKEN
- readonly GITHUB_USERNAME
-
-
- readonly FORWARD_CREDENTIALS
- readonly USE_GITHUB_REGISTRY
-
- readonly EXTRA_STATIC_CHECK_OPTIONS
-
- readonly VERSION_SUFFIX_FOR_PYPI
- readonly VERSION_SUFFIX_FOR_SVN
-
- readonly PYTHON_BASE_IMAGE_VERSION
- readonly PYTHON_BASE_IMAGE
- readonly AIRFLOW_CI_BASE_TAG
- readonly AIRFLOW_CI_IMAGE
- readonly AIRFLOW_CI_IMAGE_DEFAULT
- readonly AIRFLOW_PROD_BASE_TAG
- readonly AIRFLOW_PROD_IMAGE
- readonly AIRFLOW_PROD_BUILD_IMAGE
- readonly AIRFLOW_PROD_IMAGE_KUBERNETES
- readonly AIRFLOW_PROD_IMAGE_DEFAULT
- readonly BUILT_CI_IMAGE_FLAG_FILE
-}
diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh
index aeb540a..f48902a 100644
--- a/scripts/ci/libraries/_kind.sh
+++ b/scripts/ci/libraries/_kind.sh
@@ -17,13 +17,11 @@
# under the License.
-function get_kind_cluster_name(){
+function initialize_kind_variables(){
# Name of the KinD cluster to connect to
export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"}
- readonly KIND_CLUSTER_NAME
# Name of the KinD cluster to connect to when referred to via kubectl
export KUBECTL_CLUSTER_NAME=kind-${KIND_CLUSTER_NAME}
- readonly KUBECTL_CLUSTER_NAME
}
function dump_kind_logs() {
@@ -34,56 +32,59 @@ function dump_kind_logs() {
echo "EXIT_CODE is ${EXIT_CODE:=}"
local DUMP_DIR_NAME DUMP_DIR
- DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}
+ DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}
DUMP_DIR="/tmp/${DUMP_DIR_NAME}"
kind --name "${KIND_CLUSTER_NAME}" export logs "${DUMP_DIR}"
}
function make_sure_kubernetes_tools_are_installed() {
SYSTEM=$(uname -s| tr '[:upper:]' '[:lower:]')
-
+ KIND_VERSION=${KIND_VERSION:=${DEFAULT_KIND_VERSION}}
KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${SYSTEM}-amd64"
+ KIND_PATH="${BUILD_CACHE_DIR}/bin/kind"
+ HELM_VERSION=${HELM_VERSION:=${DEFAULT_HELM_VERSION}}
+ HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz"
+ HELM_PATH="${BUILD_CACHE_DIR}/bin/helm"
+ KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}}
+ KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${SYSTEM}/amd64/kubectl"
+ KUBECTL_PATH="${BUILD_CACHE_DIR}/bin/kubectl"
mkdir -pv "${BUILD_CACHE_DIR}/bin"
- if [[ -f "${KIND_BINARY_PATH}" ]]; then
- DOWNLOADED_KIND_VERSION=v"$(${KIND_BINARY_PATH} --version | awk '{ print $3 }')"
+ if [[ -f "${KIND_PATH}" ]]; then
+ DOWNLOADED_KIND_VERSION=v"$(${KIND_PATH} --version | awk '{ print $3 }')"
echo "Currently downloaded kind version = ${DOWNLOADED_KIND_VERSION}"
fi
- if [[ ! -f "${KIND_BINARY_PATH}" || ${DOWNLOADED_KIND_VERSION} != "${KIND_VERSION}" ]]; then
+ if [[ ! -f "${KIND_PATH}" || ${DOWNLOADED_KIND_VERSION} != "${KIND_VERSION}" ]]; then
echo
echo "Downloading Kind version ${KIND_VERSION}"
- curl --fail --location "${KIND_URL}" --output "${KIND_BINARY_PATH}"
- chmod a+x "${KIND_BINARY_PATH}"
+ curl --fail --location "${KIND_URL}" --output "${KIND_PATH}"
+ chmod a+x "${KIND_PATH}"
else
echo "Kind version ok"
echo
fi
-
- KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${SYSTEM}/amd64/kubectl"
- if [[ -f "${KUBECTL_BINARY_PATH}" ]]; then
- DOWNLOADED_KUBECTL_VERSION="$(${KUBECTL_BINARY_PATH} version --client=true --short | awk '{ print $3 }')"
+ if [[ -f "${KUBECTL_PATH}" ]]; then
+ DOWNLOADED_KUBECTL_VERSION="$(${KUBECTL_PATH} version --client=true --short | awk '{ print $3 }')"
echo "Currently downloaded kubectl version = ${DOWNLOADED_KUBECTL_VERSION}"
fi
- if [[ ! -f "${KUBECTL_BINARY_PATH}" || ${DOWNLOADED_KUBECTL_VERSION} != "${KUBECTL_VERSION}" ]]; then
+ if [[ ! -f "${KUBECTL_PATH}" || ${DOWNLOADED_KUBECTL_VERSION} != "${KUBECTL_VERSION}" ]]; then
echo
echo "Downloading Kubectl version ${KUBECTL_VERSION}"
- curl --fail --location "${KUBECTL_URL}" --output "${KUBECTL_BINARY_PATH}"
- chmod a+x "${KUBECTL_BINARY_PATH}"
+ curl --fail --location "${KUBECTL_URL}" --output "${KUBECTL_PATH}"
+ chmod a+x "${KUBECTL_PATH}"
else
echo "Kubectl version ok"
echo
fi
-
- HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz"
- if [[ -f "${HELM_BINARY_PATH}" ]]; then
- DOWNLOADED_HELM_VERSION="$(${HELM_BINARY_PATH} version --template '{{.Version}}')"
+ if [[ -f "${HELM_PATH}" ]]; then
+ DOWNLOADED_HELM_VERSION="$(${HELM_PATH} version --template '{{.Version}}')"
echo "Currently downloaded helm version = ${DOWNLOADED_HELM_VERSION}"
fi
- if [[ ! -f "${HELM_BINARY_PATH}" || ${DOWNLOADED_HELM_VERSION} != "${HELM_VERSION}" ]]; then
+ if [[ ! -f "${HELM_PATH}" || ${DOWNLOADED_HELM_VERSION} != "${HELM_VERSION}" ]]; then
echo
echo "Downloading Helm version ${HELM_VERSION}"
curl --location "${HELM_URL}" |
- tar -xvz -O "${SYSTEM}-amd64/helm" >"${HELM_BINARY_PATH}"
- chmod a+x "${HELM_BINARY_PATH}"
+ tar -xvz -O "${SYSTEM}-amd64/helm" >"${HELM_PATH}"
+ chmod a+x "${HELM_PATH}"
else
echo "Helm version ok"
echo
@@ -127,7 +128,7 @@ function perform_kind_cluster_operation() {
ALLOWED_KIND_OPERATIONS="[ start restart stop deploy test shell recreate ]"
set +u
- if [[ -z "${1=}" ]]; then
+ if [[ -z "${1}" ]]; then
echo >&2
echo >&2 "Operation must be provided as first parameter. One of: ${ALLOWED_KIND_OPERATIONS}"
echo >&2
@@ -178,6 +179,11 @@ function perform_kind_cluster_operation() {
echo
echo "Deploying Airflow to KinD"
echo
+ get_environment_for_builds_on_ci
+ make_sure_kubernetes_tools_are_installed
+ initialize_kind_variables
+ prepare_prod_build
+ build_prod_images
build_image_for_kubernetes_tests
load_image_to_kind_cluster
deploy_airflow_with_helm
diff --git a/scripts/ci/libraries/_md5sum.sh b/scripts/ci/libraries/_md5sum.sh
index 0ef2d79..c2e1ff3 100644
--- a/scripts/ci/libraries/_md5sum.sh
+++ b/scripts/ci/libraries/_md5sum.sh
@@ -79,7 +79,7 @@ function update_all_md5_files() {
move_file_md5sum "${AIRFLOW_SOURCES}/${FILE}"
done
mkdir -pv "${BUILD_CACHE_DIR}/${BRANCH_NAME}"
- touch "${BUILT_CI_IMAGE_FLAG_FILE}"
+ touch "${BUILT_IMAGE_FLAG_FILE}"
}
@@ -121,13 +121,13 @@ function check_if_docker_build_is_needed() {
if [[ ${FORCE_BUILD_IMAGES:=""} == "true" ]]; then
print_info "Docker image build is forced for ${THE_IMAGE_TYPE} image"
calculate_md5sum_for_all_files
- needs_docker_build="true"
+ export NEEDS_DOCKER_BUILD="true"
else
calculate_md5sum_for_all_files
if [[ ${FILES_MODIFIED} == "true" ]]; then
- needs_docker_build="true"
+ export NEEDS_DOCKER_BUILD="true"
fi
- if [[ ${needs_docker_build} == "true" ]]; then
+ if [[ ${NEEDS_DOCKER_BUILD} == "true" ]]; then
print_info "Docker image build is needed for ${THE_IMAGE_TYPE} image!"
else
print_info "Docker image build is not needed for ${THE_IMAGE_TYPE} image!"
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 6e8b1a1..ad13cb2 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -26,7 +26,7 @@ function pull_image_if_not_present_or_forced() {
IMAGE_HASH=$(docker images -q "${IMAGE_TO_PULL}" 2> /dev/null || true)
local PULL_IMAGE=${FORCE_PULL_IMAGES}
- if [[ -z "${IMAGE_HASH=}" ]]; then
+ if [[ "${IMAGE_HASH}" == "" ]]; then
PULL_IMAGE="true"
fi
if [[ "${PULL_IMAGE}" == "true" ]]; then
@@ -62,13 +62,15 @@ function pull_image_github_dockerhub() {
# Pulls CI image in case caching strategy is "pulled" and the image needs to be pulled
function pull_ci_images_if_needed() {
+ # Whether to force pull images to populate cache
+ export FORCE_PULL_IMAGES=${FORCE_PULL_IMAGES:="false"}
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
echo
echo "Force pull base image ${PYTHON_BASE_IMAGE}"
echo
- if [[ -n ${DETECTED_TERMINAL=} ]]; then
+ if [[ -n ${DETECTED_TERMINAL:=""} ]]; then
echo -n "
Docker pulling ${PYTHON_BASE_IMAGE}.
" > "${DETECTED_TERMINAL}"
@@ -95,6 +97,9 @@ Docker pulling ${PYTHON_BASE_IMAGE}.
# Pulls PROD image in case caching strategy is "pulled" and the image needs to be pulled
function pull_prod_images_if_needed() {
+ # Whether to force pull images to populate cache
+ export FORCE_PULL_IMAGES=${FORCE_PULL_IMAGES:="false"}
+
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
if [[ "${FORCE_PULL_IMAGES}" == "true" ]]; then
echo
@@ -128,7 +133,7 @@ function push_ci_images_to_dockerhub() {
docker push "${AIRFLOW_CI_IMAGE}"
docker tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
docker push "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
- if [[ -n ${DEFAULT_CI_IMAGE=} ]]; then
+ if [[ -n ${DEFAULT_CI_IMAGE:=""} ]]; then
# Only push default image to DockerHub registry if it is defined
docker push "${DEFAULT_CI_IMAGE}"
fi
@@ -143,7 +148,7 @@ function push_ci_images_to_github() {
AIRFLOW_CI_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}"
docker push "${AIRFLOW_CI_TAGGED_IMAGE}"
- if [[ -n ${GITHUB_SHA=} ]]; then
+ if [[ ${GITHUB_SHA:=} != "" ]]; then
# Also push image to GitHub registry with commit SHA
AIRFLOW_CI_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${COMMIT_SHA}"
docker tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_SHA_IMAGE}"
@@ -171,7 +176,7 @@ function push_ci_images() {
function push_prod_images_to_dockerhub () {
# Prod image
docker push "${AIRFLOW_PROD_IMAGE}"
- if [[ -n ${DEFAULT_PROD_IMAGE=} ]]; then
+ if [[ -n ${DEFAULT_PROD_IMAGE:=""} ]]; then
docker push "${DEFAULT_PROD_IMAGE}"
fi
# Prod build image
@@ -189,7 +194,7 @@ function push_prod_images_to_github () {
AIRFLOW_PROD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}"
docker push "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
- if [[ -n ${COMMIT_SHA=} ]]; then
+ if [[ ${COMMIT_SHA:=} != "" ]]; then
# Also push image to GitHub registry with commit SHA
AIRFLOW_PROD_SHA_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}"
docker tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_SHA_IMAGE}"
@@ -237,19 +242,23 @@ function wait_for_github_registry_image() {
GITHUB_API_CALL="${GITHUB_API_ENDPOINT}/${IMAGE_NAME}/manifests/${IMAGE_TAG}"
while true; do
- curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" 2>/dev/null > "${OUTPUT_LOG}"
- local digest
- digest=$(jq '.config.digest' < "${OUTPUT_LOG}")
- echo -n "."
- if [[ ${digest} != "null" ]]; then
- echo -e " \e[32mOK.\e[0m"
+ echo
+ echo "Calling ${GITHUB_API_CALL} with ${GITHUB_USERNAME}"
+ echo
+ curl -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}" > "${OUTPUT_LOG}"
+ echo
+ DIGEST=$(jq '.config.digest' < "${OUTPUT_LOG}")
+ if [[ ${DIGEST} != "null" ]]; then
+ echo
+ echo "Found digest: '${DIGEST}'"
+ echo
break
fi
+ jq -C '.' < "${OUTPUT_LOG}"
sleep 10
done
echo
- echo
echo "Found ${IMAGE_NAME}:${IMAGE_TAG} image"
- echo "Digest: '${digest}'"
+ echo "Digest: '${DIGEST}'"
echo
}
diff --git a/scripts/ci/libraries/_sanity_checks.sh b/scripts/ci/libraries/_sanity_checks.sh
index b250cf2..d2bc01e 100644
--- a/scripts/ci/libraries/_sanity_checks.sh
+++ b/scripts/ci/libraries/_sanity_checks.sh
@@ -49,6 +49,25 @@ function sanitize_mounted_files() {
#
# Most useful is out.log file in this directory storing verbose output of the scripts.
#
+function create_cache_directory() {
+ CACHE_TMP_FILE_DIR=$(mktemp -d)
+ export CACHE_TMP_FILE_DIR
+
+ if [[ ${SKIP_CACHE_DELETION:=} != "true" ]]; then
+ trap 'rm -rf -- "${CACHE_TMP_FILE_DIR}"' INT TERM HUP
+ fi
+
+ OUTPUT_LOG="${CACHE_TMP_FILE_DIR}/out.log"
+ export OUTPUT_LOG
+}
+
+# Removes the cache temporary directory
+function remove_cache_directory() {
+ if [[ -d ${CACHE_TMP_FILE_DIR} ]]; then
+ rm -rf "${CACHE_TMP_FILE_DIR}"
+ fi
+}
+
#
# Checks if core utils required in the host system are installed and explain what needs to be done if not
#
@@ -71,8 +90,6 @@ function check_if_coreutils_installed() {
set -e
CMDNAME="$(basename -- "$0")"
- export CMDNAME
- readonly CMDNAME
#################### Parsing options/arguments
if [[ ${GETOPT_RETVAL} != 4 || "${STAT_PRESENT}" != "0" || "${MD5SUM_PRESENT}" != "0" ]]; then
@@ -122,7 +139,7 @@ function assert_not_in_container() {
echo >&2 "You are inside the Airflow docker container!"
echo >&2 "You should only run this script from the host."
echo >&2 "Learn more about how we develop and test airflow in:"
- echo >&2 "https://github.com/apache/airflow/blob/master/TESTING.rst"
+ echo >&2 "https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst"
echo >&2
exit 1
fi
@@ -142,8 +159,8 @@ function go_to_airflow_sources {
#
function basic_sanity_checks() {
assert_not_in_container
- set_default_python_version_if_empty
go_to_airflow_sources
check_if_coreutils_installed
+ create_cache_directory
sanitize_mounted_files
}
diff --git a/scripts/ci/libraries/_script_init.sh b/scripts/ci/libraries/_script_init.sh
index 4e11e27..804037e 100755
--- a/scripts/ci/libraries/_script_init.sh
+++ b/scripts/ci/libraries/_script_init.sh
@@ -18,11 +18,18 @@
set -euo pipefail
-export AIRFLOW_SOURCES="${AIRFLOW_SOURCES:=$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../.." && pwd )}"
-readonly AIRFLOW_SOURCES
+_CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+SCRIPTS_CI_DIR="$(cd "${_CURRENT_DIR}"/.. && pwd)"
+export SCRIPTS_CI_DIR
+
+# Sets to where airflow sources are located
+AIRFLOW_SOURCES=${AIRFLOW_SOURCES:=$(cd "${SCRIPTS_CI_DIR}/../../" && pwd)}
+export AIRFLOW_SOURCES
+
# shellcheck source=scripts/ci/libraries/_all_libs.sh
-. "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"
+. "${SCRIPTS_CI_DIR}"/libraries/_all_libs.sh
initialize_common_environment
@@ -30,12 +37,4 @@ basic_sanity_checks
script_start
-determine_docker_cache_strategy
-
-get_environment_for_builds_on_ci
-
-get_docker_image_names
-
-make_constants_read_only
-
-add_trap script_end EXIT HUP INT TERM
+trap script_end EXIT
diff --git a/scripts/ci/libraries/_start_end.sh b/scripts/ci/libraries/_start_end.sh
index 635ad88..4dcb150 100644
--- a/scripts/ci/libraries/_start_end.sh
+++ b/scripts/ci/libraries/_start_end.sh
@@ -79,4 +79,5 @@ function script_end {
print_info "Exit code ${EXIT_CODE}"
print_info
fi
+ remove_cache_directory
}
diff --git a/scripts/ci/libraries/_traps.sh b/scripts/ci/libraries/_traps.sh
deleted file mode 100644
index be2e31b..0000000
--- a/scripts/ci/libraries/_traps.sh
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#######################################################################################################
-#
-# Adds trap to the traps already set.
-#
-# Arguments:
-# trap to set
-# .... list of signals to handle
-#######################################################################################################
-function add_trap() {
- trap="${1}"
- shift
- for signal in "${@}"
- do
- # adding trap to exiting trap
- local handlers
- handlers="$( trap -p "${signal}" | cut -f2 -d \' )"
- # shellcheck disable=SC2064
- trap "${handlers}${handlers:+;}${trap}" "${signal}"
- done
-}
diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh
index aa4f096..77d2fa1 100644
--- a/scripts/ci/libraries/_verbosity.sh
+++ b/scripts/ci/libraries/_verbosity.sh
@@ -24,13 +24,12 @@ function check_verbose_setup {
fi
}
-DOCKER_BINARY_PATH="${DOCKER_BINARY_PATH:=$(command -v docker || echo "/bin/docker")}"
-export DOCKER_BINARY_PATH
+DOCKER_BINARY="${DOCKER_BINARY:=$(command -v docker || echo "/bin/docker")}"
+export DOCKER_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all docker commands run will be
# printed before execution
function docker {
- set +e
if [[ ${VERBOSE:="false"} == "true" && \
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
${VERBOSE_COMMANDS:=} != "true" && \
@@ -39,62 +38,55 @@ function docker {
>&2 echo "docker" "${@}"
fi
if [[ ${PRINT_INFO_FROM_SCRIPTS} == "false" ]]; then
- ${DOCKER_BINARY_PATH} "${@}" >>"${OUTPUT_LOG}" 2>&1
+ ${DOCKER_BINARY} "${@}" >>"${OUTPUT_LOG}" 2>&1
else
- ${DOCKER_BINARY_PATH} "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
+ ${DOCKER_BINARY} "${@}" 2>&1 | tee -a "${OUTPUT_LOG}"
fi
- res="$?"
- if [[ ${res} == "0" ]]; then
+ EXIT_CODE="$?"
+ if [[ ${EXIT_CODE} == "0" ]]; then
# No matter if "set -e" is used the log will be removed on success.
# This way in the output log we only see the most recent failed command and what was echoed before
rm -f "${OUTPUT_LOG}"
fi
- set -e
- return ${res}
+ return "${EXIT_CODE}"
}
-HELM_BINARY_PATH="${HELM_BINARY_PATH:=$(command -v helm || echo "/bin/helm")}"
-export HELM_BINARY_PATH
+HELM_BINARY="${HELM_BINARY:=$(command -v helm || echo "/bin/helm")}"
+export HELM_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all helm commands run will be
# printed before execution
function helm {
- set +e
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "helm" "${@}"
fi
- ${HELM_BINARY_PATH} "${@}" | tee -a "${OUTPUT_LOG}"
- local res="$?"
- if [[ ${res} == "0" ]]; then
+ ${HELM_BINARY} "${@}" | tee -a "${OUTPUT_LOG}"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
rm -f "${OUTPUT_LOG}"
fi
- set -e
- return ${res}
}
-KUBECTL_BINARY_PATH=${KUBECTL_BINARY_PATH:=$(command -v kubectl || echo "/bin/kubectl")}
-export KUBECTL_BINARY_PATH
+KUBECTL_BINARY=${KUBECTL_BINARY:=$(command -v kubectl || echo "/bin/kubectl")}
+export KUBECTL_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kubectl commands run will be
# printed before execution
function kubectl {
- set +e
if [[ ${VERBOSE:="false"} == "true" && ${VERBOSE_COMMANDS:=} != "true" ]]; then
# do not print echo if VERBOSE_COMMAND is set (set -x does it already)
>&2 echo "kubectl" "${@}"
fi
- ${KUBECTL_BINARY_PATH} "${@}" | tee -a "${OUTPUT_LOG}"
- local res="$?"
- if [[ ${res} == "0" ]]; then
+ ${KUBECTL_BINARY} "${@}" | tee -a "${OUTPUT_LOG}"
+ if [[ ${EXIT_CODE} == "0" ]]; then
+ # No matter if "set -e" is used the log will be removed on success.
rm -f "${OUTPUT_LOG}"
fi
- set -e
- return ${res}
}
-KIND_BINARY_PATH="${KIND_BINARY_PATH:=$(command -v kind || echo "/bin/kind")}"
-export KIND_BINARY_PATH
+KIND_BINARY="${KIND_BINARY:=$(command -v kind || echo "/bin/kind")}"
+export KIND_BINARY
# In case "VERBOSE" is set to "true" (--verbose flag in Breeze) all kind commands run will be
# printed before execution
@@ -104,7 +96,7 @@ function kind {
>&2 echo "kind" "${@}"
fi
# kind outputs nice output on terminal.
- ${KIND_BINARY_PATH} "${@}"
+ ${KIND_BINARY} "${@}"
}
# Prints verbose information in case VERBOSE variable is set
diff --git a/scripts/ci/openapi/client_codegen_diff.sh b/scripts/ci/openapi/client_codegen_diff.sh
index 5260680..76ba677 100755
--- a/scripts/ci/openapi/client_codegen_diff.sh
+++ b/scripts/ci/openapi/client_codegen_diff.sh
@@ -17,6 +17,7 @@
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
set -e
TARGET_REMOTE=origin
diff --git a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
index 92ec950..f8651a9 100755
--- a/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
+++ b/scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
@@ -17,37 +17,21 @@
# under the License.
set -euo pipefail
PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-export PRE_COMMIT_DIR
-readonly PRE_COMMIT_DIR
-
AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
-export AIRFLOW_SOURCES
-readonly AIRFLOW_SOURCES
-
cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
TMP_FILE=$(mktemp)
-export TMP_FILE
-readonly TMP_FILE
-
TMP_OUTPUT=$(mktemp)
-export TMP_OUTPUT
-readonly TMP_OUTPUT
echo "
.. code-block:: text
" >"${TMP_FILE}"
export MAX_SCREEN_WIDTH=100
-readonly MAX_SCREEN_WIDTH
-
export FORCE_SCREEN_WIDTH="true"
-readonly FORCE_SCREEN_WIDTH
-
export VERBOSE="false"
-readonly VERBOSE
./breeze help-all | sed 's/^/ /' | sed 's/ *$//' >>"${TMP_FILE}"
diff --git a/scripts/ci/pre_commit/pre_commit_check_integrations.sh b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
index abeee51..b204944 100755
--- a/scripts/ci/pre_commit/pre_commit_check_integrations.sh
+++ b/scripts/ci/pre_commit/pre_commit_check_integrations.sh
@@ -16,6 +16,9 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
+PRE_COMMIT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+AIRFLOW_SOURCES=$(cd "${PRE_COMMIT_DIR}/../../../" && pwd);
+cd "${AIRFLOW_SOURCES}" || exit 1
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
@@ -23,8 +26,6 @@ export SKIP_CHECK_REMOTE_IMAGE="true"
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-cd "${AIRFLOW_SOURCES}"
-
. breeze-complete
if [[ "${AVAILABLE_INTEGRATIONS} all" != "${_BREEZE_ALLOWED_INTEGRATIONS}" ]]; then
diff --git a/scripts/ci/pre_commit/pre_commit_ci_build.sh b/scripts/ci/pre_commit/pre_commit_ci_build.sh
index 646377e..56e7cbe 100755
--- a/scripts/ci/pre_commit/pre_commit_ci_build.sh
+++ b/scripts/ci/pre_commit/pre_commit_ci_build.sh
@@ -20,6 +20,7 @@ export REMEMBER_LAST_ANSWER="${2}"
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
diff --git a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
index c24ee29..c687992 100755
--- a/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
+++ b/scripts/ci/pre_commit/pre_commit_local_yml_mounts.sh
@@ -15,16 +15,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -euo pipefail
export PRINT_INFO_FROM_SCRIPTS="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
TMP_OUTPUT=$(mktemp)
# Remove temp file if it's hanging around
-add_trap "rm -rf -- '${TMP_OUTPUT}' 2>/dev/null" EXIT HUP INT TERM
+trap 'rm -rf -- "${TMP_OUTPUT}" 2>/dev/null' EXIT
LOCAL_YML_FILE="${AIRFLOW_SOURCES}/scripts/ci/docker-compose/local.yml"
diff --git a/scripts/ci/pre_commit/pre_commit_mermaid.sh b/scripts/ci/pre_commit/pre_commit_mermaid.sh
index b364511..fc52c91 100755
--- a/scripts/ci/pre_commit/pre_commit_mermaid.sh
+++ b/scripts/ci/pre_commit/pre_commit_mermaid.sh
@@ -16,11 +16,9 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
+AIRFLOW_SOURCES="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd ../../../ && pwd )"
export NO_TERMINAL_OUTPUT_FROM_SCRIPTS="true"
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
if ! command -v npm; then
echo 'You need to have npm installed in order to generate .mermaid graphs automatically.'
echo
@@ -28,7 +26,8 @@ if ! command -v npm; then
exit 0
fi
-tmp_file="${CACHE_TMP_FILE_DIR}/tmp.mermaid"
+tmp_file=$(mktemp)
+
cd "${AIRFLOW_SOURCES}"
MERMAID_INSTALLATION_DIR="${AIRFLOW_SOURCES}/.build/mermaid/"
@@ -43,7 +42,7 @@ else
fi
# shellcheck disable=SC2064
-add_trap "rm -rf '${tmp_file}'" EXIT HUP INT TERM
+trap "rm -rf ${tmp_file}" EXIT
for file in "${@}"
do
diff --git a/scripts/ci/static_checks/bat_tests.sh b/scripts/ci/static_checks/bat_tests.sh
index a5163d0..5b74885 100755
--- a/scripts/ci/static_checks/bat_tests.sh
+++ b/scripts/ci/static_checks/bat_tests.sh
@@ -16,6 +16,10 @@
# specific language governing permissions and limitations
# under the License.
# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+
+initialize_common_environment
+
function run_bats_tests() {
FILES=("$@")
if [[ "${#FILES[@]}" == "0" ]]; then
diff --git a/scripts/ci/static_checks/flake8.sh b/scripts/ci/static_checks/flake8.sh
index e14924e..f6613ab 100755
--- a/scripts/ci/static_checks/flake8.sh
+++ b/scripts/ci/static_checks/flake8.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -34,6 +36,8 @@ function run_flake8() {
fi
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/mypy.sh b/scripts/ci/static_checks/mypy.sh
index 47770c4..01e03cf 100755
--- a/scripts/ci/static_checks/mypy.sh
+++ b/scripts/ci/static_checks/mypy.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -31,6 +33,8 @@ function run_mypy() {
"--" "/opt/airflow/scripts/in_container/run_mypy.sh" "${FILES[@]}"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/pylint.sh b/scripts/ci/static_checks/pylint.sh
index 6d30f5c..dddb6d9 100755
--- a/scripts/ci/static_checks/pylint.sh
+++ b/scripts/ci/static_checks/pylint.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -33,6 +35,8 @@ function run_pylint() {
fi
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/refresh_pylint_todo.sh b/scripts/ci/static_checks/refresh_pylint_todo.sh
index 0473410..3657277 100755
--- a/scripts/ci/static_checks/refresh_pylint_todo.sh
+++ b/scripts/ci/static_checks/refresh_pylint_todo.sh
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-export FORCE_ANSWER_TO_QUESTIONS="quit"
+export FORCE_ANSWER_TO_QUESTIONS=quit
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -26,6 +26,8 @@ function refresh_pylint_todo() {
/opt/airflow/scripts/in_container/refresh_pylint_todo.sh
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/static_checks/run_static_checks.sh b/scripts/ci/static_checks/run_static_checks.sh
index 6400c8d..a5382e8 100755
--- a/scripts/ci/static_checks/run_static_checks.sh
+++ b/scripts/ci/static_checks/run_static_checks.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -25,6 +27,8 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
exit
fi
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index 9ef3c23..8cc0e3e 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+export VERBOSE=${VERBOSE:="false"}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
@@ -25,6 +27,7 @@ if [[ -f ${BUILD_CACHE_DIR}/.skip_tests ]]; then
exit
fi
+
function run_airflow_testing_in_docker() {
set +u
set +e
@@ -73,6 +76,8 @@ function run_airflow_testing_in_docker() {
return "${EXIT_CODE}"
}
+get_environment_for_builds_on_ci
+
prepare_ci_build
rebuild_ci_image_if_needed
@@ -116,7 +121,7 @@ if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml")
fi
-if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE} ]]; then
+if [[ ${INSTALL_AIRFLOW_VERSION} != "" || ${INSTALL_AIRFLOW_REFERENCE} != "" ]]; then
DOCKER_COMPOSE_LOCAL+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml")
fi
diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh
index 7ef4873..2c62c74 100755
--- a/scripts/ci/tools/ci_clear_tmp.sh
+++ b/scripts/ci/tools/ci_clear_tmp.sh
@@ -15,12 +15,28 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
#
# Fixes ownership for files created inside container (files owned by root will be owned by host user)
#
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+export AIRFLOW_CI_IMAGE=\
+${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci
+
+export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE}
+export WEBSERVER_HOST_PORT=28080
+HOST_USER_ID="$(id -ur)"
+HOST_GROUP_ID="$(id -gr)"
+HOST_OS="$(uname -s)"
+
+export HOST_USER_ID
+export HOST_GROUP_ID
+export HOST_OS
+
docker-compose \
-f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
-f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \
diff --git a/scripts/ci/tools/ci_count_changed_files.sh b/scripts/ci/tools/ci_count_changed_files.sh
index 94bcaf1..9a5a46f 100755
--- a/scripts/ci/tools/ci_count_changed_files.sh
+++ b/scripts/ci/tools/ci_count_changed_files.sh
@@ -21,9 +21,12 @@
# Parameters
# $1: Revision to compare
# $2: Pattern to match
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+get_environment_for_builds_on_ci
+
if [[ ${CI_EVENT_TYPE} == "push" ]]; then
echo
echo "Always run all tests on push"
diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/ci_fix_ownership.sh
index 975a29b..b534b20 100755
--- a/scripts/ci/tools/ci_fix_ownership.sh
+++ b/scripts/ci/tools/ci_fix_ownership.sh
@@ -19,9 +19,25 @@
#
# Fixes ownership for files created inside container (files owned by root will be owned by host user)
#
+export PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:-3.6}
+
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+export AIRFLOW_CI_IMAGE=\
+${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci
+
+export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE}
+export WEBSERVER_HOST_PORT=28080
+HOST_USER_ID="$(id -ur)"
+HOST_GROUP_ID="$(id -gr)"
+HOST_OS="$(uname -s)"
+
+export HOST_USER_ID
+export HOST_GROUP_ID
+export HOST_OS
+export BACKEND="sqlite"
+
docker-compose \
-f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
-f "${SCRIPTS_CI_DIR}/docker-compose/local.yml" \
diff --git a/scripts/in_container/_in_container_script_init.sh b/scripts/in_container/_in_container_script_init.sh
index 2abb02a..76aa32a 100755
--- a/scripts/in_container/_in_container_script_init.sh
+++ b/scripts/in_container/_in_container_script_init.sh
@@ -28,4 +28,4 @@ in_container_basic_sanity_check
in_container_script_start
-add_trap in_container_script_end EXIT HUP INT TERM
+trap in_container_script_end EXIT
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 6ede33f..fd058a5 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -16,27 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-#######################################################################################################
-#
-# Adds trap to the traps already set.
-#
-# Arguments:
-# trap to set
-# .... list of signals to handle
-#######################################################################################################
-function add_trap() {
- trap="${1}"
- shift
- for signal in "${@}"
- do
- # adding trap to exiting trap
- local handlers
- handlers="$( trap -p "${signal}" | cut -f2 -d \' )"
- # shellcheck disable=SC2064
- trap "${handlers}${handlers:+;}${trap}" "${signal}"
- done
-}
-
function assert_in_container() {
export VERBOSE=${VERBOSE:="false"}
if [[ ! -f /.dockerenv ]]; then
@@ -61,7 +40,7 @@ function in_container_script_end() {
EXIT_CODE=$?
if [[ ${EXIT_CODE} != 0 ]]; then
if [[ "${PRINT_INFO_FROM_SCRIPTS=="true"}" == "true" ]] ;then
- if [[ -n ${OUT_FILE_PRINTED_ON_ERROR=} ]]; then
+ if [[ -n ${OUT_FILE_PRINTED_ON_ERROR:=} ]]; then
echo " ERROR ENCOUNTERED!"
echo
echo " Output:"
@@ -271,7 +250,7 @@ function setup_kerberos() {
}
function dump_airflow_logs() {
- DUMP_FILE=/files/airflow_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID}.log.tar.gz
+ DUMP_FILE=/files/airflow_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID:="default"}_${CI_JOB_ID:="default"}.log.tar.gz
echo "###########################################################################################"
echo " Dumping logs from all the airflow tasks"
echo "###########################################################################################"
diff --git a/scripts/in_container/check_environment.sh b/scripts/in_container/check_environment.sh
index 31ee76f..0a1fd14 100755
--- a/scripts/in_container/check_environment.sh
+++ b/scripts/in_container/check_environment.sh
@@ -101,7 +101,7 @@ function resetdb_if_requested() {
echo "==============================================================================================="
echo " Checking integrations and backends"
echo "==============================================================================================="
-if [[ -n ${BACKEND=} ]]; then
+if [[ -n ${BACKEND:=} ]]; then
check_db_backend 20
echo "-----------------------------------------------------------------------------------------------"
fi
@@ -125,7 +125,7 @@ fi
resetdb_if_requested
-if [[ -n ${DISABLED_INTEGRATIONS=} ]]; then
+if [[ ${DISABLED_INTEGRATIONS} != "" ]]; then
echo
echo "Disabled integrations:${DISABLED_INTEGRATIONS}"
echo
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index 25fb480..302d30c 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -35,7 +35,7 @@ echo
echo "Airflow home: ${AIRFLOW_HOME}"
echo "Airflow sources: ${AIRFLOW_SOURCES}"
echo "Airflow core SQL connection: ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}"
-if [[ -n "${AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS=}" ]]; then
+if [[ -n "${AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS:=}" ]]; then
echo "Airflow collation for IDs: ${AIRFLOW__CORE__SQL_ENGINE_COLLATION_FOR_IDS}"
fi
@@ -57,7 +57,7 @@ else
export RUN_AIRFLOW_1_10="false"
fi
-if [[ -z ${INSTALL_AIRFLOW_VERSION=} ]]; then
+if [[ ${INSTALL_AIRFLOW_VERSION} == "" ]]; then
if [[ ! -d "${AIRFLOW_SOURCES}/airflow/www/node_modules" ]]; then
echo
echo "Installing node modules as they are not yet installed (Sources mounted from Host)"
@@ -192,7 +192,7 @@ if [[ ${#@} -gt 0 && -n "$1" ]]; then
TESTS_TO_RUN=("${@}")
fi
-if [[ -n ${RUN_INTEGRATION_TESTS=} ]]; then
+if [[ -n ${RUN_INTEGRATION_TESTS:=""} ]]; then
# Integration tests
for INT in ${RUN_INTEGRATION_TESTS}
do
diff --git a/scripts/in_container/prod/entrypoint_prod.sh b/scripts/in_container/prod/entrypoint_prod.sh
index 98708b7..5e3b241 100755
--- a/scripts/in_container/prod/entrypoint_prod.sh
+++ b/scripts/in_container/prod/entrypoint_prod.sh
@@ -44,7 +44,7 @@ function verify_db_connection {
echo DB_BACKEND="${DB_BACKEND:=${DETECTED_DB_BACKEND}}"
- if [[ -z "${DETECTED_DB_PORT=}" ]]; then
+ if [[ -z "${DETECTED_DB_PORT}" ]]; then
if [[ ${DB_BACKEND} == "postgres"* ]]; then
DETECTED_DB_PORT=5432
elif [[ ${DB_BACKEND} == "mysql"* ]]; then
@@ -106,7 +106,7 @@ verify_db_connection "${AIRFLOW__CORE__SQL_ALCHEMY_CONN}"
AIRFLOW__CELERY__BROKER_URL=${AIRFLOW__CELERY__BROKER_URL:=}
-if [[ -n ${AIRFLOW__CELERY__BROKER_URL=} ]] && \
+if [[ -n ${AIRFLOW__CELERY__BROKER_URL} ]] && \
[[ ${AIRFLOW_COMMAND} =~ ^(scheduler|worker|flower)$ ]]; then
verify_db_connection "${AIRFLOW__CELERY__BROKER_URL}"
fi
diff --git a/scripts/in_container/run_cli_tool.sh b/scripts/in_container/run_cli_tool.sh
index 706098e..2093373 100755
--- a/scripts/in_container/run_cli_tool.sh
+++ b/scripts/in_container/run_cli_tool.sh
@@ -18,19 +18,19 @@
# under the License.
set -euo pipefail
-if [ -z "${AIRFLOW_CI_IMAGE=}" ]; then
+if [ -z "${AIRFLOW_CI_IMAGE}" ]; then
>&2 echo "Missing environment variable AIRFLOW_CI_IMAGE"
exit 1
fi
-if [ -z "${HOST_AIRFLOW_SOURCES=}" ]; then
+if [ -z "${HOST_AIRFLOW_SOURCES}" ]; then
>&2 echo "Missing environment variable HOST_AIRFLOW_SOURCES"
exit 1
fi
-if [ -z "${HOST_USER_ID=}" ]; then
+if [ -z "${HOST_USER_ID}" ]; then
>&2 echo "Missing environment variable HOST_USER_ID"
exit 1
fi
-if [ -z "${HOST_GROUP_ID=}" ]; then
+if [ -z "${HOST_GROUP_ID}" ]; then
>&2 echo "Missing environment variable HOST_GROUP_ID"
exit 1
fi
@@ -65,7 +65,7 @@ env > "${ENV_TMP_FILE}"
cleanup() {
rm "${ENV_TMP_FILE}"
}
-trap cleanup EXIT HUP INT TERM
+trap cleanup EXIT
CONTAINER_ID="$(head -n 1 < /proc/self/cgroup | cut -d ":" -f 3 | cut -d "/" -f 3)"
diff --git a/scripts/in_container/run_docs_build.sh b/scripts/in_container/run_docs_build.sh
index 1304ee8..c98cb6e 100755
--- a/scripts/in_container/run_docs_build.sh
+++ b/scripts/in_container/run_docs_build.sh
@@ -18,7 +18,10 @@
# shellcheck source=scripts/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
sudo rm -rf "${AIRFLOW_SOURCES}/docs/_build/*"
sudo rm -rf "${AIRFLOW_SOURCES}/docs/_api/*"
diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh
index d361b0e..484c472 100755
--- a/scripts/in_container/run_generate_constraints.sh
+++ b/scripts/in_container/run_generate_constraints.sh
@@ -18,7 +18,10 @@
# shellcheck source=scripts/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
CONSTRAINTS_DIR="/files/constraints-${PYTHON_MAJOR_MINOR_VERSION}"
diff --git a/scripts/in_container/run_prepare_backport_packages.sh b/scripts/in_container/run_prepare_backport_packages.sh
index 4911f81..6ebef75 100755
--- a/scripts/in_container/run_prepare_backport_packages.sh
+++ b/scripts/in_container/run_prepare_backport_packages.sh
@@ -20,7 +20,11 @@
LIST_OF_DIRS_FILE=$(mktemp)
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
+
cd "${AIRFLOW_SOURCES}/airflow/providers" || exit 1
find . -type d | sed 's/.\///; s/\//\./g' | grep -E 'hooks|operators|sensors|secrets' \
@@ -153,7 +157,7 @@ cd "${AIRFLOW_SOURCES}" || exit 1
pushd dist
-if [[ -n ${VERSION_SUFFIX_FOR_SVN=} ]]; then
+if [[ ${VERSION_SUFFIX_FOR_SVN} != "" ]]; then
for FILE in *.tar.gz
do
mv "${FILE}" "${FILE//\.tar\.gz/${VERSION_SUFFIX_FOR_SVN}-bin.tar.gz}"
diff --git a/scripts/in_container/run_prepare_backport_readme.sh b/scripts/in_container/run_prepare_backport_readme.sh
index eeeee8d..bdb8edc 100755
--- a/scripts/in_container/run_prepare_backport_readme.sh
+++ b/scripts/in_container/run_prepare_backport_readme.sh
@@ -20,7 +20,10 @@
OUT_FILE_PRINTED_ON_ERROR=$(mktemp)
-add_trap "in_container_fix_ownership" EXIT HUP INT TERM
+# adding trap to exiting trap
+HANDLERS="$( trap -p EXIT | cut -f2 -d \' )"
+# shellcheck disable=SC2064
+trap "${HANDLERS}${HANDLERS:+;}in_container_fix_ownership" EXIT
cd "${AIRFLOW_SOURCES}" || exit 1
diff --git a/tests/bats/bats_utils.bash b/tests/bats/bats_utils.bash
index 2df8786..e2f174e 100644
--- a/tests/bats/bats_utils.bash
+++ b/tests/bats/bats_utils.bash
@@ -17,17 +17,12 @@
AIRFLOW_SOURCES=$(pwd)
export AIRFLOW_SOURCES
-readonly AIRFLOW_SOURCES
+export SCRIPTS_CI_DIR=${AIRFLOW_SOURCES}/scripts/ci
-export DOCKER_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/docker.sh
-export KUBECTL_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/kubectl.sh
-export KIND_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/kind.sh
-export HELM_BINARY_PATH=${AIRFLOW_SOURCES}/tests/bats/mock/helm.sh
-export SKIP_IN_CONTAINER_CHECK="true"
+export DOCKER_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/docker.sh
+export KUBECTL_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/kubectl.sh
+export KIND_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/kind.sh
+export HELM_BINARY=${AIRFLOW_SOURCES}/tests/bats/mock/helm.sh
# shellcheck source=scripts/ci/libraries/_all_libs.sh
-source "scripts/ci/libraries/_all_libs.sh"
-
-initialize_common_environment
-
-basic_sanity_checks
+source "${SCRIPTS_CI_DIR}/libraries/_all_libs.sh"
diff --git a/tests/bats/test_breeze_params.bats b/tests/bats/test_breeze_params.bats
index 0795ee2..7d051b4 100644
--- a/tests/bats/test_breeze_params.bats
+++ b/tests/bats/test_breeze_params.bats
@@ -25,10 +25,12 @@ setup() {
}
teardown() {
+ load bats_utils
rm -f "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
}
@test "Test missing value for a parameter" {
+ load bats_utils
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
diff <(echo "${output}") - <<EOF
@@ -41,6 +43,10 @@ EOF
}
@test "Test wrong value for a parameter but proper stored in the .build/PARAM" {
+ load bats_utils
+
+ initialize_common_environment
+
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=x
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
@@ -57,6 +63,10 @@ EOF
}
@test "Test wrong value for a parameter stored in the .build/PARAM" {
+ load bats_utils
+
+ initialize_common_environment
+
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=x
echo "x" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
@@ -75,6 +85,10 @@ EOF
@test "Test correct value for a parameter" {
+ load bats_utils
+
+ initialize_common_environment
+
export _BREEZE_ALLOWED_TEST_PARAMS="a b c"
export TEST_PARAM=a
run check_and_save_allowed_param "TEST_PARAM" "Test Param" "--message"
@@ -85,6 +99,10 @@ EOF
}
@test "Test correct value for a parameter from multi-line values" {
+ load bats_utils
+
+ initialize_common_environment
+
_BREEZE_ALLOWED_TEST_PARAMS=$(cat <<-EOF
a
b
@@ -102,6 +120,10 @@ EOF
@test "Test read_parameter from missing file" {
+ load bats_utils
+
+ initialize_common_environment
+
run read_from_file TEST_PARAM
[ -z "${TEST_FILE}" ]
diff <(echo "${output}") <(echo "")
@@ -110,6 +132,10 @@ EOF
}
@test "Test read_parameter from file" {
+ load bats_utils
+
+ initialize_common_environment
+
echo "a" > "${AIRFLOW_SOURCES}/.build/.TEST_PARAM"
run read_from_file TEST_PARAM
diff <(echo "${output}") <(echo "a")
diff --git a/tests/bats/test_empty_test.bats b/tests/bats/test_empty_test.bats
index 2f79819..308d78d 100644
--- a/tests/bats/test_empty_test.bats
+++ b/tests/bats/test_empty_test.bats
@@ -20,6 +20,8 @@
@test "empty test" {
load bats_utils
+ initialize_common_environment
+
run pwd
[ "${status}" == 0 ]
}
diff --git a/tests/bats/test_local_mounts.bats b/tests/bats/test_local_mounts.bats
index aef961f..97cf984 100644
--- a/tests/bats/test_local_mounts.bats
+++ b/tests/bats/test_local_mounts.bats
@@ -21,6 +21,8 @@
@test "convert volume list to docker params" {
load bats_utils
+ initialize_common_environment
+
read -r -a RES <<< "$(convert_local_mounts_to_docker_params)"
[[ ${#RES[@]} -gt 0 ]] # Array should be non-zero length