You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/04/25 07:05:52 UTC
[airflow] branch main updated (f02b0b6b40 -> e9f9d33b57)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from f02b0b6b40 Further improvement of Databricks Jobs operators (#23199)
new e2ea6a7e56 nIcer handling of cached Breeze parameters
new e9f9d33b57 Split breeze commands across multiple files
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
BREEZE.rst | 9 +-
dev/breeze/src/airflow_breeze/breeze.py | 2391 +-------------------
.../build_image/ci/build_ci_image.py | 19 +-
.../build_image/prod/build_prod_image.py | 18 +-
.../src/airflow_breeze/commands/ci_image_tools.py | 332 +++
.../src/airflow_breeze/commands/common_options.py | 403 ++++
.../commands/configuration_and_maintenance.py | 486 ++++
.../commands/configure_rich_click.py | 70 +
.../airflow_breeze/commands/custom_param_types.py | 115 +
.../airflow_breeze/commands/developer_commands.py | 480 ++++
dev/breeze/src/airflow_breeze/commands/main.py | 61 +
.../commands/production_image_tools.py | 384 ++++
.../airflow_breeze/commands/release_management.py | 397 ++++
dev/breeze/src/airflow_breeze/commands/testing.py | 93 +
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 11 +-
dev/breeze/src/airflow_breeze/utils/cache.py | 32 +-
.../src/airflow_breeze/utils/python_versions.py | 43 +
dev/breeze/src/airflow_breeze/utils/recording.py | 75 +
dev/breeze/tests/test_build_image.py | 60 -
dev/breeze/tests/test_prod_image.py | 94 -
images/breeze/output-build-image.svg | 7 +-
images/breeze/output-build-prod-image.svg | 7 +-
images/breeze/output-cleanup.svg | 11 +-
images/breeze/output-commands.svg | 12 +-
images/breeze/output-config.svg | 15 +-
images/breeze/output-docker-compose-tests.svg | 5 +-
images/breeze/output-find-newer-dependencies.svg | 4 +-
images/breeze/output-generate-constraints.svg | 7 +-
images/breeze/output-pull-image.svg | 4 +-
images/breeze/output-pull-prod-image.svg | 4 +-
images/breeze/output-shell.svg | 12 +-
images/breeze/output-start-airflow.svg | 15 +-
images/breeze/output-verify-image.svg | 5 +-
images/breeze/output-verify-prod-image.svg | 5 +-
34 files changed, 3023 insertions(+), 2663 deletions(-)
create mode 100644 dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/common_options.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/configure_rich_click.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/custom_param_types.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/developer_commands.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/main.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/production_image_tools.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/release_management.py
create mode 100644 dev/breeze/src/airflow_breeze/commands/testing.py
create mode 100644 dev/breeze/src/airflow_breeze/utils/python_versions.py
create mode 100644 dev/breeze/src/airflow_breeze/utils/recording.py
delete mode 100644 dev/breeze/tests/test_build_image.py
delete mode 100644 dev/breeze/tests/test_prod_image.py
[airflow] 02/02: Split breeze commands across multiple files
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e9f9d33b57220347e0171931ba251368e1d5ee35
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Mon Apr 25 00:31:45 2022 +0200
Split breeze commands across multiple files
The new breeze.py commands grew into separated groups of commands and
breeze.py became far too big.
This change is a refactor to split breeze.py into separated groups of
commands.
No new functionality is added, just refactoring to separate files.
Fixes: #23204
---
dev/breeze/src/airflow_breeze/breeze.py | 2474 +-------------------
.../src/airflow_breeze/commands/ci_image_tools.py | 332 +++
.../src/airflow_breeze/commands/common_options.py | 403 ++++
.../commands/configuration_and_maintenance.py | 486 ++++
.../commands/configure_rich_click.py | 70 +
.../airflow_breeze/commands/custom_param_types.py | 115 +
.../airflow_breeze/commands/developer_commands.py | 480 ++++
dev/breeze/src/airflow_breeze/commands/main.py | 61 +
.../commands/production_image_tools.py | 384 +++
.../airflow_breeze/commands/release_management.py | 397 ++++
dev/breeze/src/airflow_breeze/commands/testing.py | 93 +
.../src/airflow_breeze/utils/python_versions.py | 43 +
dev/breeze/src/airflow_breeze/utils/recording.py | 75 +
images/breeze/output-cleanup.svg | 11 +-
14 files changed, 2948 insertions(+), 2476 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py
index 850e70a0cd..276021fd40 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -15,2480 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import atexit
-import multiprocessing as mp
-import os
-import shutil
-import subprocess
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-from typing import IO, Any, List, Optional, Tuple
-import rich
-
-from airflow_breeze import NAME, VERSION
-from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.ci_group import ci_group
-from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm
-from airflow_breeze.utils.constraints import run_generate_constraints, run_generate_constraints_in_parallel
-from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies
-from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
-from airflow_breeze.utils.reinstall import ask_to_reinstall_breeze, reinstall_breeze, warn_non_editable
-from airflow_breeze.utils.run_tests import run_docker_compose_tests, verify_an_image
-
-try:
- # We handle ImportError so that click autocomplete works
- import rich_click as click
-
- try:
- click.formatting.FORCED_WIDTH = os.get_terminal_size().columns - 2
- except OSError:
- pass
- click.rich_click.SHOW_METAVARS_COLUMN = False
- click.rich_click.SHOW_ARGUMENTS = False
- click.rich_click.APPEND_METAVARS_HELP = True
- click.rich_click.STYLE_ERRORS_SUGGESTION = "bright_blue italic"
- click.rich_click.ERRORS_SUGGESTION = "\nTry running the '--help' flag for more information.\n"
- click.rich_click.ERRORS_EPILOGUE = (
- "\nTo find out more, visit [bright_blue]https://github.com/apache/airflow/blob/main/BREEZE.rst[/]\n"
- )
- click.rich_click.OPTION_GROUPS = {
- "breeze": [
- {
- "name": "Basic flags for the default (shell) command",
- "options": [
- "--python",
- "--backend",
- "--use-airflow-version",
- "--postgres-version",
- "--mysql-version",
- "--mssql-version",
- "--forward-credentials",
- "--db-reset",
- ],
- },
- {
- "name": "Advanced flags for the default (shell) command",
- "options": [
- "--force-build",
- "--mount-sources",
- "--integration",
- ],
- },
- ],
- "breeze shell": [
- {
- "name": "Basic flags",
- "options": [
- "--python",
- "--backend",
- "--use-airflow-version",
- "--postgres-version",
- "--mysql-version",
- "--mssql-version",
- "--forward-credentials",
- "--db-reset",
- ],
- },
- {
- "name": "Advanced flag for running",
- "options": [
- "--force-build",
- "--mount-sources",
- "--integration",
- ],
- },
- ],
- "breeze start-airflow": [
- {
- "name": "Basic flags",
- "options": [
- "--python",
- "--backend",
- "--use-airflow-version",
- "--postgres-version",
- "--mysql-version",
- "--mssql-version",
- "--load-example-dags",
- "--load-default-connections",
- "--forward-credentials",
- "--db-reset",
- ],
- },
- {
- "name": "Advanced flag for running",
- "options": [
- "--force-build",
- "--mount-sources",
- "--integration",
- ],
- },
- ],
- "breeze build-image": [
- {
- "name": "Basic usage",
- "options": [
- "--python",
- "--upgrade-to-newer-dependencies",
- "--debian-version",
- "--image-tag",
- "--docker-cache",
- "--force-build",
- ],
- },
- {
- "name": "Building multiple images",
- "options": [
- "--build_multiple_images",
- "--python-versions",
- ],
- },
- {
- "name": "Advanced options (for power users)",
- "options": [
- "--install-providers-from-sources",
- "--additional-python-deps",
- "--runtime-apt-deps",
- "--runtime-apt-command",
- "--additional-extras",
- "--additional-runtime-apt-deps",
- "--additional-runtime-apt-env",
- "--additional-runtime-apt-command",
- "--additional-dev-apt-deps",
- "--additional-dev-apt-env",
- "--additional-dev-apt-command",
- "--dev-apt-deps",
- "--dev-apt-command",
- ],
- },
- {
- "name": "Preparing cache and push (for maintainers and CI)",
- "options": [
- "--platform",
- "--prepare-buildx-cache",
- "--push-image",
- "--empty-image",
- "--github-token",
- "--github-username",
- "--login-to-github-registry",
- ],
- },
- ],
- "breeze pull-image": [
- {
- "name": "Pull image flags",
- "options": [
- "--image-tag",
- "--python",
- "--verify-image",
- "--wait-for-image",
- "--tag-as-latest",
- ],
- },
- {
- "name": "Parallel running",
- "options": [
- "--run-in-parallel",
- "--parallelism",
- "--python-versions",
- ],
- },
- ],
- "breeze verify-image": [
- {
- "name": "Verify image flags",
- "options": [
- "--image-name",
- "--python",
- "--image-tag",
- ],
- }
- ],
- "breeze build-prod-image": [
- {
- "name": "Basic usage",
- "options": [
- "--python",
- "--install-airflow-version",
- "--upgrade-to-newer-dependencies",
- "--debian-version",
- "--image-tag",
- "--docker-cache",
- ],
- },
- {
- "name": "Building multiple images",
- "options": [
- "--build_multiple_images",
- "--python-versions",
- ],
- },
- {
- "name": "Options for customizing images",
- "options": [
- "--install-providers-from-sources",
- "--additional-python-deps",
- "--additional-extras",
- "--additional-runtime-apt-deps",
- "--additional-runtime-apt-env",
- "--additional-runtime-apt-command",
- "--additional-dev-apt-deps",
- "--additional-dev-apt-env",
- "--additional-dev-apt-command",
- "--extras",
- "--runtime-apt-deps",
- "--runtime-apt-command",
- "--dev-apt-deps",
- "--dev-apt-command",
- ],
- },
- {
- "name": "Customization options (for specific customization needs)",
- "options": [
- "--cleanup-docker-context-files",
- "--install-from-docker-context-files",
- "--disable-mysql-client-installation",
- "--disable-mssql-client-installation",
- "--disable-postgres-client-installation",
- "--disable-airflow-repo-cache",
- "--disable-pypi",
- "--install-airflow-reference",
- "--installation-method",
- ],
- },
- {
- "name": "Preparing cache and push (for maintainers and CI)",
- "options": [
- "--github-token",
- "--github-username",
- "--login-to-github-registry",
- "--push-image",
- "--prepare-buildx-cache",
- "--platform",
- "--empty-image",
- ],
- },
- ],
- "breeze pull-prod-image": [
- {
- "name": "Pull image flags",
- "options": [
- "--image-tag",
- "--python",
- "--verify-image",
- "--wait-for-image",
- "--tag-as-latest",
- ],
- },
- {
- "name": "Parallel running",
- "options": [
- "--run-in-parallel",
- "--parallelism",
- "--python-versions",
- ],
- },
- ],
- "breeze verify-prod-image": [
- {
- "name": "Verify image flags",
- "options": [
- "--image-name",
- "--python",
- "--image-tag",
- ],
- }
- ],
- "breeze docker-compose-tests": [
- {
- "name": "Docker-compose tests flag",
- "options": [
- "--image-name",
- "--python",
- "--image-tag",
- ],
- }
- ],
- "breeze static-checks": [
- {
- "name": "Pre-commit flags",
- "options": [
- "--type",
- "--files",
- "--all-files",
- "--show-diff-on-failure",
- "--last-commit",
- ],
- },
- ],
- "breeze build-docs": [
- {
- "name": "Doc flags",
- "options": [
- "--docs-only",
- "--spellcheck-only",
- "--package-filter",
- ],
- },
- ],
- "breeze stop": [
- {
- "name": "Stop flags",
- "options": [
- "--preserve-volumes",
- ],
- },
- ],
- "breeze cleanup": [
- {
- "name": "Cleanup flags",
- "options": [
- "--all",
- ],
- },
- ],
- "breeze setup-autocomplete": [
- {
- "name": "Setup autocomplete flags",
- "options": [
- "--force",
- ],
- },
- ],
- "breeze config": [
- {
- "name": "Config flags",
- "options": [
- "--python",
- "--backend",
- "--cheatsheet",
- "--asciiart",
- ],
- },
- ],
- "breeze prepare-airflow-package": [
- {"name": "Package flags", "options": ["--package-format", "--version-suffix-for-pypi"]}
- ],
- "breeze prepare-provider-packages": [
- {
- "name": "Package flags",
- "options": [
- "--package-format",
- "--version-suffix-for-pypi",
- "--package-list-file",
- ],
- }
- ],
- "breeze prepare-provider-documentation": [
- {"name": "Provider documentation preparation flags", "options": ["--skip-package-verification"]}
- ],
- "breeze find-newer-dependencies": [
- {
- "name": "Find newer dependencies flags",
- "options": [
- "--python",
- "--timezone",
- "--constraints-branch",
- "--updated-on-or-after",
- "--max-age",
- ],
- }
- ],
- "breeze generate-constraints": [
- {
- "name": "Generate constraints flags",
- "options": [
- "--image-tag",
- "--python",
- "--generate-constraints-mode",
- ],
- },
- {
- "name": "Parallel running",
- "options": [
- "--run-in-parallel",
- "--parallelism",
- "--python-versions",
- ],
- },
- ],
- "breeze self-upgrade": [
- {
- "name": "Self-upgrade flags",
- "options": [
- "--use-current-airflow-sources",
- "--force",
- ],
- }
- ],
- }
- click.rich_click.COMMAND_GROUPS = {
- "breeze": [
- {
- "name": "Developer tools",
- "commands": [
- "shell",
- "start-airflow",
- "stop",
- "build-docs",
- "static-checks",
- ],
- },
- {
- "name": "Testing",
- "commands": [
- "docker-compose-tests",
- ],
- },
- {
- "name": "Configuration & maintenance",
- "commands": [
- "cleanup",
- "self-upgrade",
- "setup-autocomplete",
- "config",
- "resource-check",
- "free-space",
- "fix-ownership",
- "version",
- ],
- },
- {
- "name": "CI Image tools",
- "commands": [
- "build-image",
- "pull-image",
- "verify-image",
- ],
- },
- {
- "name": "Production Image tools",
- "commands": [
- "build-prod-image",
- "pull-prod-image",
- "verify-prod-image",
- ],
- },
- {
- "name": "Release management",
- "commands": [
- "prepare-provider-documentation",
- "prepare-provider-packages",
- "prepare-airflow-package",
- "generate-constraints",
- "find-newer-dependencies",
- ],
- },
- ]
- }
-
-
-except ImportError:
- import click # type: ignore[no-redef]
-
-from click import Context, IntRange
-
-from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
-from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.build_image.prod.build_prod_image import build_production_image
-from airflow_breeze.global_constants import (
- ALLOWED_BACKENDS,
- ALLOWED_BUILD_CACHE,
- ALLOWED_DEBIAN_VERSIONS,
- ALLOWED_EXECUTORS,
- ALLOWED_GENERATE_CONSTRAINTS_MODES,
- ALLOWED_INSTALLATION_METHODS,
- ALLOWED_INTEGRATIONS,
- ALLOWED_MOUNT_OPTIONS,
- ALLOWED_MSSQL_VERSIONS,
- ALLOWED_MYSQL_VERSIONS,
- ALLOWED_PACKAGE_FORMATS,
- ALLOWED_PLATFORMS,
- ALLOWED_POSTGRES_VERSIONS,
- ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
- DEFAULT_EXTRAS,
- DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- MOUNT_ALL,
- MOUNT_SELECTED,
- get_available_packages,
-)
-from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
-from airflow_breeze.shell.enter_shell import enter_shell
-from airflow_breeze.utils.cache import (
- check_if_cache_exists,
- check_if_values_allowed,
- delete_cache,
- read_and_validate_value_from_cache,
- read_from_cache_file,
- touch_cache_file,
- write_to_cache_file,
-)
-from airflow_breeze.utils.console import console
-from airflow_breeze.utils.docker_command_utils import (
- check_docker_resources,
- construct_env_variables_docker_compose_command,
- get_extra_docker_flags,
-)
-from airflow_breeze.utils.path_utils import (
- AIRFLOW_SOURCES_ROOT,
- BUILD_CACHE_DIR,
- create_directories,
- find_airflow_sources_root_to_operate_on,
- get_installation_airflow_sources,
- get_installation_sources_config_metadata_hash,
- get_package_setup_metadata_hash,
- get_used_airflow_sources,
- get_used_sources_setup_metadata_hash,
- in_autocomplete,
-)
-from airflow_breeze.utils.run_utils import assert_pre_commit_installed, filter_out_none, run_command
-from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE
+from airflow_breeze.commands.configure_rich_click import click # noqa
+from airflow_breeze.commands.main import main
+from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on
find_airflow_sources_root_to_operate_on()
-output_file_for_recording = os.environ.get('RECORD_BREEZE_OUTPUT_FILE')
-
-
-class BetterChoice(click.Choice):
- """
- Nicer formatted choice class for click. We have a lot of parameters sometimes, and formatting
- them without spaces causes ugly artifacts as the words are broken. This one adds spaces so
- that when the long list of choices does not wrap on words.
- """
-
- name = "BetterChoice"
-
- def get_metavar(self, param) -> str:
- choices_str = " | ".join(self.choices)
- # Use curly braces to indicate a required argument.
- if param.required and param.param_type_name == "argument":
- return f"{{{choices_str}}}"
-
- if param.param_type_name == "argument" and param.nargs == -1:
- # avoid double [[ for multiple args
- return f"{choices_str}"
-
- # Use square braces to indicate an option or optional argument.
- return f"[{choices_str}]"
-
-
-@dataclass
-class CacheableDefault:
- value: Any
-
- def __repr__(self):
- return self.value
-
-
-class CacheableChoice(click.Choice):
- """
- This class implements caching of values from the last use.
- """
-
- def convert(self, value, param, ctx):
- param_name = param.envvar if param.envvar else param.name.upper()
- if isinstance(value, CacheableDefault):
- is_cached, new_value = read_and_validate_value_from_cache(param_name, value.value)
- if not is_cached:
- console.print(
- f"\n[bright_blue]Default value of {param.name} " f"parameter {new_value} used.[/]\n"
- )
- else:
- allowed, allowed_values = check_if_values_allowed(param_name, value)
- if allowed:
- new_value = value
- write_to_cache_file(param_name, new_value, check_allowed_values=False)
- else:
- new_value = allowed_values[0]
- console.print(
- f"\n[yellow]The value {value} is not allowed for parameter {param.name}. "
- f"Setting default value to {new_value}"
- )
- write_to_cache_file(param_name, new_value, check_allowed_values=False)
- return super().convert(new_value, param, ctx)
-
- def get_metavar(self, param) -> str:
- param_name = param.envvar if param.envvar else param.name.upper()
- current_value = (
- read_from_cache_file(param_name) if not output_file_for_recording else param.default.value
- )
- if not current_value:
- current_choices = self.choices
- else:
- current_choices = [
- f">{choice}<" if choice == current_value else choice for choice in self.choices
- ]
- choices_str = " | ".join(current_choices)
- # Use curly braces to indicate a required argument.
- if param.required and param.param_type_name == "argument":
- return f"{{{choices_str}}}"
-
- if param.param_type_name == "argument" and param.nargs == -1:
- # avoid double [[ for multiple args
- return f"{choices_str}"
-
- # Use square braces to indicate an option or optional argument.
- return f"[{choices_str}]"
-
- def __init__(self, choices, case_sensitive: bool = True) -> None:
- super().__init__(choices=choices, case_sensitive=case_sensitive)
-
-
-option_verbose = click.option(
- "-v", "--verbose", is_flag=True, help="Print verbose information about performed steps.", envvar='VERBOSE'
-)
-
-option_dry_run = click.option(
- "-D",
- "--dry-run",
- is_flag=True,
- help="If dry-run is set, commands are only printed, not executed.",
- envvar='DRY_RUN',
-)
-
-option_answer = click.option(
- "-a",
- "--answer",
- type=BetterChoice(['y', 'n', 'q', 'yes', 'no', 'quit']),
- help="Force answer to questions.",
- envvar='ANSWER',
-)
-
-option_python = click.option(
- '-p',
- '--python',
- type=CacheableChoice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS),
- default=CacheableDefault(value=ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0]),
- show_default=True,
- help='Python major/minor version used in Airflow image for images.',
- envvar='PYTHON_MAJOR_MINOR_VERSION',
-)
-
-option_backend = click.option(
- '-b',
- '--backend',
- type=CacheableChoice(ALLOWED_BACKENDS),
- default=CacheableDefault(value=ALLOWED_BACKENDS[0]),
- show_default=True,
- help="Database backend to use.",
- envvar='BACKEND',
-)
-
-option_integration = click.option(
- '--integration',
- help="Integration(s) to enable when running (can be more than one).",
- type=BetterChoice(ALLOWED_INTEGRATIONS),
- multiple=True,
-)
-
-option_postgres_version = click.option(
- '-P',
- '--postgres-version',
- type=CacheableChoice(ALLOWED_POSTGRES_VERSIONS),
- default=CacheableDefault(ALLOWED_POSTGRES_VERSIONS[0]),
- show_default=True,
- help="Version of Postgres used.",
-)
-
-option_mysql_version = click.option(
- '-M',
- '--mysql-version',
- help="Version of MySQL used.",
- type=CacheableChoice(ALLOWED_MYSQL_VERSIONS),
- default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]),
- show_default=True,
-)
-
-option_mssql_version = click.option(
- '-S',
- '--mssql-version',
- help="Version of MsSQL used.",
- type=CacheableChoice(ALLOWED_MSSQL_VERSIONS),
- default=CacheableDefault(ALLOWED_MSSQL_VERSIONS[0]),
- show_default=True,
-)
-
-option_executor = click.option(
- '--executor',
- help='Executor to use for a kubernetes cluster. Default is KubernetesExecutor.',
- type=BetterChoice(ALLOWED_EXECUTORS),
-)
-
-option_forward_credentials = click.option(
- '-f', '--forward-credentials', help="Forward local credentials to container when running.", is_flag=True
-)
-
-option_use_airflow_version = click.option(
- '-V',
- '--use-airflow-version',
- help="Use (reinstall at entry) Airflow version from PyPI.",
- envvar='USE_AIRFLOW_VERSION',
-)
-
-option_mount_sources = click.option(
- '--mount-sources',
- type=BetterChoice(ALLOWED_MOUNT_OPTIONS),
- default=ALLOWED_MOUNT_OPTIONS[0],
- show_default=True,
- help="Choose scope of local sources should be mounted (default = selected).",
-)
-
-option_force_build = click.option(
- '-f', '--force-build', help="Force image build no matter if it is determined as needed.", is_flag=True
-)
-
-option_db_reset = click.option(
- '-d',
- '--db-reset',
- help="Reset DB when entering the container.",
- is_flag=True,
- envvar='DB_RESET',
-)
-
-option_github_repository = click.option(
- '-g',
- '--github-repository',
- help='GitHub repository used to pull, push run images.',
- default="apache/airflow",
- show_default=True,
- envvar='GITHUB_REPOSITORY',
-)
-
-
-@click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']})
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_backend
-@option_postgres_version
-@option_mysql_version
-@option_mssql_version
-@option_forward_credentials
-@option_force_build
-@option_use_airflow_version
-@option_mount_sources
-@option_integration
-@option_db_reset
-@option_answer
-@click.pass_context
-def main(ctx: Context, **kwargs):
- create_directories()
- if not ctx.invoked_subcommand:
- ctx.forward(shell, extra_args={})
-
-
-option_docker_cache = click.option(
- '-c',
- '--docker-cache',
- help='Cache option for image used during the build.',
- default=ALLOWED_BUILD_CACHE[0],
- show_default=True,
- type=BetterChoice(ALLOWED_BUILD_CACHE),
-)
-
-option_login_to_github_registry = click.option(
- '--login-to-github-registry',
- help='Logs in to GitHub registry.',
- envvar='LOGIN_TO_GITHUB_REGISTRY',
-)
-
-
-option_github_token = click.option(
- '--github-token',
- help='The token used to authenticate to GitHub.',
- envvar='GITHUB_TOKEN',
-)
-
-option_github_username = click.option(
- '--github-username',
- help='The user name used to authenticate to GitHub.',
- envvar='GITHUB_USERNAME',
-)
-
-option_github_image_id = click.option(
- '-s',
- '--github-image-id',
- help='Commit SHA of the image. \
- Breeze can automatically pull the commit SHA id specified Default: latest',
-)
-
-option_image_tag = click.option(
- '-t',
- '--image-tag',
- help='Tag added to the default naming conventions of Airflow CI/PROD images.',
- envvar='IMAGE_TAG',
-)
-
-option_image_name = click.option(
- '-n', '--image-name', help='Name of the image to verify (overrides --python and --image-tag).'
-)
-
-option_platform = click.option(
- '--platform',
- help='Platform for Airflow image.',
- envvar='PLATFORM',
- type=BetterChoice(ALLOWED_PLATFORMS),
-)
-
-option_debian_version = click.option(
- '-d',
- '--debian-version',
- help='Debian version used for the image.',
- type=BetterChoice(ALLOWED_DEBIAN_VERSIONS),
- default=ALLOWED_DEBIAN_VERSIONS[0],
- show_default=True,
- envvar='DEBIAN_VERSION',
-)
-option_upgrade_to_newer_dependencies = click.option(
- "-u",
- '--upgrade-to-newer-dependencies',
- default="false",
- show_default=True,
- help='When other than "false", upgrade all PIP packages to latest.',
- envvar='UPGRADE_TO_NEWER_DEPENDENCIES',
-)
-option_additional_extras = click.option(
- '--additional-extras',
- help='Additional extra package while installing Airflow in the image.',
- envvar='ADDITIONAL_AIRFLOW_EXTRAS',
-)
-option_additional_dev_apt_deps = click.option(
- '--additional-dev-apt-deps',
- help='Additional apt dev dependencies to use when building the images.',
- envvar='ADDITIONAL_DEV_APT_DEPS',
-)
-option_additional_runtime_apt_deps = click.option(
- '--additional-runtime-apt-deps',
- help='Additional apt runtime dependencies to use when building the images.',
- envvar='ADDITIONAL_RUNTIME_APT_DEPS',
-)
-option_additional_python_deps = click.option(
- '--additional-python-deps',
- help='Additional python dependencies to use when building the images.',
- envvar='ADDITIONAL_PYTHON_DEPS',
-)
-option_additional_dev_apt_command = click.option(
- '--additional-dev-apt-command',
- help='Additional command executed before dev apt deps are installed.',
- envvar='ADDITIONAL_DEV_APT_COMMAND',
-)
-option_additional_runtime_apt_command = click.option(
- '--additional-runtime-apt-command',
- help='Additional command executed before runtime apt deps are installed.',
- envvar='ADDITIONAL_RUNTIME_APT_COMMAND',
-)
-option_additional_dev_apt_env = click.option(
- '--additional-dev-apt-env',
- help='Additional environment variables set when adding dev dependencies.',
- envvar='ADDITIONAL_DEV_APT_ENV',
-)
-option_additional_runtime_apt_env = click.option(
- '--additional-runtime-apt-env',
- help='Additional environment variables set when adding runtime dependencies.',
- envvar='ADDITIONAL_RUNTIME_APT_ENV',
-)
-option_dev_apt_command = click.option(
- '--dev-apt-command',
- help='Command executed before dev apt deps are installed.',
- envvar='DEV_APT_COMMAND',
-)
-option_dev_apt_deps = click.option(
- '--dev-apt-deps',
- help='Apt dev dependencies to use when building the images.',
- envvar='DEV_APT_DEPS',
-)
-option_runtime_apt_command = click.option(
- '--runtime-apt-command',
- help='Command executed before runtime apt deps are installed.',
- envvar='RUNTIME_APT_COMMAND',
-)
-option_runtime_apt_deps = click.option(
- '--runtime-apt-deps',
- help='Apt runtime dependencies to use when building the images.',
- envvar='RUNTIME_APT_DEPS',
-)
-
-option_skip_rebuild_check = click.option(
- '-r',
- '--skip-rebuild-check',
- help="Skips checking if rebuild is needed",
- is_flag=True,
- envvar='SKIP_REBUILD_CHECK',
-)
-
-option_prepare_buildx_cache = click.option(
- '--prepare-buildx-cache',
- help='Prepares build cache rather than build images locally.',
- is_flag=True,
- envvar='PREPARE_BUILDX_CACHE',
-)
-
-option_push_image = click.option(
- '--push-image',
- help='Push image after building it.',
- is_flag=True,
- envvar='PUSH_IMAGE',
-)
-
-option_empty_image = click.option(
- '--empty-image',
- help='Prepare empty image tagged with the same name as the Airflow image.',
- is_flag=True,
- envvar='EMPTY_IMAGE',
-)
-
-option_wait_for_image = click.option(
- '--wait-for-image',
- help='Wait until image is available.',
- is_flag=True,
- envvar='WAIT_FOR_IMAGE',
-)
-
-option_tag_as_latest = click.option(
- '--tag-as-latest',
- help='Tags the image as latest after pulling.',
- is_flag=True,
- envvar='TAG_AS_LATEST',
-)
-
-option_verify_image = click.option(
- '--verify-image',
- help='Verify image.',
- is_flag=True,
- envvar='VERIFY_IMAGE',
-)
-
-option_install_providers_from_sources = click.option(
- '--install-providers-from-sources',
- help="Install providers from sources when installing.",
- is_flag=True,
- envvar='INSTALL_PROVIDERS_FROM_SOURCES',
-)
-
-option_load_example_dags = click.option(
- '-e',
- '--load-example-dags',
- help="Enable configuration to load example DAGs when starting Airflow.",
- is_flag=True,
- envvar='LOAD_EXAMPLES',
-)
-
-option_load_default_connection = click.option(
- '-c',
- '--load-default-connections',
- help="Enable configuration to load default connections when starting Airflow.",
- is_flag=True,
- envvar='LOAD_DEFAULT_CONNECTIONS',
-)
-
-option_version_suffix_for_pypi = click.option(
- '--version-suffix-for-pypi',
- help='Version suffix used for PyPI packages (alpha, beta, rc1, etc.).',
- default="",
- envvar='VERSION_SUFFIX_FOR_PYPI',
-)
-
-option_package_format = click.option(
- '--package-format',
- type=BetterChoice(ALLOWED_PACKAGE_FORMATS),
- help='Format of packages.',
- default=ALLOWED_PACKAGE_FORMATS[0],
- show_default=True,
- envvar='PACKAGE_FORMAT',
-)
-
-option_python_versions = click.option(
- '--python-versions',
- help="Space separated list of python versions used for build with multiple versions.",
- default=" ".join(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS),
- show_default=True,
- envvar="PYTHON_VERSIONS",
-)
-
-option_run_in_parallel = click.option(
- '--run-in-parallel',
- help="Run the operation in parallel on all or selected subset of Python versions.",
- is_flag=True,
- envvar='RUN_IN_PARALLEL',
-)
-
-option_parallelism = click.option(
- '--parallelism',
- help="Maximum number of processes to use while running the operation in parallel.",
- type=IntRange(1, mp.cpu_count() * 2 if not output_file_for_recording else 8),
- default=mp.cpu_count() if not output_file_for_recording else 4,
- envvar='PARALLELISM',
- show_default=True,
-)
-
-option_build_multiple_images = click.option(
- '--build_multiple_images',
- help="Run the operation sequentially on all or selected subset of Python versions.",
- is_flag=True,
- envvar='BUILD_MULTIPLE_IMAGES',
-)
-
-option_with_ci_group = click.option(
- '-g',
- '--with-ci-group',
- is_flag=True,
- help="Uses CI group for the command to fold long logs in logical groups.",
- envvar='WITH_CI_GROUP',
-)
-
-argument_packages = click.argument(
- "packages",
- nargs=-1,
- required=False,
- type=BetterChoice(get_available_packages(short_version=True)),
-)
-
-
-@option_verbose
-@main.command()
-def version(verbose: bool, python: str):
- """Print information about version of apache-airflow-breeze."""
- console.print(ASCIIART, style=ASCIIART_STYLE)
- console.print(f"\n[bright_blue]Breeze version: {VERSION}[/]")
- console.print(f"[bright_blue]Breeze installed from: {get_installation_airflow_sources()}[/]")
- console.print(f"[bright_blue]Used Airflow sources : {get_used_airflow_sources()}[/]\n")
- if verbose:
- console.print(
- f"[bright_blue]Installation sources config hash : "
- f"{get_installation_sources_config_metadata_hash()}[/]"
- )
- console.print(
- f"[bright_blue]Used sources config hash : " f"{get_used_sources_setup_metadata_hash()}[/]"
- )
- console.print(
- f"[bright_blue]Package config hash : " f"{(get_package_setup_metadata_hash())}[/]\n"
- )
-
-
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-# Make sure that whatever you add here as an option is also
-# Added in the "main" command above. The min command above
-# Is used for a shorthand of shell and except the extra
-# Args it should have the same parameters.
-# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-@main.command()
-@option_verbose
-@option_dry_run
-@option_python
-@option_backend
-@option_github_repository
-@option_postgres_version
-@option_mysql_version
-@option_mssql_version
-@option_forward_credentials
-@option_force_build
-@option_use_airflow_version
-@option_mount_sources
-@option_integration
-@option_db_reset
-@option_answer
-@click.argument('extra-args', nargs=-1, type=click.UNPROCESSED)
-def shell(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- backend: str,
- integration: Tuple[str],
- postgres_version: str,
- mysql_version: str,
- mssql_version: str,
- forward_credentials: bool,
- mount_sources: str,
- use_airflow_version: str,
- force_build: bool,
- db_reset: bool,
- answer: Optional[str],
- extra_args: Tuple,
-):
- """Enter breeze.py environment. this is the default command use when no other is selected."""
- set_forced_answer(answer)
- if verbose or dry_run:
- console.print("\n[green]Welcome to breeze.py[/]\n")
- console.print(f"\n[green]Root of Airflow Sources = {AIRFLOW_SOURCES_ROOT}[/]\n")
- enter_shell(
- verbose=verbose,
- dry_run=dry_run,
- python=python,
- github_repository=github_repository,
- backend=backend,
- integration=integration,
- postgres_version=postgres_version,
- mysql_version=mysql_version,
- mssql_version=mssql_version,
- forward_credentials=str(forward_credentials),
- mount_sources=mount_sources,
- use_airflow_version=use_airflow_version,
- force_build=force_build,
- db_reset=db_reset,
- extra_args=extra_args,
- answer=answer,
- )
-
-
-@option_verbose
-@main.command(name='start-airflow')
-@option_dry_run
-@option_python
-@option_github_repository
-@option_backend
-@option_postgres_version
-@option_load_example_dags
-@option_load_default_connection
-@option_mysql_version
-@option_mssql_version
-@option_forward_credentials
-@option_force_build
-@option_use_airflow_version
-@option_mount_sources
-@option_integration
-@option_db_reset
-@option_answer
-@click.argument('extra-args', nargs=-1, type=click.UNPROCESSED)
-def start_airflow(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- backend: str,
- integration: Tuple[str],
- postgres_version: str,
- load_example_dags: bool,
- load_default_connections: bool,
- mysql_version: str,
- mssql_version: str,
- forward_credentials: bool,
- mount_sources: str,
- use_airflow_version: str,
- force_build: bool,
- db_reset: bool,
- answer: Optional[str],
- extra_args: Tuple,
-):
- """Enter breeze.py environment and starts all Airflow components in the tmux session."""
- set_forced_answer(answer)
- enter_shell(
- verbose=verbose,
- dry_run=dry_run,
- python=python,
- github_repository=github_repository,
- backend=backend,
- integration=integration,
- postgres_version=postgres_version,
- load_default_connections=load_default_connections,
- load_example_dags=load_example_dags,
- mysql_version=mysql_version,
- mssql_version=mssql_version,
- forward_credentials=str(forward_credentials),
- mount_sources=mount_sources,
- use_airflow_version=use_airflow_version,
- force_build=force_build,
- db_reset=db_reset,
- start_airflow=True,
- extra_args=extra_args,
- answer=answer,
- )
-
-
-@main.command(name='build-image')
-@option_github_repository
-@option_verbose
-@option_dry_run
-@option_with_ci_group
-@option_answer
-@option_python
-@option_build_multiple_images
-@option_python_versions
-@option_upgrade_to_newer_dependencies
-@option_platform
-@option_debian_version
-@option_github_token
-@option_github_username
-@option_login_to_github_registry
-@option_docker_cache
-@option_image_tag
-@option_prepare_buildx_cache
-@option_push_image
-@option_empty_image
-@option_install_providers_from_sources
-@option_additional_extras
-@option_additional_dev_apt_deps
-@option_additional_runtime_apt_deps
-@option_additional_python_deps
-@option_additional_dev_apt_command
-@option_runtime_apt_command
-@option_additional_dev_apt_env
-@option_additional_runtime_apt_env
-@option_additional_runtime_apt_command
-@option_dev_apt_command
-@option_dev_apt_deps
-@option_runtime_apt_command
-@option_runtime_apt_deps
-@option_force_build
-def build_image(
- verbose: bool,
- dry_run: bool,
- with_ci_group: bool,
- build_multiple_images: bool,
- python_versions: str,
- answer: str,
- **kwargs,
-):
- """Build CI image. Include building multiple images for all python versions (sequentially)."""
-
- def run_build(ci_image_params: BuildCiParams) -> None:
- return_code, info = build_ci_image(
- verbose=verbose, dry_run=dry_run, with_ci_group=with_ci_group, ci_image_params=ci_image_params
- )
- if return_code != 0:
- console.print(f"[red]Error when building image! {info}")
- sys.exit(return_code)
-
- set_forced_answer(answer)
- parameters_passed = filter_out_none(**kwargs)
- if build_multiple_images:
- with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
- python_version_list = get_python_version_list(python_versions)
- for python in python_version_list:
- params = BuildCiParams(**parameters_passed)
- params.python = python
- params.answer = answer
- run_build(ci_image_params=params)
- else:
- params = BuildCiParams(**parameters_passed)
- run_build(ci_image_params=params)
-
-
-def get_python_version_list(python_versions: str) -> List[str]:
- """
- Retrieve and validate space-separated list of Python versions and return them in the form of list.
- :param python_versions: space separated list of Python versions
- :return: List of python versions
- """
- python_version_list = python_versions.split(" ")
- errors = False
- for python in python_version_list:
- if python not in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS:
- console.print(f"[red]The Python version {python} passed in {python_versions} is wrong.[/]")
- errors = True
- if errors:
- console.print(
- f"\nSome of the Python versions passed are not in the "
- f"list: {ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS}. Quitting.\n"
- )
- sys.exit(1)
- return python_version_list
-
-
-@main.command(name='pull-image')
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_run_in_parallel
-@option_parallelism
-@option_python_versions
-@option_verify_image
-@option_wait_for_image
-@option_tag_as_latest
-@option_image_tag
-@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def pull_image(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- run_in_parallel: bool,
- python_versions: str,
- parallelism: int,
- image_tag: Optional[str],
- wait_for_image: bool,
- tag_as_latest: bool,
- verify_image: bool,
- extra_pytest_args: Tuple,
-):
- """Pull and optionally verify CI images - possibly in parallel for all Python versions."""
- if run_in_parallel:
- python_version_list = get_python_version_list(python_versions)
- ci_image_params_list = [
- BuildCiParams(image_tag=image_tag, python=python, github_repository=github_repository)
- for python in python_version_list
- ]
- run_pull_in_parallel(
- dry_run=dry_run,
- parallelism=parallelism,
- image_params_list=ci_image_params_list,
- python_version_list=python_version_list,
- verbose=verbose,
- verify_image=verify_image,
- wait_for_image=wait_for_image,
- tag_as_latest=tag_as_latest,
- extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (),
- )
- else:
- image_params = BuildCiParams(image_tag=image_tag, python=python, github_repository=github_repository)
- return_code, info = run_pull_image(
- image_params=image_params,
- dry_run=dry_run,
- verbose=verbose,
- wait_for_image=wait_for_image,
- tag_as_latest=tag_as_latest,
- poll_time=10.0,
- )
- if return_code != 0:
- console.print(f"[red]There was an error when pulling CI image: {info}[/]")
- sys.exit(return_code)
-
-
-@main.command(
- name='verify-image',
- context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ),
-)
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_image_tag
-@option_image_name
-@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def verify_image(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- image_name: str,
- image_tag: str,
- extra_pytest_args: Tuple,
-):
- """Verify CI image."""
- if image_name is None:
- build_params = BuildCiParams(python=python, image_tag=image_tag, github_repository=github_repository)
- image_name = build_params.airflow_image_name_with_tag
- console.print(f"[bright_blue]Verifying CI image: {image_name}[/]")
- return_code, info = verify_an_image(
- image_name=image_name,
- verbose=verbose,
- dry_run=dry_run,
- image_type='CI',
- extra_pytest_args=extra_pytest_args,
- )
- sys.exit(return_code)
-
-
-@option_verbose
-@option_dry_run
-@option_answer
-@option_with_ci_group
-@main.command(name='build-prod-image')
-@option_python
-@option_build_multiple_images
-@option_python_versions
-@option_upgrade_to_newer_dependencies
-@option_platform
-@option_debian_version
-@option_github_repository
-@option_github_token
-@option_github_username
-@option_login_to_github_registry
-@option_docker_cache
-@option_image_tag
-@option_prepare_buildx_cache
-@option_push_image
-@option_empty_image
-@click.option(
- '--installation-method',
- help="Install Airflow from: sources or PyPI.",
- type=BetterChoice(ALLOWED_INSTALLATION_METHODS),
-)
-@option_install_providers_from_sources
-@click.option(
- '--install-from-docker-context-files',
- help='Install wheels from local docker-context-files when building image.',
- is_flag=True,
-)
-@click.option(
- '--cleanup-docker-context-files',
- help='Clean up docker context files before running build.',
- is_flag=True,
-)
-@click.option(
- '--extras', default=",".join(DEFAULT_EXTRAS), show_default=True, help="Extras to install by default."
-)
-@click.option('--disable-mysql-client-installation', help="Do not install MySQL client.", is_flag=True)
-@click.option('--disable-mssql-client-installation', help="Do not install MsSQl client.", is_flag=True)
-@click.option('--disable-postgres-client-installation', help="Do not install Postgres client.", is_flag=True)
-@click.option(
- '--disable-airflow-repo-cache',
- help="Disable cache from Airflow repository during building.",
- is_flag=True,
-)
-@click.option('--disable-pypi', help="Disable PyPI during building.", is_flag=True)
-@click.option(
- '--install-airflow-reference',
- help="Install Airflow using GitHub tag or branch.",
-)
-@click.option('-V', '--install-airflow-version', help="Install version of Airflow from PyPI.")
-@option_additional_extras
-@option_additional_dev_apt_deps
-@option_additional_runtime_apt_deps
-@option_additional_python_deps
-@option_additional_dev_apt_command
-@option_additional_dev_apt_env
-@option_additional_runtime_apt_env
-@option_additional_runtime_apt_command
-@option_dev_apt_command
-@option_dev_apt_deps
-@option_runtime_apt_command
-@option_runtime_apt_deps
-def build_prod_image(
- verbose: bool,
- dry_run: bool,
- with_ci_group: bool,
- build_multiple_images: bool,
- python_versions: str,
- answer: Optional[str],
- **kwargs,
-):
- """
- Build Production image. Include building multiple images for all or selected Python versions sequentially.
- """
-
- def run_build(prod_image_params: BuildProdParams) -> None:
- return_code, info = build_production_image(
- verbose=verbose, dry_run=dry_run, with_ci_group=with_ci_group, prod_image_params=prod_image_params
- )
- if return_code != 0:
- console.print(f"[red]Error when building image! {info}")
- sys.exit(return_code)
-
- set_forced_answer(answer)
- parameters_passed = filter_out_none(**kwargs)
- if build_multiple_images:
- with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
- python_version_list = get_python_version_list(python_versions)
- for python in python_version_list:
- params = BuildProdParams(**parameters_passed)
- params.python = python
- params.answer = answer
- run_build(prod_image_params=params)
- else:
- params = BuildProdParams(**parameters_passed)
- run_build(prod_image_params=params)
-
-
-@main.command(name='pull-prod-image')
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_run_in_parallel
-@option_parallelism
-@option_python_versions
-@option_image_tag
-@option_wait_for_image
-@option_tag_as_latest
-@option_verify_image
-@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def pull_prod_image(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- run_in_parallel: bool,
- parallelism: int,
- python_versions: str,
- image_tag: Optional[str],
- wait_for_image: bool,
- tag_as_latest: bool,
- verify_image: bool,
- extra_pytest_args: Tuple,
-):
- """Pull and optionally verify Production images - possibly in parallel for all Python versions."""
- if run_in_parallel:
- python_version_list = get_python_version_list(python_versions)
- prod_image_params_list = [
- BuildProdParams(image_tag=image_tag, python=python, github_repository=github_repository)
- for python in python_version_list
- ]
- run_pull_in_parallel(
- dry_run=dry_run,
- parallelism=parallelism,
- image_params_list=prod_image_params_list,
- python_version_list=python_version_list,
- verbose=verbose,
- verify_image=verify_image,
- wait_for_image=wait_for_image,
- tag_as_latest=tag_as_latest,
- extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (),
- )
- else:
- image_params = BuildProdParams(
- image_tag=image_tag, python=python, github_repository=github_repository
- )
- return_code, info = run_pull_image(
- image_params=image_params,
- dry_run=dry_run,
- verbose=verbose,
- wait_for_image=wait_for_image,
- tag_as_latest=tag_as_latest,
- poll_time=10.0,
- )
- if return_code != 0:
- console.print(f"[red]There was an error when pulling PROD image: {info}[/]")
- sys.exit(return_code)
-
-
-@main.command(
- name='verify-prod-image',
- context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ),
-)
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_image_tag
-@option_image_name
-@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def verify_prod_image(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- image_name: str,
- image_tag: str,
- extra_pytest_args: Tuple,
-):
- """Verify Production image."""
- if image_name is None:
- build_params = BuildProdParams(
- python=python, image_tag=image_tag, github_repository=github_repository
- )
- image_name = build_params.airflow_image_name_with_tag
- console.print(f"[bright_blue]Verifying PROD image: {image_name}[/]")
- return_code, info = verify_an_image(
- image_name=image_name,
- verbose=verbose,
- dry_run=dry_run,
- image_type='PROD',
- extra_pytest_args=extra_pytest_args,
- )
- sys.exit(return_code)
-
-
-@main.command(
- name='docker-compose-tests',
- context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ),
-)
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_image_tag
-@option_image_name
-@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
-def docker_compose_tests(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- image_name: str,
- image_tag: str,
- extra_pytest_args: Tuple,
-):
- """Run docker-compose tests."""
- if image_name is None:
- build_params = BuildProdParams(
- python=python, image_tag=image_tag, github_repository=github_repository
- )
- image_name = build_params.airflow_image_name_with_tag
- console.print(f"[bright_blue]Running docker-compose with PROD image: {image_name}[/]")
- return_code, info = run_docker_compose_tests(
- image_name=image_name,
- verbose=verbose,
- dry_run=dry_run,
- extra_pytest_args=extra_pytest_args,
- )
- sys.exit(return_code)
-
-
-BREEZE_COMMENT = "Added by Updated Airflow Breeze autocomplete setup"
-START_LINE = f"# START: {BREEZE_COMMENT}\n"
-END_LINE = f"# END: {BREEZE_COMMENT}\n"
-
-
-def remove_autogenerated_code(script_path: str):
- lines = Path(script_path).read_text().splitlines(keepends=True)
- new_lines = []
- pass_through = True
- for line in lines:
- if line == START_LINE:
- pass_through = False
- continue
- if line.startswith(END_LINE):
- pass_through = True
- continue
- if pass_through:
- new_lines.append(line)
- Path(script_path).write_text("".join(new_lines))
-
-
-def backup(script_path_file: Path):
- shutil.copy(str(script_path_file), str(script_path_file) + ".bak")
-
-
-def write_to_shell(command_to_execute: str, dry_run: bool, script_path: str, force_setup: bool) -> bool:
- skip_check = False
- script_path_file = Path(script_path)
- if not script_path_file.exists():
- skip_check = True
- if not skip_check:
- if BREEZE_COMMENT in script_path_file.read_text():
- if not force_setup:
- console.print(
- "\n[bright_yellow]Autocompletion is already setup. Skipping. "
- "You can force autocomplete installation by adding --force[/]\n"
- )
- return False
- else:
- backup(script_path_file)
- remove_autogenerated_code(script_path)
- text = ''
- if script_path_file.exists():
- console.print(f"\nModifying the {script_path} file!\n")
- console.print(f"\nCopy of the original file is held in {script_path}.bak !\n")
- if not dry_run:
- backup(script_path_file)
- text = script_path_file.read_text()
- else:
- console.print(f"\nCreating the {script_path} file!\n")
- if not dry_run:
- script_path_file.write_text(
- text
- + ("\n" if not text.endswith("\n") else "")
- + START_LINE
- + command_to_execute
- + "\n"
- + END_LINE
- )
- else:
- console.print(f"[bright_blue]The autocomplete script would be added to {script_path}[/]")
- console.print(
- f"\n[bright_yellow]IMPORTANT!!!! Please exit and re-enter your shell or run:[/]"
- f"\n\n source {script_path}\n"
- )
- return True
-
-
-@option_verbose
-@option_dry_run
-@click.option(
- '-f',
- '--force',
- is_flag=True,
- help='Force autocomplete setup even if already setup before (overrides the setup).',
-)
-@option_answer
-@main.command(name='setup-autocomplete')
-def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Optional[str]):
- """
- Enables autocompletion of breeze commands.
- """
- set_forced_answer(answer)
- # Determine if the shell is bash/zsh/powershell. It helps to build the autocomplete path
- detected_shell = os.environ.get('SHELL')
- detected_shell = None if detected_shell is None else detected_shell.split(os.sep)[-1]
- if detected_shell not in ['bash', 'zsh', 'fish']:
- console.print(f"\n[red] The shell {detected_shell} is not supported for autocomplete![/]\n")
- sys.exit(1)
- console.print(f"Installing {detected_shell} completion for local user")
- autocomplete_path = (
- AIRFLOW_SOURCES_ROOT / "dev" / "breeze" / "autocomplete" / f"{NAME}-complete-{detected_shell}.sh"
- )
- console.print(f"[bright_blue]Activation command script is available here: {autocomplete_path}[/]\n")
- console.print(f"[bright_yellow]We need to add above script to your {detected_shell} profile.[/]\n")
- given_answer = user_confirm("Should we proceed ?", default_answer=Answer.NO, timeout=3)
- if given_answer == Answer.YES:
- if detected_shell == 'bash':
- script_path = str(Path('~').expanduser() / '.bash_completion')
- command_to_execute = f"source {autocomplete_path}"
- write_to_shell(command_to_execute, dry_run, script_path, force)
- elif detected_shell == 'zsh':
- script_path = str(Path('~').expanduser() / '.zshrc')
- command_to_execute = f"source {autocomplete_path}"
- write_to_shell(command_to_execute, dry_run, script_path, force)
- elif detected_shell == 'fish':
- # Include steps for fish shell
- script_path = str(Path('~').expanduser() / f'.config/fish/completions/{NAME}.fish')
- if os.path.exists(script_path) and not force:
- console.print(
- "\n[bright_yellow]Autocompletion is already setup. Skipping. "
- "You can force autocomplete installation by adding --force/]\n"
- )
- else:
- with open(autocomplete_path) as source_file, open(script_path, 'w') as destination_file:
- for line in source_file:
- destination_file.write(line)
- else:
- # Include steps for powershell
- subprocess.check_call(['powershell', 'Set-ExecutionPolicy Unrestricted -Scope CurrentUser'])
- script_path = (
- subprocess.check_output(['powershell', '-NoProfile', 'echo $profile']).decode("utf-8").strip()
- )
- command_to_execute = f". {autocomplete_path}"
- write_to_shell(command_to_execute, dry_run, script_path, force)
- elif given_answer == Answer.NO:
- console.print(
- "\nPlease follow the https://click.palletsprojects.com/en/8.1.x/shell-completion/ "
- "to setup autocompletion for breeze manually if you want to use it.\n"
- )
- else:
- sys.exit(0)
-
-
-@main.command(name='config')
-@option_python
-@option_backend
-@option_postgres_version
-@option_mysql_version
-@option_mssql_version
-@click.option('-C/-c', '--cheatsheet/--no-cheatsheet', help="Enable/disable cheatsheet.", default=None)
-@click.option('-A/-a', '--asciiart/--no-asciiart', help="Enable/disable ASCIIart.", default=None)
-def change_config(
- python: str,
- backend: str,
- postgres_version: str,
- mysql_version: str,
- mssql_version: str,
- cheatsheet: bool,
- asciiart: bool,
-):
- """
- Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).
- """
- asciiart_file = "suppress_asciiart"
- cheatsheet_file = "suppress_cheatsheet"
-
- if asciiart is not None:
- if asciiart:
- delete_cache(asciiart_file)
- console.print('[bright_blue]Enable ASCIIART![/]')
- else:
- touch_cache_file(asciiart_file)
- console.print('[bright_blue]Disable ASCIIART![/]')
- if cheatsheet is not None:
- if cheatsheet:
- delete_cache(cheatsheet_file)
- console.print('[bright_blue]Enable Cheatsheet[/]')
- elif cheatsheet is not None:
- touch_cache_file(cheatsheet_file)
- console.print('[bright_blue]Disable Cheatsheet[/]')
-
- def get_status(file: str):
- return "disabled" if check_if_cache_exists(file) else "enabled"
-
- console.print()
- console.print("[bright_blue]Current configuration:[/]")
- console.print()
- console.print(f"[bright_blue]* Python: {python}[/]")
- console.print(f"[bright_blue]* Backend: {backend}[/]")
- console.print()
- console.print(f"[bright_blue]* Postgres version: {postgres_version}[/]")
- console.print(f"[bright_blue]* MySQL version: {mysql_version}[/]")
- console.print(f"[bright_blue]* MsSQL version: {mssql_version}[/]")
- console.print()
- console.print(f"[bright_blue]* ASCIIART: {get_status(asciiart_file)}[/]")
- console.print(f"[bright_blue]* Cheatsheet: {get_status(cheatsheet_file)}[/]")
- console.print()
-
-
-@dataclass
-class DocParams:
- package_filter: Tuple[str]
- docs_only: bool
- spellcheck_only: bool
-
- @property
- def args_doc_builder(self) -> List[str]:
- doc_args = []
- if self.docs_only:
- doc_args.append("--docs-only")
- if self.spellcheck_only:
- doc_args.append("--spellcheck-only")
- if self.package_filter and len(self.package_filter) > 0:
- for single_filter in self.package_filter:
- doc_args.extend(["--package-filter", single_filter])
- return doc_args
-
-
-@main.command(name='build-docs')
-@option_verbose
-@option_dry_run
-@option_github_repository
-@click.option('-d', '--docs-only', help="Only build documentation.", is_flag=True)
-@click.option('-s', '--spellcheck-only', help="Only run spell checking.", is_flag=True)
-@click.option(
- '-p',
- '--package-filter',
- help="List of packages to consider.",
- type=BetterChoice(get_available_packages()),
- multiple=True,
-)
-def build_docs(
- verbose: bool,
- dry_run: bool,
- github_repository: str,
- docs_only: bool,
- spellcheck_only: bool,
- package_filter: Tuple[str],
-):
- """Build documentation in the container."""
- params = BuildCiParams(github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
- ci_image_name = params.airflow_image_name
- doc_builder = DocParams(
- package_filter=package_filter,
- docs_only=docs_only,
- spellcheck_only=spellcheck_only,
- )
- extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED)
- env = construct_env_variables_docker_compose_command(params)
- cmd = [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "GITHUB_ACTIONS=",
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "--pull",
- "never",
- ci_image_name,
- "/opt/airflow/scripts/in_container/run_docs_build.sh",
- *doc_builder.args_doc_builder,
- ]
- process = run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False)
- sys.exit(process.returncode)
-
-
-@main.command(
- name="static-checks",
- help="Run static checks.",
- context_settings=dict(
- ignore_unknown_options=True,
- allow_extra_args=True,
- ),
-)
-@click.option(
- '-t',
- '--type',
- help="Type(s) of the static checks to run (multiple can be added).",
- type=BetterChoice(PRE_COMMIT_LIST),
- multiple=True,
-)
-@click.option('-a', '--all-files', help="Run checks on all files.", is_flag=True)
-@click.option('-f', '--files', help="List of files to run the checks on.", multiple=True)
-@click.option(
- '-s', '--show-diff-on-failure', help="Show diff for files modified by the checks.", is_flag=True
-)
-@click.option(
- '-c',
- '--last-commit',
- help="Run checks for all files in last commit. Mutually exclusive with --commit-ref.",
- is_flag=True,
-)
-@click.option(
- '-r',
- '--commit-ref',
- help="Run checks for this commit reference only "
- "(can be any git commit-ish reference). "
- "Mutually exclusive with --last-commit.",
-)
-@option_verbose
-@option_dry_run
-@option_github_repository
-@click.argument('precommit_args', nargs=-1, type=click.UNPROCESSED)
-def static_checks(
- verbose: bool,
- dry_run: bool,
- github_repository: str,
- all_files: bool,
- show_diff_on_failure: bool,
- last_commit: bool,
- commit_ref: str,
- type: Tuple[str],
- files: bool,
- precommit_args: Tuple,
-):
- assert_pre_commit_installed(verbose=verbose)
- command_to_execute = [sys.executable, "-m", "pre_commit", 'run']
- if last_commit and commit_ref:
- console.print("\n[red]You cannot specify both --last-commit and --commit-ref[/]\n")
- sys.exit(1)
- for single_check in type:
- command_to_execute.append(single_check)
- if all_files:
- command_to_execute.append("--all-files")
- if show_diff_on_failure:
- command_to_execute.append("--show-diff-on-failure")
- if last_commit:
- command_to_execute.extend(["--from-ref", "HEAD^", "--to-ref", "HEAD"])
- if commit_ref:
- command_to_execute.extend(["--from-ref", f"{commit_ref}^", "--to-ref", f"{commit_ref}"])
- if files:
- command_to_execute.append("--files")
- if verbose or dry_run:
- command_to_execute.append("--verbose")
- if precommit_args:
- command_to_execute.extend(precommit_args)
- env = os.environ.copy()
- env['GITHUB_REPOSITORY'] = github_repository
- static_checks_result = run_command(
- command_to_execute,
- verbose=verbose,
- dry_run=dry_run,
- check=False,
- no_output_dump_on_exception=True,
- text=True,
- env=env,
- )
- if static_checks_result.returncode != 0:
- console.print("[red]There were errors during pre-commit check. They should be fixed[/]")
- sys.exit(static_checks_result.returncode)
-
-
-@main.command(name="stop", help="Stop running breeze environment.")
-@option_verbose
-@option_dry_run
-@click.option(
- "-p",
- "--preserve-volumes",
- help="Skip removing volumes when stopping Breeze.",
- is_flag=True,
-)
-def stop(verbose: bool, dry_run: bool, preserve_volumes: bool):
- command_to_execute = ['docker-compose', 'down', "--remove-orphans"]
- if not preserve_volumes:
- command_to_execute.append("--volumes")
- shell_params = ShellParams(verbose=verbose)
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables)
-
-
-@click.option(
- '-f',
- '--force',
- is_flag=True,
- help='Force upgrade without asking question to the user.',
-)
-@click.option(
- '-a',
- '--use-current-airflow-sources',
- is_flag=True,
- help='Use current workdir Airflow sources for upgrade'
- + (f" rather than from {get_installation_airflow_sources()}." if not output_file_for_recording else "."),
-)
-@main.command(
- name='self-upgrade',
- help="Self upgrade Breeze. By default it re-installs Breeze "
- f"from {get_installation_airflow_sources()}."
- if not output_file_for_recording
- else "Self upgrade Breeze.",
-)
-def self_upgrade(force: bool, use_current_airflow_sources: bool):
- if use_current_airflow_sources:
- airflow_sources: Optional[Path] = get_used_airflow_sources()
- else:
- airflow_sources = get_installation_airflow_sources()
- if airflow_sources is not None:
- breeze_sources = airflow_sources / "dev" / "breeze"
- if force:
- reinstall_breeze(breeze_sources)
- else:
- ask_to_reinstall_breeze(breeze_sources)
- else:
- warn_non_editable()
- sys.exit(1)
-
-
-@main.command(
- name='prepare-airflow-package',
- help="Prepare sdist/whl package of Airflow.",
-)
-@option_verbose
-@option_dry_run
-@option_github_repository
-@option_with_ci_group
-@option_package_format
-@option_version_suffix_for_pypi
-def prepare_airflow_packages(
- verbose: bool,
- dry_run: bool,
- github_repository: str,
- with_ci_group: bool,
- package_format: str,
- version_suffix_for_pypi: str,
-):
- shell_params = ShellParams(
- verbose=verbose, github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION
- )
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- env_variables['INSTALL_PROVIDERS_FROM_SOURCES'] = "false"
- extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
- with ci_group("Prepare Airflow package", enabled=with_ci_group):
- run_command(
- [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "-e",
- f"PACKAGE_FORMAT={package_format}",
- "-e",
- f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh",
- ],
- verbose=verbose,
- dry_run=dry_run,
- env=env_variables,
- )
-
-
-@main.command(
- name='prepare-provider-documentation',
- help="Prepare CHANGELOG, README and COMMITS information for providers.",
-)
-@option_verbose
-@option_dry_run
-@option_github_repository
-@option_with_ci_group
-@option_answer
-@click.option(
- '--skip-package-verification',
- help="Skip Provider package verification.",
- is_flag=True,
-)
-@argument_packages
-def prepare_provider_documentation(
- verbose: bool,
- dry_run: bool,
- github_repository: str,
- with_ci_group: bool,
- answer: Optional[str],
- skip_package_verification: bool,
- packages: List[str],
-):
- set_forced_answer(answer)
- shell_params = ShellParams(
- verbose=verbose,
- mount_sources=MOUNT_ALL,
- skip_package_verification=skip_package_verification,
- github_repository=github_repository,
- python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- answer=answer,
- )
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
- if answer and answer.lower() in ['y', 'yes']:
- term_flag = "-t"
- else:
- term_flag = "-it"
- cmd_to_run = [
- "docker",
- "run",
- term_flag,
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh",
- ]
- if packages:
- cmd_to_run.extend(packages)
- with ci_group("Prepare provider documentation", enabled=with_ci_group):
- run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
-
-
-@main.command(
- name='prepare-provider-packages',
- help="Prepare sdist/whl packages of Airflow Providers.",
-)
-@option_verbose
-@option_dry_run
-@option_github_repository
-@option_with_ci_group
-@option_package_format
-@option_version_suffix_for_pypi
-@click.option(
- '--package-list-file',
- type=click.File('rt'),
- help='Read list of packages from text file (one package per line)',
-)
-@argument_packages
-def prepare_provider_packages(
- verbose: bool,
- dry_run: bool,
- github_repository: str,
- with_ci_group: bool,
- package_format: str,
- version_suffix_for_pypi: str,
- package_list_file: IO,
- packages: Tuple[str, ...],
-):
- packages_list = list(packages)
- if package_list_file:
- packages_list.extend([package.strip() for package in package_list_file.readlines()])
- shell_params = ShellParams(
- verbose=verbose,
- mount_sources=MOUNT_ALL,
- github_repository=github_repository,
- python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- )
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
- cmd_to_run = [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "-e",
- f"PACKAGE_FORMAT={package_format}",
- "-e",
- f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_prepare_provider_packages.sh",
- ]
- cmd_to_run.extend(packages_list)
- with ci_group("Prepare provider packages", enabled=with_ci_group):
- run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
-
-
-@main.command(
- name='generate-constraints',
- help="Generates pinned constraint files with all extras from setup.py in parallel.",
-)
-@click.option(
- '--generate-constraints-mode',
- type=BetterChoice(ALLOWED_GENERATE_CONSTRAINTS_MODES),
- default=ALLOWED_GENERATE_CONSTRAINTS_MODES[0],
- show_default=True,
- help='Mode of generating constraints',
-)
-@option_verbose
-@option_dry_run
-@option_python
-@option_github_repository
-@option_run_in_parallel
-@option_parallelism
-@option_python_versions
-@option_image_tag
-@option_answer
-@option_with_ci_group
-def generate_constraints(
- verbose: bool,
- dry_run: bool,
- python: str,
- github_repository: str,
- run_in_parallel: bool,
- parallelism: int,
- python_versions: str,
- image_tag: str,
- answer: Optional[str],
- generate_constraints_mode: str,
- with_ci_group: bool,
-):
- set_forced_answer(answer)
- if run_in_parallel:
- given_answer = user_confirm(
- f"Did you build all CI images {python_versions} with --upgrade-to-newer-dependencies "
- f"true flag set?",
- timeout=None,
- )
- else:
- given_answer = user_confirm(
- f"Did you build CI image {python} with --upgrade-to-newer-dependencies true flag set?",
- timeout=None,
- )
- if given_answer != Answer.YES:
- if run_in_parallel:
- console.print("\n[yellow]Use this command to build the images:[/]\n")
- console.print(
- f" breeze build-image --run-in-parallel --python-versions '{python_versions}' "
- f"--upgrade-to-newer-dependencies true\n"
- )
- else:
- shell_params = ShellParams(
- image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
- )
- console.print("\n[yellow]Use this command to build the image:[/]\n")
- console.print(
- f" breeze build-image --python'{shell_params.python}' "
- f"--upgrade-to-newer-dependencies true\n"
- )
- sys.exit(1)
- if run_in_parallel:
- python_version_list = get_python_version_list(python_versions)
- shell_params_list = [
- ShellParams(
- image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
- )
- for python in python_version_list
- ]
- with ci_group(f"Generating constraints with {generate_constraints_mode}", enabled=with_ci_group):
- run_generate_constraints_in_parallel(
- shell_params_list=shell_params_list,
- parallelism=parallelism,
- dry_run=dry_run,
- verbose=verbose,
- python_version_list=python_version_list,
- generate_constraints_mode=generate_constraints_mode,
- )
- else:
- with ci_group(f"Generating constraints with {generate_constraints_mode}", enabled=with_ci_group):
- shell_params = ShellParams(
- image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
- )
- return_code, info = run_generate_constraints(
- shell_params=shell_params,
- dry_run=dry_run,
- verbose=verbose,
- generate_constraints_mode=generate_constraints_mode,
- )
- if return_code != 0:
- console.print(f"[red]There was an error when generating constraints: {info}[/]")
- sys.exit(return_code)
-
-
-@main.command(name="free-space", help="Free space for jobs run in CI.")
-@option_verbose
-@option_dry_run
-@option_answer
-def free_space(verbose: bool, dry_run: bool, answer: str):
- set_forced_answer(answer)
- if user_confirm("Are you sure to run free-space and perform cleanup?", timeout=None) == Answer.YES:
- run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run)
- run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run)
- run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False)
- run_command(
- ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run
- )
- run_command(["df", "-h"], verbose=verbose, dry_run=dry_run)
- run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False)
-
-
-@main.command(name="resource-check", help="Check if available docker resources are enough.")
-@option_verbose
-@option_dry_run
-def resource_check(verbose: bool, dry_run: bool):
- shell_params = ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
- check_docker_resources(verbose, shell_params.airflow_image_name, dry_run=dry_run)
-
-
-option_timezone = click.option(
- "--timezone",
- default="UTC",
- type=str,
- help="Timezone to use during the check",
-)
-
-option_updated_on_or_after = click.option(
- "--updated-on-or-after",
- type=str,
- help="Date when the release was updated after",
-)
-
-option_max_age = click.option(
- "--max-age",
- type=int,
- default=3,
- help="Max age of the last release (used if no updated-on-or-after if specified)",
-)
-
-option_branch = click.option(
- "--constraints-branch",
- default='constraints-main',
- help="Constraint branch to use to find newer dependencies",
-)
-
-
-@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.")
-@option_timezone
-@option_branch
-@option_python
-@option_updated_on_or_after
-@option_max_age
-def breeze_find_newer_dependencies(
- constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int
-):
- return find_newer_dependencies(
- constraints_branch=constraints_branch,
- python=python,
- timezone=timezone,
- updated_on_or_after=updated_on_or_after,
- max_age=max_age,
- )
-
-
-@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.")
-@option_verbose
-@option_dry_run
-def fix_ownership(verbose: bool, dry_run: bool):
- shell_params = ShellParams(
- verbose=verbose,
- mount_sources=MOUNT_ALL,
- python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- )
- extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
- env = construct_env_variables_docker_compose_command(shell_params)
- cmd = [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "GITHUB_ACTIONS=",
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_fix_ownership.sh",
- ]
- run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False)
- # Always succeed
- sys.exit(0)
-
-
-@main.command(
- name="cleanup",
- help="Cleans the cache of parameters, docker cache and optionally - currently downloaded images.",
-)
-@click.option(
- '--all',
- is_flag=True,
- help='Also remove currently downloaded Breeze images.',
-)
-@option_verbose
-@option_answer
-@option_dry_run
-def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, answer: Optional[str]):
- set_forced_answer(answer)
- if all:
- console.print(
- "\n[bright_yellow]Removing cache of parameters, clean up docker cache "
- "and remove locally downloaded images[/]"
- )
- else:
- console.print("\n[bright_yellow]Removing cache of parameters, and cleans up docker cache[/]")
- if all:
- docker_images_command_to_execute = [
- 'docker',
- 'images',
- '--filter',
- 'label=org.apache.airflow.image',
- '--format',
- '{{.Repository}}:{{.Tag}}',
- ]
- command_result = run_command(
- docker_images_command_to_execute, verbose=verbose, text=True, capture_output=True
- )
- images = command_result.stdout.splitlines() if command_result and command_result.stdout else []
- if images:
- console.print("[light_blue]Removing images:[/]")
- for image in images:
- console.print(f"[light_blue] * {image}[/]")
- console.print()
- docker_rmi_command_to_execute = [
- 'docker',
- 'rmi',
- '--force',
- ]
- docker_rmi_command_to_execute.extend(images)
- given_answer = user_confirm("Are you sure?", timeout=None)
- if given_answer == Answer.YES:
- run_command(docker_rmi_command_to_execute, verbose=verbose, dry_run=dry_run, check=False)
- elif given_answer == Answer.QUIT:
- sys.exit(0)
- else:
- console.print("[light_blue]No locally downloaded images to remove[/]\n")
- console.print("Pruning docker images")
- given_answer = user_confirm("Are you sure?", timeout=None)
- if given_answer == Answer.YES:
- system_prune_command_to_execute = ['docker', 'system', 'prune']
- run_command(system_prune_command_to_execute, verbose=verbose, dry_run=dry_run, check=False)
- elif given_answer == Answer.QUIT:
- sys.exit(0)
- console.print(f"Removing build cache dir ${BUILD_CACHE_DIR}")
- given_answer = user_confirm("Are you sure?", timeout=None)
- if given_answer == Answer.YES:
- if not dry_run:
- shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True)
- elif given_answer == Answer.QUIT:
- sys.exit(0)
-
-
-help_console = None
-
-
-def enable_recording_of_help_output(path: str, title: Optional[str], width: Optional[str]):
- if not title:
- title = "Breeze screenshot"
- if not width:
- width_int = 120
- else:
- width_int = int(width)
-
- def save_ouput_as_svg():
- if help_console:
- help_console.save_svg(path=path, title=title)
-
- class RecordingConsole(rich.console.Console):
- def __init__(self, **kwargs):
- super().__init__(record=True, width=width_int, force_terminal=True, **kwargs)
- global help_console
- help_console = self
-
- atexit.register(save_ouput_as_svg)
- click.rich_click.MAX_WIDTH = width_int
- click.formatting.FORCED_WIDTH = width_int - 2
- click.rich_click.COLOR_SYSTEM = "standard"
- # monkeypatch rich_click console to record help (rich_click does not allow passing extra args to console)
- click.rich_click.Console = RecordingConsole
-
-
-if output_file_for_recording and not in_autocomplete():
- enable_recording_of_help_output(
- path=output_file_for_recording,
- title=os.environ.get('RECORD_BREEZE_TITLE'),
- width=os.environ.get('RECORD_BREEZE_WIDTH'),
- )
-
if __name__ == '__main__':
main()
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py b/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
new file mode 100644
index 0000000000..21cc93f1f0
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
@@ -0,0 +1,332 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from typing import Optional, Tuple
+
+import click
+
+from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
+from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
+from airflow_breeze.commands.common_options import (
+ option_additional_dev_apt_command,
+ option_additional_dev_apt_deps,
+ option_additional_dev_apt_env,
+ option_additional_extras,
+ option_additional_python_deps,
+ option_additional_runtime_apt_command,
+ option_additional_runtime_apt_deps,
+ option_additional_runtime_apt_env,
+ option_answer,
+ option_build_multiple_images,
+ option_debian_version,
+ option_dev_apt_command,
+ option_dev_apt_deps,
+ option_docker_cache,
+ option_dry_run,
+ option_empty_image,
+ option_force_build,
+ option_github_repository,
+ option_github_token,
+ option_github_username,
+ option_image_name,
+ option_image_tag,
+ option_install_providers_from_sources,
+ option_login_to_github_registry,
+ option_parallelism,
+ option_platform,
+ option_prepare_buildx_cache,
+ option_push_image,
+ option_python,
+ option_python_versions,
+ option_run_in_parallel,
+ option_runtime_apt_command,
+ option_runtime_apt_deps,
+ option_tag_as_latest,
+ option_upgrade_to_newer_dependencies,
+ option_verbose,
+ option_verify_image,
+ option_wait_for_image,
+ option_with_ci_group,
+)
+from airflow_breeze.commands.main import main
+from airflow_breeze.utils.ci_group import ci_group
+from airflow_breeze.utils.confirm import set_forced_answer
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
+from airflow_breeze.utils.python_versions import get_python_version_list
+from airflow_breeze.utils.run_tests import verify_an_image
+from airflow_breeze.utils.run_utils import filter_out_none
+
+CI_IMAGE_TOOLS_COMMANDS = {
+ "name": "CI Image tools",
+ "commands": [
+ "build-image",
+ "pull-image",
+ "verify-image",
+ ],
+}
+
+CI_IMAGE_TOOLS_PARAMETERS = {
+ "breeze build-image": [
+ {
+ "name": "Basic usage",
+ "options": [
+ "--python",
+ "--upgrade-to-newer-dependencies",
+ "--debian-version",
+ "--image-tag",
+ "--docker-cache",
+ "--force-build",
+ ],
+ },
+ {
+ "name": "Building multiple images",
+ "options": [
+ "--build_multiple_images",
+ "--python-versions",
+ ],
+ },
+ {
+ "name": "Advanced options (for power users)",
+ "options": [
+ "--install-providers-from-sources",
+ "--additional-python-deps",
+ "--runtime-apt-deps",
+ "--runtime-apt-command",
+ "--additional-extras",
+ "--additional-runtime-apt-deps",
+ "--additional-runtime-apt-env",
+ "--additional-runtime-apt-command",
+ "--additional-dev-apt-deps",
+ "--additional-dev-apt-env",
+ "--additional-dev-apt-command",
+ "--dev-apt-deps",
+ "--dev-apt-command",
+ ],
+ },
+ {
+ "name": "Preparing cache and push (for maintainers and CI)",
+ "options": [
+ "--platform",
+ "--prepare-buildx-cache",
+ "--push-image",
+ "--empty-image",
+ "--github-token",
+ "--github-username",
+ "--login-to-github-registry",
+ ],
+ },
+ ],
+ "breeze pull-image": [
+ {
+ "name": "Pull image flags",
+ "options": [
+ "--image-tag",
+ "--python",
+ "--verify-image",
+ "--wait-for-image",
+ "--tag-as-latest",
+ ],
+ },
+ {
+ "name": "Parallel running",
+ "options": [
+ "--run-in-parallel",
+ "--parallelism",
+ "--python-versions",
+ ],
+ },
+ ],
+ "breeze verify-image": [
+ {
+ "name": "Verify image flags",
+ "options": [
+ "--image-name",
+ "--python",
+ "--image-tag",
+ ],
+ }
+ ],
+}
+
+
+@main.command(name='build-image')
+@option_github_repository
+@option_verbose
+@option_dry_run
+@option_with_ci_group
+@option_answer
+@option_python
+@option_build_multiple_images
+@option_python_versions
+@option_upgrade_to_newer_dependencies
+@option_platform
+@option_debian_version
+@option_github_token
+@option_github_username
+@option_login_to_github_registry
+@option_docker_cache
+@option_image_tag
+@option_prepare_buildx_cache
+@option_push_image
+@option_empty_image
+@option_install_providers_from_sources
+@option_additional_extras
+@option_additional_dev_apt_deps
+@option_additional_runtime_apt_deps
+@option_additional_python_deps
+@option_additional_dev_apt_command
+@option_runtime_apt_command
+@option_additional_dev_apt_env
+@option_additional_runtime_apt_env
+@option_additional_runtime_apt_command
+@option_dev_apt_command
+@option_dev_apt_deps
+@option_runtime_apt_command
+@option_runtime_apt_deps
+@option_force_build
+def build_image(
+ verbose: bool,
+ dry_run: bool,
+ with_ci_group: bool,
+ build_multiple_images: bool,
+ python_versions: str,
+ answer: str,
+ **kwargs,
+):
+ """Build CI image. Include building multiple images for all python versions (sequentially)."""
+
+ def run_build(ci_image_params: BuildCiParams) -> None:
+ return_code, info = build_ci_image(
+ verbose=verbose, dry_run=dry_run, with_ci_group=with_ci_group, ci_image_params=ci_image_params
+ )
+ if return_code != 0:
+ console.print(f"[red]Error when building image! {info}")
+ sys.exit(return_code)
+
+ set_forced_answer(answer)
+ parameters_passed = filter_out_none(**kwargs)
+ if build_multiple_images:
+ with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
+ python_version_list = get_python_version_list(python_versions)
+ for python in python_version_list:
+ params = BuildCiParams(**parameters_passed)
+ params.python = python
+ params.answer = answer
+ run_build(ci_image_params=params)
+ else:
+ params = BuildCiParams(**parameters_passed)
+ run_build(ci_image_params=params)
+
+
+@main.command(name='pull-image')
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_run_in_parallel
+@option_parallelism
+@option_python_versions
+@option_verify_image
+@option_wait_for_image
+@option_tag_as_latest
+@option_image_tag
+@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
+def pull_image(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ run_in_parallel: bool,
+ python_versions: str,
+ parallelism: int,
+ image_tag: Optional[str],
+ wait_for_image: bool,
+ tag_as_latest: bool,
+ verify_image: bool,
+ extra_pytest_args: Tuple,
+):
+ """Pull and optionally verify CI images - possibly in parallel for all Python versions."""
+ if run_in_parallel:
+ python_version_list = get_python_version_list(python_versions)
+ ci_image_params_list = [
+ BuildCiParams(image_tag=image_tag, python=python, github_repository=github_repository)
+ for python in python_version_list
+ ]
+ run_pull_in_parallel(
+ dry_run=dry_run,
+ parallelism=parallelism,
+ image_params_list=ci_image_params_list,
+ python_version_list=python_version_list,
+ verbose=verbose,
+ verify_image=verify_image,
+ wait_for_image=wait_for_image,
+ tag_as_latest=tag_as_latest,
+ extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (),
+ )
+ else:
+ image_params = BuildCiParams(image_tag=image_tag, python=python, github_repository=github_repository)
+ return_code, info = run_pull_image(
+ image_params=image_params,
+ dry_run=dry_run,
+ verbose=verbose,
+ wait_for_image=wait_for_image,
+ tag_as_latest=tag_as_latest,
+ poll_time=10.0,
+ )
+ if return_code != 0:
+ console.print(f"[red]There was an error when pulling CI image: {info}[/]")
+ sys.exit(return_code)
+
+
+@main.command(
+ name='verify-image',
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+)
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_image_tag
+@option_image_name
+@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
+def verify_image(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ image_name: str,
+ image_tag: str,
+ extra_pytest_args: Tuple,
+):
+ """Verify CI image."""
+ if image_name is None:
+ build_params = BuildCiParams(python=python, image_tag=image_tag, github_repository=github_repository)
+ image_name = build_params.airflow_image_name_with_tag
+ console.print(f"[bright_blue]Verifying CI image: {image_name}[/]")
+ return_code, info = verify_an_image(
+ image_name=image_name,
+ verbose=verbose,
+ dry_run=dry_run,
+ image_type='CI',
+ extra_pytest_args=extra_pytest_args,
+ )
+ sys.exit(return_code)
diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py
new file mode 100644
index 0000000000..41016563b2
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/common_options.py
@@ -0,0 +1,403 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import multiprocessing as mp
+
+import click
+
+from airflow_breeze.commands.custom_param_types import BetterChoice, CacheableChoice, CacheableDefault
+from airflow_breeze.global_constants import (
+ ALLOWED_BACKENDS,
+ ALLOWED_BUILD_CACHE,
+ ALLOWED_DEBIAN_VERSIONS,
+ ALLOWED_EXECUTORS,
+ ALLOWED_INTEGRATIONS,
+ ALLOWED_MOUNT_OPTIONS,
+ ALLOWED_MSSQL_VERSIONS,
+ ALLOWED_MYSQL_VERSIONS,
+ ALLOWED_PACKAGE_FORMATS,
+ ALLOWED_PLATFORMS,
+ ALLOWED_POSTGRES_VERSIONS,
+ ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+ get_available_packages,
+)
+from airflow_breeze.utils.recording import output_file_for_recording
+
+option_verbose = click.option(
+ "-v", "--verbose", is_flag=True, help="Print verbose information about performed steps.", envvar='VERBOSE'
+)
+option_dry_run = click.option(
+ "-D",
+ "--dry-run",
+ is_flag=True,
+ help="If dry-run is set, commands are only printed, not executed.",
+ envvar='DRY_RUN',
+)
+option_answer = click.option(
+ "-a",
+ "--answer",
+ type=BetterChoice(['y', 'n', 'q', 'yes', 'no', 'quit']),
+ help="Force answer to questions.",
+ envvar='ANSWER',
+)
+option_github_repository = click.option(
+ '-g',
+ '--github-repository',
+ help='GitHub repository used to pull, push run images.',
+ default="apache/airflow",
+ show_default=True,
+ envvar='GITHUB_REPOSITORY',
+)
+option_python = click.option(
+ '-p',
+ '--python',
+ type=CacheableChoice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS),
+ default=CacheableDefault(value=ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0]),
+ show_default=True,
+ help='Python major/minor version used in Airflow image for images.',
+ envvar='PYTHON_MAJOR_MINOR_VERSION',
+)
+option_backend = click.option(
+ '-b',
+ '--backend',
+ type=CacheableChoice(ALLOWED_BACKENDS),
+ default=CacheableDefault(value=ALLOWED_BACKENDS[0]),
+ show_default=True,
+ help="Database backend to use.",
+ envvar='BACKEND',
+)
+option_integration = click.option(
+ '--integration',
+ help="Integration(s) to enable when running (can be more than one).",
+ type=BetterChoice(ALLOWED_INTEGRATIONS),
+ multiple=True,
+)
+option_postgres_version = click.option(
+ '-P',
+ '--postgres-version',
+ type=CacheableChoice(ALLOWED_POSTGRES_VERSIONS),
+ default=CacheableDefault(ALLOWED_POSTGRES_VERSIONS[0]),
+ show_default=True,
+ help="Version of Postgres used.",
+)
+option_mysql_version = click.option(
+ '-M',
+ '--mysql-version',
+ help="Version of MySQL used.",
+ type=CacheableChoice(ALLOWED_MYSQL_VERSIONS),
+ default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]),
+ show_default=True,
+)
+option_mssql_version = click.option(
+ '-S',
+ '--mssql-version',
+ help="Version of MsSQL used.",
+ type=CacheableChoice(ALLOWED_MSSQL_VERSIONS),
+ default=CacheableDefault(ALLOWED_MSSQL_VERSIONS[0]),
+ show_default=True,
+)
+option_executor = click.option(
+ '--executor',
+ help='Executor to use for a kubernetes cluster. Default is KubernetesExecutor.',
+ type=BetterChoice(ALLOWED_EXECUTORS),
+)
+option_forward_credentials = click.option(
+ '-f', '--forward-credentials', help="Forward local credentials to container when running.", is_flag=True
+)
+option_use_airflow_version = click.option(
+ '-V',
+ '--use-airflow-version',
+ help="Use (reinstall at entry) Airflow version from PyPI.",
+ envvar='USE_AIRFLOW_VERSION',
+)
+option_mount_sources = click.option(
+ '--mount-sources',
+ type=BetterChoice(ALLOWED_MOUNT_OPTIONS),
+ default=ALLOWED_MOUNT_OPTIONS[0],
+ show_default=True,
+ help="Choose scope of local sources should be mounted (default = selected).",
+)
+option_force_build = click.option(
+ '-f', '--force-build', help="Force image build no matter if it is determined as needed.", is_flag=True
+)
+option_db_reset = click.option(
+ '-d',
+ '--db-reset',
+ help="Reset DB when entering the container.",
+ is_flag=True,
+ envvar='DB_RESET',
+)
+option_docker_cache = click.option(
+ '-c',
+ '--docker-cache',
+ help='Cache option for image used during the build.',
+ default=ALLOWED_BUILD_CACHE[0],
+ show_default=True,
+ type=BetterChoice(ALLOWED_BUILD_CACHE),
+)
+option_login_to_github_registry = click.option(
+ '--login-to-github-registry',
+ help='Logs in to GitHub registry.',
+ envvar='LOGIN_TO_GITHUB_REGISTRY',
+)
+option_github_token = click.option(
+ '--github-token',
+ help='The token used to authenticate to GitHub.',
+ envvar='GITHUB_TOKEN',
+)
+option_github_username = click.option(
+ '--github-username',
+ help='The user name used to authenticate to GitHub.',
+ envvar='GITHUB_USERNAME',
+)
+option_github_image_id = click.option(
+ '-s',
+ '--github-image-id',
+ help='Commit SHA of the image. \
+ Breeze can automatically pull the commit SHA id specified Default: latest',
+)
+option_image_tag = click.option(
+ '-t',
+ '--image-tag',
+ help='Tag added to the default naming conventions of Airflow CI/PROD images.',
+ envvar='IMAGE_TAG',
+)
+option_image_name = click.option(
+ '-n', '--image-name', help='Name of the image to verify (overrides --python and --image-tag).'
+)
+option_platform = click.option(
+ '--platform',
+ help='Platform for Airflow image.',
+ envvar='PLATFORM',
+ type=BetterChoice(ALLOWED_PLATFORMS),
+)
+option_debian_version = click.option(
+ '-d',
+ '--debian-version',
+ help='Debian version used for the image.',
+ type=BetterChoice(ALLOWED_DEBIAN_VERSIONS),
+ default=ALLOWED_DEBIAN_VERSIONS[0],
+ show_default=True,
+ envvar='DEBIAN_VERSION',
+)
+option_upgrade_to_newer_dependencies = click.option(
+ "-u",
+ '--upgrade-to-newer-dependencies',
+ default="false",
+ show_default=True,
+ help='When other than "false", upgrade all PIP packages to latest.',
+ envvar='UPGRADE_TO_NEWER_DEPENDENCIES',
+)
+option_additional_extras = click.option(
+ '--additional-extras',
+ help='Additional extra package while installing Airflow in the image.',
+ envvar='ADDITIONAL_AIRFLOW_EXTRAS',
+)
+option_additional_dev_apt_deps = click.option(
+ '--additional-dev-apt-deps',
+ help='Additional apt dev dependencies to use when building the images.',
+ envvar='ADDITIONAL_DEV_APT_DEPS',
+)
+option_additional_runtime_apt_deps = click.option(
+ '--additional-runtime-apt-deps',
+ help='Additional apt runtime dependencies to use when building the images.',
+ envvar='ADDITIONAL_RUNTIME_APT_DEPS',
+)
+option_additional_python_deps = click.option(
+ '--additional-python-deps',
+ help='Additional python dependencies to use when building the images.',
+ envvar='ADDITIONAL_PYTHON_DEPS',
+)
+option_additional_dev_apt_command = click.option(
+ '--additional-dev-apt-command',
+ help='Additional command executed before dev apt deps are installed.',
+ envvar='ADDITIONAL_DEV_APT_COMMAND',
+)
+option_additional_runtime_apt_command = click.option(
+ '--additional-runtime-apt-command',
+ help='Additional command executed before runtime apt deps are installed.',
+ envvar='ADDITIONAL_RUNTIME_APT_COMMAND',
+)
+option_additional_dev_apt_env = click.option(
+ '--additional-dev-apt-env',
+ help='Additional environment variables set when adding dev dependencies.',
+ envvar='ADDITIONAL_DEV_APT_ENV',
+)
+option_additional_runtime_apt_env = click.option(
+ '--additional-runtime-apt-env',
+ help='Additional environment variables set when adding runtime dependencies.',
+ envvar='ADDITIONAL_RUNTIME_APT_ENV',
+)
+option_dev_apt_command = click.option(
+ '--dev-apt-command',
+ help='Command executed before dev apt deps are installed.',
+ envvar='DEV_APT_COMMAND',
+)
+option_dev_apt_deps = click.option(
+ '--dev-apt-deps',
+ help='Apt dev dependencies to use when building the images.',
+ envvar='DEV_APT_DEPS',
+)
+option_runtime_apt_command = click.option(
+ '--runtime-apt-command',
+ help='Command executed before runtime apt deps are installed.',
+ envvar='RUNTIME_APT_COMMAND',
+)
+option_runtime_apt_deps = click.option(
+ '--runtime-apt-deps',
+ help='Apt runtime dependencies to use when building the images.',
+ envvar='RUNTIME_APT_DEPS',
+)
+option_skip_rebuild_check = click.option(
+ '-r',
+ '--skip-rebuild-check',
+ help="Skips checking if rebuild is needed",
+ is_flag=True,
+ envvar='SKIP_REBUILD_CHECK',
+)
+option_prepare_buildx_cache = click.option(
+ '--prepare-buildx-cache',
+ help='Prepares build cache rather than build images locally.',
+ is_flag=True,
+ envvar='PREPARE_BUILDX_CACHE',
+)
+option_push_image = click.option(
+ '--push-image',
+ help='Push image after building it.',
+ is_flag=True,
+ envvar='PUSH_IMAGE',
+)
+option_empty_image = click.option(
+ '--empty-image',
+ help='Prepare empty image tagged with the same name as the Airflow image.',
+ is_flag=True,
+ envvar='EMPTY_IMAGE',
+)
+option_wait_for_image = click.option(
+ '--wait-for-image',
+ help='Wait until image is available.',
+ is_flag=True,
+ envvar='WAIT_FOR_IMAGE',
+)
+option_tag_as_latest = click.option(
+ '--tag-as-latest',
+ help='Tags the image as latest after pulling.',
+ is_flag=True,
+ envvar='TAG_AS_LATEST',
+)
+option_verify_image = click.option(
+ '--verify-image',
+ help='Verify image.',
+ is_flag=True,
+ envvar='VERIFY_IMAGE',
+)
+option_install_providers_from_sources = click.option(
+ '--install-providers-from-sources',
+ help="Install providers from sources when installing.",
+ is_flag=True,
+ envvar='INSTALL_PROVIDERS_FROM_SOURCES',
+)
+option_load_example_dags = click.option(
+ '-e',
+ '--load-example-dags',
+ help="Enable configuration to load example DAGs when starting Airflow.",
+ is_flag=True,
+ envvar='LOAD_EXAMPLES',
+)
+option_load_default_connection = click.option(
+ '-c',
+ '--load-default-connections',
+ help="Enable configuration to load default connections when starting Airflow.",
+ is_flag=True,
+ envvar='LOAD_DEFAULT_CONNECTIONS',
+)
+option_version_suffix_for_pypi = click.option(
+ '--version-suffix-for-pypi',
+ help='Version suffix used for PyPI packages (alpha, beta, rc1, etc.).',
+ default="",
+ envvar='VERSION_SUFFIX_FOR_PYPI',
+)
+option_package_format = click.option(
+ '--package-format',
+ type=BetterChoice(ALLOWED_PACKAGE_FORMATS),
+ help='Format of packages.',
+ default=ALLOWED_PACKAGE_FORMATS[0],
+ show_default=True,
+ envvar='PACKAGE_FORMAT',
+)
+option_python_versions = click.option(
+ '--python-versions',
+ help="Space separated list of python versions used for build with multiple versions.",
+ default=" ".join(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS),
+ show_default=True,
+ envvar="PYTHON_VERSIONS",
+)
+option_run_in_parallel = click.option(
+ '--run-in-parallel',
+ help="Run the operation in parallel on all or selected subset of Python versions.",
+ is_flag=True,
+ envvar='RUN_IN_PARALLEL',
+)
+option_parallelism = click.option(
+ '--parallelism',
+ help="Maximum number of processes to use while running the operation in parallel.",
+ type=click.IntRange(1, mp.cpu_count() * 2 if not output_file_for_recording else 8),
+ default=mp.cpu_count() if not output_file_for_recording else 4,
+ envvar='PARALLELISM',
+ show_default=True,
+)
+option_build_multiple_images = click.option(
+ '--build_multiple_images',
+ help="Run the operation sequentially on all or selected subset of Python versions.",
+ is_flag=True,
+ envvar='BUILD_MULTIPLE_IMAGES',
+)
+option_with_ci_group = click.option(
+ '-g',
+ '--with-ci-group',
+ is_flag=True,
+ help="Uses CI group for the command to fold long logs in logical groups.",
+ envvar='WITH_CI_GROUP',
+)
+argument_packages = click.argument(
+ "packages",
+ nargs=-1,
+ required=False,
+ type=BetterChoice(get_available_packages(short_version=True)),
+)
+option_timezone = click.option(
+ "--timezone",
+ default="UTC",
+ type=str,
+ help="Timezone to use during the check",
+)
+option_updated_on_or_after = click.option(
+ "--updated-on-or-after",
+ type=str,
+ help="Date when the release was updated after",
+)
+option_max_age = click.option(
+ "--max-age",
+ type=int,
+ default=3,
+ help="Max age of the last release (used if no updated-on-or-after if specified)",
+)
+option_branch = click.option(
+ "--constraints-branch",
+ default='constraints-main',
+ help="Constraint branch to use to find newer dependencies",
+)
diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
new file mode 100644
index 0000000000..153a11b208
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
@@ -0,0 +1,486 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from airflow_breeze import NAME, VERSION
+from airflow_breeze.commands.common_options import (
+ option_answer,
+ option_backend,
+ option_dry_run,
+ option_github_repository,
+ option_mssql_version,
+ option_mysql_version,
+ option_postgres_version,
+ option_python,
+ option_verbose,
+)
+from airflow_breeze.commands.main import main
+from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL
+from airflow_breeze.shell.shell_params import ShellParams
+from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file
+from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.docker_command_utils import (
+ check_docker_resources,
+ construct_env_variables_docker_compose_command,
+ get_extra_docker_flags,
+)
+from airflow_breeze.utils.path_utils import (
+ AIRFLOW_SOURCES_ROOT,
+ BUILD_CACHE_DIR,
+ get_installation_airflow_sources,
+ get_installation_sources_config_metadata_hash,
+ get_package_setup_metadata_hash,
+ get_used_airflow_sources,
+ get_used_sources_setup_metadata_hash,
+)
+from airflow_breeze.utils.recording import output_file_for_recording
+from airflow_breeze.utils.reinstall import ask_to_reinstall_breeze, reinstall_breeze, warn_non_editable
+from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.visuals import ASCIIART, ASCIIART_STYLE
+
+CONFIGURATION_AND_MAINTENANCE_COMMANDS = {
+ "name": "Configuration & maintenance",
+ "commands": [
+ "cleanup",
+ "self-upgrade",
+ "setup-autocomplete",
+ "config",
+ "resource-check",
+ "free-space",
+ "fix-ownership",
+ "version",
+ ],
+}
+
+CONFIGURATION_AND_MAINTENANCE_PARAMETERS = {
+ "breeze cleanup": [
+ {
+ "name": "Cleanup flags",
+ "options": [
+ "--all",
+ ],
+ },
+ ],
+ "breeze self-upgrade": [
+ {
+ "name": "Self-upgrade flags",
+ "options": [
+ "--use-current-airflow-sources",
+ "--force",
+ ],
+ }
+ ],
+ "breeze setup-autocomplete": [
+ {
+ "name": "Setup autocomplete flags",
+ "options": [
+ "--force",
+ ],
+ },
+ ],
+ "breeze config": [
+ {
+ "name": "Config flags",
+ "options": [
+ "--python",
+ "--backend",
+ "--cheatsheet",
+ "--asciiart",
+ ],
+ },
+ ],
+}
+
+
+@main.command(
+ name="cleanup",
+ help="Cleans the cache of parameters, docker cache and optionally - currently downloaded images.",
+)
+@click.option(
+ '--all',
+ is_flag=True,
+ help='Also remove currently downloaded Breeze images.',
+)
+@option_verbose
+@option_answer
+@option_dry_run
+@option_github_repository
+def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, answer: Optional[str]):
+ set_forced_answer(answer)
+ if all:
+ console.print(
+ "\n[bright_yellow]Removing cache of parameters, clean up docker cache "
+ "and remove locally downloaded images[/]"
+ )
+ else:
+ console.print("\n[bright_yellow]Removing cache of parameters, and cleans up docker cache[/]")
+ if all:
+ docker_images_command_to_execute = [
+ 'docker',
+ 'images',
+ '--filter',
+ 'label=org.apache.airflow.image',
+ '--format',
+ '{{.Repository}}:{{.Tag}}',
+ ]
+ command_result = run_command(
+ docker_images_command_to_execute, verbose=verbose, text=True, capture_output=True
+ )
+ images = command_result.stdout.splitlines() if command_result and command_result.stdout else []
+ if images:
+ console.print("[light_blue]Removing images:[/]")
+ for image in images:
+ console.print(f"[light_blue] * {image}[/]")
+ console.print()
+ docker_rmi_command_to_execute = [
+ 'docker',
+ 'rmi',
+ '--force',
+ ]
+ docker_rmi_command_to_execute.extend(images)
+ given_answer = user_confirm("Are you sure?", timeout=None)
+ if given_answer == Answer.YES:
+ run_command(docker_rmi_command_to_execute, verbose=verbose, dry_run=dry_run, check=False)
+ elif given_answer == Answer.QUIT:
+ sys.exit(0)
+ else:
+ console.print("[light_blue]No locally downloaded images to remove[/]\n")
+ console.print("Pruning docker images")
+ given_answer = user_confirm("Are you sure?", timeout=None)
+ if given_answer == Answer.YES:
+ system_prune_command_to_execute = ['docker', 'system', 'prune']
+ run_command(system_prune_command_to_execute, verbose=verbose, dry_run=dry_run, check=False)
+ elif given_answer == Answer.QUIT:
+ sys.exit(0)
+ console.print(f"Removing build cache dir ${BUILD_CACHE_DIR}")
+ given_answer = user_confirm("Are you sure?", timeout=None)
+ if given_answer == Answer.YES:
+ if not dry_run:
+ shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True)
+ elif given_answer == Answer.QUIT:
+ sys.exit(0)
+
+
+@click.option(
+ '-f',
+ '--force',
+ is_flag=True,
+ help='Force upgrade without asking question to the user.',
+)
+@click.option(
+ '-a',
+ '--use-current-airflow-sources',
+ is_flag=True,
+ help='Use current workdir Airflow sources for upgrade'
+ + (f" rather than from {get_installation_airflow_sources()}." if not output_file_for_recording else "."),
+)
+@main.command(
+ name='self-upgrade',
+ help="Self upgrade Breeze. By default it re-installs Breeze "
+ f"from {get_installation_airflow_sources()}."
+ if not output_file_for_recording
+ else "Self upgrade Breeze.",
+)
+def self_upgrade(force: bool, use_current_airflow_sources: bool):
+ if use_current_airflow_sources:
+ airflow_sources: Optional[Path] = get_used_airflow_sources()
+ else:
+ airflow_sources = get_installation_airflow_sources()
+ if airflow_sources is not None:
+ breeze_sources = airflow_sources / "dev" / "breeze"
+ if force:
+ reinstall_breeze(breeze_sources)
+ else:
+ ask_to_reinstall_breeze(breeze_sources)
+ else:
+ warn_non_editable()
+ sys.exit(1)
+
+
+@option_verbose
+@option_dry_run
+@click.option(
+ '-f',
+ '--force',
+ is_flag=True,
+ help='Force autocomplete setup even if already setup before (overrides the setup).',
+)
+@option_answer
+@main.command(name='setup-autocomplete')
+def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Optional[str]):
+ """
+ Enables autocompletion of breeze commands.
+ """
+ set_forced_answer(answer)
+ # Determine if the shell is bash/zsh/powershell. It helps to build the autocomplete path
+ detected_shell = os.environ.get('SHELL')
+ detected_shell = None if detected_shell is None else detected_shell.split(os.sep)[-1]
+ if detected_shell not in ['bash', 'zsh', 'fish']:
+ console.print(f"\n[red] The shell {detected_shell} is not supported for autocomplete![/]\n")
+ sys.exit(1)
+ console.print(f"Installing {detected_shell} completion for local user")
+ autocomplete_path = (
+ AIRFLOW_SOURCES_ROOT / "dev" / "breeze" / "autocomplete" / f"{NAME}-complete-{detected_shell}.sh"
+ )
+ console.print(f"[bright_blue]Activation command script is available here: {autocomplete_path}[/]\n")
+ console.print(f"[bright_yellow]We need to add above script to your {detected_shell} profile.[/]\n")
+ given_answer = user_confirm("Should we proceed ?", default_answer=Answer.NO, timeout=3)
+ if given_answer == Answer.YES:
+ if detected_shell == 'bash':
+ script_path = str(Path('~').expanduser() / '.bash_completion')
+ command_to_execute = f"source {autocomplete_path}"
+ write_to_shell(command_to_execute, dry_run, script_path, force)
+ elif detected_shell == 'zsh':
+ script_path = str(Path('~').expanduser() / '.zshrc')
+ command_to_execute = f"source {autocomplete_path}"
+ write_to_shell(command_to_execute, dry_run, script_path, force)
+ elif detected_shell == 'fish':
+ # Include steps for fish shell
+ script_path = str(Path('~').expanduser() / f'.config/fish/completions/{NAME}.fish')
+ if os.path.exists(script_path) and not force:
+ console.print(
+ "\n[bright_yellow]Autocompletion is already setup. Skipping. "
+ "You can force autocomplete installation by adding --force/]\n"
+ )
+ else:
+ with open(autocomplete_path) as source_file, open(script_path, 'w') as destination_file:
+ for line in source_file:
+ destination_file.write(line)
+ else:
+ # Include steps for powershell
+ subprocess.check_call(['powershell', 'Set-ExecutionPolicy Unrestricted -Scope CurrentUser'])
+ script_path = (
+ subprocess.check_output(['powershell', '-NoProfile', 'echo $profile']).decode("utf-8").strip()
+ )
+ command_to_execute = f". {autocomplete_path}"
+ write_to_shell(command_to_execute, dry_run, script_path, force)
+ elif given_answer == Answer.NO:
+ console.print(
+ "\nPlease follow the https://click.palletsprojects.com/en/8.1.x/shell-completion/ "
+ "to setup autocompletion for breeze manually if you want to use it.\n"
+ )
+ else:
+ sys.exit(0)
+
+
+@option_verbose
+@main.command()
+def version(verbose: bool, python: str):
+ """Print information about version of apache-airflow-breeze."""
+ console.print(ASCIIART, style=ASCIIART_STYLE)
+ console.print(f"\n[bright_blue]Breeze version: {VERSION}[/]")
+ console.print(f"[bright_blue]Breeze installed from: {get_installation_airflow_sources()}[/]")
+ console.print(f"[bright_blue]Used Airflow sources : {get_used_airflow_sources()}[/]\n")
+ if verbose:
+ console.print(
+ f"[bright_blue]Installation sources config hash : "
+ f"{get_installation_sources_config_metadata_hash()}[/]"
+ )
+ console.print(
+ f"[bright_blue]Used sources config hash : " f"{get_used_sources_setup_metadata_hash()}[/]"
+ )
+ console.print(
+ f"[bright_blue]Package config hash : " f"{(get_package_setup_metadata_hash())}[/]\n"
+ )
+
+
+@main.command(name='config')
+@option_python
+@option_backend
+@option_postgres_version
+@option_mysql_version
+@option_mssql_version
+@click.option('-C/-c', '--cheatsheet/--no-cheatsheet', help="Enable/disable cheatsheet.", default=None)
+@click.option('-A/-a', '--asciiart/--no-asciiart', help="Enable/disable ASCIIart.", default=None)
+def change_config(
+ python: str,
+ backend: str,
+ postgres_version: str,
+ mysql_version: str,
+ mssql_version: str,
+ cheatsheet: bool,
+ asciiart: bool,
+):
+ """
+ Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).
+ """
+ asciiart_file = "suppress_asciiart"
+ cheatsheet_file = "suppress_cheatsheet"
+
+ if asciiart is not None:
+ if asciiart:
+ delete_cache(asciiart_file)
+ console.print('[bright_blue]Enable ASCIIART![/]')
+ else:
+ touch_cache_file(asciiart_file)
+ console.print('[bright_blue]Disable ASCIIART![/]')
+ if cheatsheet is not None:
+ if cheatsheet:
+ delete_cache(cheatsheet_file)
+ console.print('[bright_blue]Enable Cheatsheet[/]')
+ elif cheatsheet is not None:
+ touch_cache_file(cheatsheet_file)
+ console.print('[bright_blue]Disable Cheatsheet[/]')
+
+ def get_status(file: str):
+ return "disabled" if check_if_cache_exists(file) else "enabled"
+
+ console.print()
+ console.print("[bright_blue]Current configuration:[/]")
+ console.print()
+ console.print(f"[bright_blue]* Python: {python}[/]")
+ console.print(f"[bright_blue]* Backend: {backend}[/]")
+ console.print()
+ console.print(f"[bright_blue]* Postgres version: {postgres_version}[/]")
+ console.print(f"[bright_blue]* MySQL version: {mysql_version}[/]")
+ console.print(f"[bright_blue]* MsSQL version: {mssql_version}[/]")
+ console.print()
+ console.print(f"[bright_blue]* ASCIIART: {get_status(asciiart_file)}[/]")
+ console.print(f"[bright_blue]* Cheatsheet: {get_status(cheatsheet_file)}[/]")
+ console.print()
+
+
+@main.command(name="free-space", help="Free space for jobs run in CI.")
+@option_verbose
+@option_dry_run
+@option_answer
+def free_space(verbose: bool, dry_run: bool, answer: str):
+ set_forced_answer(answer)
+ if user_confirm("Are you sure to run free-space and perform cleanup?", timeout=None) == Answer.YES:
+ run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run)
+ run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run)
+ run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False)
+ run_command(
+ ["docker", "system", "prune", "--all", "--force", "--volumes"], verbose=verbose, dry_run=dry_run
+ )
+ run_command(["df", "-h"], verbose=verbose, dry_run=dry_run)
+ run_command(["docker", "logout", "ghcr.io"], verbose=verbose, dry_run=dry_run, check=False)
+
+
+@main.command(name="resource-check", help="Check if available docker resources are enough.")
+@option_verbose
+@option_dry_run
+def resource_check(verbose: bool, dry_run: bool):
+ shell_params = ShellParams(verbose=verbose, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
+ check_docker_resources(verbose, shell_params.airflow_image_name, dry_run=dry_run)
+
+
+@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.")
+@option_verbose
+@option_dry_run
+def fix_ownership(verbose: bool, dry_run: bool):
+ shell_params = ShellParams(
+ verbose=verbose,
+ mount_sources=MOUNT_ALL,
+ python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ )
+ extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
+ env = construct_env_variables_docker_compose_command(shell_params)
+ cmd = [
+ "docker",
+ "run",
+ "-t",
+ *extra_docker_flags,
+ "-e",
+ "GITHUB_ACTIONS=",
+ "-e",
+ "SKIP_ENVIRONMENT_INITIALIZATION=true",
+ "--pull",
+ "never",
+ shell_params.airflow_image_name_with_tag,
+ "/opt/airflow/scripts/in_container/run_fix_ownership.sh",
+ ]
+ run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False)
+ # Always succeed
+ sys.exit(0)
+
+
+def write_to_shell(command_to_execute: str, dry_run: bool, script_path: str, force_setup: bool) -> bool:
+ skip_check = False
+ script_path_file = Path(script_path)
+ if not script_path_file.exists():
+ skip_check = True
+ if not skip_check:
+ if BREEZE_COMMENT in script_path_file.read_text():
+ if not force_setup:
+ console.print(
+ "\n[bright_yellow]Autocompletion is already setup. Skipping. "
+ "You can force autocomplete installation by adding --force[/]\n"
+ )
+ return False
+ else:
+ backup(script_path_file)
+ remove_autogenerated_code(script_path)
+ text = ''
+ if script_path_file.exists():
+ console.print(f"\nModifying the {script_path} file!\n")
+ console.print(f"\nCopy of the original file is held in {script_path}.bak !\n")
+ if not dry_run:
+ backup(script_path_file)
+ text = script_path_file.read_text()
+ else:
+ console.print(f"\nCreating the {script_path} file!\n")
+ if not dry_run:
+ script_path_file.write_text(
+ text
+ + ("\n" if not text.endswith("\n") else "")
+ + START_LINE
+ + command_to_execute
+ + "\n"
+ + END_LINE
+ )
+ else:
+ console.print(f"[bright_blue]The autocomplete script would be added to {script_path}[/]")
+ console.print(
+ f"\n[bright_yellow]IMPORTANT!!!! Please exit and re-enter your shell or run:[/]"
+ f"\n\n source {script_path}\n"
+ )
+ return True
+
+
+BREEZE_COMMENT = "Added by Updated Airflow Breeze autocomplete setup"
+START_LINE = f"# START: {BREEZE_COMMENT}\n"
+END_LINE = f"# END: {BREEZE_COMMENT}\n"
+
+
+def remove_autogenerated_code(script_path: str):
+ lines = Path(script_path).read_text().splitlines(keepends=True)
+ new_lines = []
+ pass_through = True
+ for line in lines:
+ if line == START_LINE:
+ pass_through = False
+ continue
+ if line.startswith(END_LINE):
+ pass_through = True
+ continue
+ if pass_through:
+ new_lines.append(line)
+ Path(script_path).write_text("".join(new_lines))
+
+
+def backup(script_path_file: Path):
+ shutil.copy(str(script_path_file), str(script_path_file) + ".bak")
diff --git a/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py b/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py
new file mode 100644
index 0000000000..2eeac20db0
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/configure_rich_click.py
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+from airflow_breeze.utils import recording # isort:skip # noqa
+
+try:
+ # We handle ImportError so that click autocomplete works
+ import rich_click as click
+
+ from airflow_breeze.commands.ci_image_tools import CI_IMAGE_TOOLS_COMMANDS, CI_IMAGE_TOOLS_PARAMETERS
+ from airflow_breeze.commands.configuration_and_maintenance import (
+ CONFIGURATION_AND_MAINTENANCE_COMMANDS,
+ CONFIGURATION_AND_MAINTENANCE_PARAMETERS,
+ )
+ from airflow_breeze.commands.developer_commands import DEVELOPER_COMMANDS, DEVELOPER_PARAMETERS
+ from airflow_breeze.commands.production_image_tools import (
+ PRODUCTION_IMAGE_TOOLS_COMMANDS,
+ PRODUCTION_IMAGE_TOOLS_PARAMETERS,
+ )
+ from airflow_breeze.commands.release_management import (
+ RELEASE_MANAGEMENT_COMMANDS,
+ RELEASE_MANAGEMENT_PARAMETERS,
+ )
+ from airflow_breeze.commands.testing import TESTING_COMMANDS, TESTING_PARAMETERS
+
+ click.rich_click.SHOW_METAVARS_COLUMN = False
+ click.rich_click.SHOW_ARGUMENTS = False
+ click.rich_click.APPEND_METAVARS_HELP = True
+ click.rich_click.STYLE_ERRORS_SUGGESTION = "bright_blue italic"
+ click.rich_click.ERRORS_SUGGESTION = "\nTry running the '--help' flag for more information.\n"
+ click.rich_click.ERRORS_EPILOGUE = (
+ "\nTo find out more, visit [bright_blue]https://github.com/apache/airflow/blob/main/BREEZE.rst[/]\n"
+ )
+ click.rich_click.OPTION_GROUPS = {
+ **DEVELOPER_PARAMETERS,
+ **TESTING_PARAMETERS,
+ **CONFIGURATION_AND_MAINTENANCE_PARAMETERS,
+ **CI_IMAGE_TOOLS_PARAMETERS,
+ **PRODUCTION_IMAGE_TOOLS_PARAMETERS,
+ **RELEASE_MANAGEMENT_PARAMETERS,
+ }
+ click.rich_click.COMMAND_GROUPS = {
+ "breeze": [
+ DEVELOPER_COMMANDS,
+ TESTING_COMMANDS,
+ CONFIGURATION_AND_MAINTENANCE_COMMANDS,
+ CI_IMAGE_TOOLS_COMMANDS,
+ PRODUCTION_IMAGE_TOOLS_COMMANDS,
+ RELEASE_MANAGEMENT_COMMANDS,
+ ]
+ }
+
+
+except ImportError:
+ import click # type: ignore[no-redef]
diff --git a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py b/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
new file mode 100644
index 0000000000..7a779865f6
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass
+from typing import Any
+
+import click
+
+from airflow_breeze.utils.cache import (
+ check_if_values_allowed,
+ read_and_validate_value_from_cache,
+ read_from_cache_file,
+ write_to_cache_file,
+)
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.recording import output_file_for_recording
+
+
+class BetterChoice(click.Choice):
+ """
+ Nicer formatted choice class for click. We have a lot of parameters sometimes, and formatting
+ them without spaces causes ugly artifacts as the words are broken. This one adds spaces so
+ that when the long list of choices does not wrap on words.
+ """
+
+ name = "BetterChoice"
+
+ def get_metavar(self, param) -> str:
+ choices_str = " | ".join(self.choices)
+ # Use curly braces to indicate a required argument.
+ if param.required and param.param_type_name == "argument":
+ return f"{{{choices_str}}}"
+
+ if param.param_type_name == "argument" and param.nargs == -1:
+ # avoid double [[ for multiple args
+ return f"{choices_str}"
+
+ # Use square braces to indicate an option or optional argument.
+ return f"[{choices_str}]"
+
+
+@dataclass
+class CacheableDefault:
+ value: Any
+
+ def __repr__(self):
+ return self.value
+
+
+class CacheableChoice(click.Choice):
+ """
+ This class implements caching of values from the last use.
+ """
+
+ def convert(self, value, param, ctx):
+ param_name = param.envvar if param.envvar else param.name.upper()
+ if isinstance(value, CacheableDefault):
+ is_cached, new_value = read_and_validate_value_from_cache(param_name, value.value)
+ if not is_cached:
+ console.print(
+ f"\n[bright_blue]Default value of {param.name} " f"parameter {new_value} used.[/]\n"
+ )
+ else:
+ allowed, allowed_values = check_if_values_allowed(param_name, value)
+ if allowed:
+ new_value = value
+ write_to_cache_file(param_name, new_value, check_allowed_values=False)
+ else:
+ new_value = allowed_values[0]
+ console.print(
+ f"\n[yellow]The value {value} is not allowed for parameter {param.name}. "
+ f"Setting default value to {new_value}"
+ )
+ write_to_cache_file(param_name, new_value, check_allowed_values=False)
+ return super().convert(new_value, param, ctx)
+
+ def get_metavar(self, param) -> str:
+ param_name = param.envvar if param.envvar else param.name.upper()
+ current_value = (
+ read_from_cache_file(param_name) if not output_file_for_recording else param.default.value
+ )
+ if not current_value:
+ current_choices = self.choices
+ else:
+ current_choices = [
+ f">{choice}<" if choice == current_value else choice for choice in self.choices
+ ]
+ choices_str = " | ".join(current_choices)
+ # Use curly braces to indicate a required argument.
+ if param.required and param.param_type_name == "argument":
+ return f"{{{choices_str}}}"
+
+ if param.param_type_name == "argument" and param.nargs == -1:
+ # avoid double [[ for multiple args
+ return f"{choices_str}"
+
+ # Use square braces to indicate an option or optional argument.
+ return f"[{choices_str}]"
+
+ def __init__(self, choices, case_sensitive: bool = True) -> None:
+ super().__init__(choices=choices, case_sensitive=case_sensitive)
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
new file mode 100644
index 0000000000..c909821d38
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -0,0 +1,480 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+import rich_click as click
+
+from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
+from airflow_breeze.commands.common_options import (
+ option_answer,
+ option_backend,
+ option_db_reset,
+ option_dry_run,
+ option_force_build,
+ option_forward_credentials,
+ option_github_repository,
+ option_integration,
+ option_load_default_connection,
+ option_load_example_dags,
+ option_mount_sources,
+ option_mssql_version,
+ option_mysql_version,
+ option_postgres_version,
+ option_python,
+ option_use_airflow_version,
+ option_verbose,
+)
+from airflow_breeze.commands.custom_param_types import BetterChoice
+from airflow_breeze.commands.main import main
+from airflow_breeze.global_constants import (
+ DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ MOUNT_SELECTED,
+ get_available_packages,
+)
+from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
+from airflow_breeze.shell.enter_shell import enter_shell
+from airflow_breeze.shell.shell_params import ShellParams
+from airflow_breeze.utils.confirm import set_forced_answer
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.docker_command_utils import (
+ construct_env_variables_docker_compose_command,
+ get_extra_docker_flags,
+)
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+from airflow_breeze.utils.run_utils import assert_pre_commit_installed, run_command
+
+DEVELOPER_COMMANDS = {
+ "name": "Developer tools",
+ "commands": [
+ "shell",
+ "start-airflow",
+ "stop",
+ "build-docs",
+ "static-checks",
+ ],
+}
+
+DEVELOPER_PARAMETERS = {
+ "breeze": [
+ {
+ "name": "Basic flags for the default (shell) command",
+ "options": [
+ "--python",
+ "--backend",
+ "--use-airflow-version",
+ "--postgres-version",
+ "--mysql-version",
+ "--mssql-version",
+ "--forward-credentials",
+ "--db-reset",
+ ],
+ },
+ {
+ "name": "Advanced flags for the default (shell) command",
+ "options": [
+ "--force-build",
+ "--mount-sources",
+ "--integration",
+ ],
+ },
+ ],
+ "breeze shell": [
+ {
+ "name": "Basic flags",
+ "options": [
+ "--python",
+ "--backend",
+ "--use-airflow-version",
+ "--postgres-version",
+ "--mysql-version",
+ "--mssql-version",
+ "--forward-credentials",
+ "--db-reset",
+ ],
+ },
+ {
+ "name": "Advanced flag for running",
+ "options": [
+ "--force-build",
+ "--mount-sources",
+ "--integration",
+ ],
+ },
+ ],
+ "breeze start-airflow": [
+ {
+ "name": "Basic flags",
+ "options": [
+ "--python",
+ "--backend",
+ "--use-airflow-version",
+ "--postgres-version",
+ "--mysql-version",
+ "--mssql-version",
+ "--load-example-dags",
+ "--load-default-connections",
+ "--forward-credentials",
+ "--db-reset",
+ ],
+ },
+ {
+ "name": "Advanced flag for running",
+ "options": [
+ "--force-build",
+ "--mount-sources",
+ "--integration",
+ ],
+ },
+ ],
+ "breeze stop": [
+ {
+ "name": "Stop flags",
+ "options": [
+ "--preserve-volumes",
+ ],
+ },
+ ],
+ "breeze build-docs": [
+ {
+ "name": "Doc flags",
+ "options": [
+ "--docs-only",
+ "--spellcheck-only",
+ "--package-filter",
+ ],
+ },
+ ],
+ "breeze static-checks": [
+ {
+ "name": "Pre-commit flags",
+ "options": [
+ "--type",
+ "--files",
+ "--all-files",
+ "--show-diff-on-failure",
+ "--last-commit",
+ ],
+ },
+ ],
+}
+
+
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# Make sure that whatever you add here as an option is also
+# Added in the "main" command in breeze.py. The min command above
+# Is used for a shorthand of shell and except the extra
+# Args it should have the same parameters.
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+@main.command()
+@option_verbose
+@option_dry_run
+@option_python
+@option_backend
+@option_github_repository
+@option_postgres_version
+@option_mysql_version
+@option_mssql_version
+@option_forward_credentials
+@option_force_build
+@option_use_airflow_version
+@option_mount_sources
+@option_integration
+@option_db_reset
+@option_answer
+@click.argument('extra-args', nargs=-1, type=click.UNPROCESSED)
+def shell(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ backend: str,
+ integration: Tuple[str],
+ postgres_version: str,
+ mysql_version: str,
+ mssql_version: str,
+ forward_credentials: bool,
+ mount_sources: str,
+ use_airflow_version: str,
+ force_build: bool,
+ db_reset: bool,
+ answer: Optional[str],
+ extra_args: Tuple,
+):
+ """Enter breeze.py environment. this is the default command use when no other is selected."""
+ set_forced_answer(answer)
+ if verbose or dry_run:
+ console.print("\n[green]Welcome to breeze.py[/]\n")
+ console.print(f"\n[green]Root of Airflow Sources = {AIRFLOW_SOURCES_ROOT}[/]\n")
+ enter_shell(
+ verbose=verbose,
+ dry_run=dry_run,
+ python=python,
+ github_repository=github_repository,
+ backend=backend,
+ integration=integration,
+ postgres_version=postgres_version,
+ mysql_version=mysql_version,
+ mssql_version=mssql_version,
+ forward_credentials=str(forward_credentials),
+ mount_sources=mount_sources,
+ use_airflow_version=use_airflow_version,
+ force_build=force_build,
+ db_reset=db_reset,
+ extra_args=extra_args,
+ answer=answer,
+ )
+
+
+@option_verbose
+@main.command(name='start-airflow')
+@option_dry_run
+@option_python
+@option_github_repository
+@option_backend
+@option_postgres_version
+@option_load_example_dags
+@option_load_default_connection
+@option_mysql_version
+@option_mssql_version
+@option_forward_credentials
+@option_force_build
+@option_use_airflow_version
+@option_mount_sources
+@option_integration
+@option_db_reset
+@option_answer
+@click.argument('extra-args', nargs=-1, type=click.UNPROCESSED)
+def start_airflow(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ backend: str,
+ integration: Tuple[str],
+ postgres_version: str,
+ load_example_dags: bool,
+ load_default_connections: bool,
+ mysql_version: str,
+ mssql_version: str,
+ forward_credentials: bool,
+ mount_sources: str,
+ use_airflow_version: str,
+ force_build: bool,
+ db_reset: bool,
+ answer: Optional[str],
+ extra_args: Tuple,
+):
+ """Enter breeze.py environment and starts all Airflow components in the tmux session."""
+ set_forced_answer(answer)
+ enter_shell(
+ verbose=verbose,
+ dry_run=dry_run,
+ python=python,
+ github_repository=github_repository,
+ backend=backend,
+ integration=integration,
+ postgres_version=postgres_version,
+ load_default_connections=load_default_connections,
+ load_example_dags=load_example_dags,
+ mysql_version=mysql_version,
+ mssql_version=mssql_version,
+ forward_credentials=str(forward_credentials),
+ mount_sources=mount_sources,
+ use_airflow_version=use_airflow_version,
+ force_build=force_build,
+ db_reset=db_reset,
+ start_airflow=True,
+ extra_args=extra_args,
+ answer=answer,
+ )
+
+
+@main.command(name='build-docs')
+@option_verbose
+@option_dry_run
+@option_github_repository
+@click.option('-d', '--docs-only', help="Only build documentation.", is_flag=True)
+@click.option('-s', '--spellcheck-only', help="Only run spell checking.", is_flag=True)
+@click.option(
+ '-p',
+ '--package-filter',
+ help="List of packages to consider.",
+ type=BetterChoice(get_available_packages()),
+ multiple=True,
+)
+def build_docs(
+ verbose: bool,
+ dry_run: bool,
+ github_repository: str,
+ docs_only: bool,
+ spellcheck_only: bool,
+ package_filter: Tuple[str],
+):
+ """Build documentation in the container."""
+ params = BuildCiParams(github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION)
+ ci_image_name = params.airflow_image_name
+ doc_builder = DocBuildParams(
+ package_filter=package_filter,
+ docs_only=docs_only,
+ spellcheck_only=spellcheck_only,
+ )
+ extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED)
+ env = construct_env_variables_docker_compose_command(params)
+ cmd = [
+ "docker",
+ "run",
+ "-t",
+ *extra_docker_flags,
+ "-e",
+ "GITHUB_ACTIONS=",
+ "-e",
+ "SKIP_ENVIRONMENT_INITIALIZATION=true",
+ "--pull",
+ "never",
+ ci_image_name,
+ "/opt/airflow/scripts/in_container/run_docs_build.sh",
+ *doc_builder.args_doc_builder,
+ ]
+ process = run_command(cmd, verbose=verbose, dry_run=dry_run, text=True, env=env, check=False)
+ sys.exit(process.returncode)
+
+
+@main.command(
+ name="static-checks",
+ help="Run static checks.",
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+)
+@click.option(
+ '-t',
+ '--type',
+ help="Type(s) of the static checks to run (multiple can be added).",
+ type=BetterChoice(PRE_COMMIT_LIST),
+ multiple=True,
+)
+@click.option('-a', '--all-files', help="Run checks on all files.", is_flag=True)
+@click.option('-f', '--files', help="List of files to run the checks on.", multiple=True)
+@click.option(
+ '-s', '--show-diff-on-failure', help="Show diff for files modified by the checks.", is_flag=True
+)
+@click.option(
+ '-c',
+ '--last-commit',
+ help="Run checks for all files in last commit. Mutually exclusive with --commit-ref.",
+ is_flag=True,
+)
+@click.option(
+ '-r',
+ '--commit-ref',
+ help="Run checks for this commit reference only "
+ "(can be any git commit-ish reference). "
+ "Mutually exclusive with --last-commit.",
+)
+@option_verbose
+@option_dry_run
+@option_github_repository
+@click.argument('precommit_args', nargs=-1, type=click.UNPROCESSED)
+def static_checks(
+ verbose: bool,
+ dry_run: bool,
+ github_repository: str,
+ all_files: bool,
+ show_diff_on_failure: bool,
+ last_commit: bool,
+ commit_ref: str,
+ type: Tuple[str],
+ files: bool,
+ precommit_args: Tuple,
+):
+ assert_pre_commit_installed(verbose=verbose)
+ command_to_execute = [sys.executable, "-m", "pre_commit", 'run']
+ if last_commit and commit_ref:
+ console.print("\n[red]You cannot specify both --last-commit and --commit-ref[/]\n")
+ sys.exit(1)
+ for single_check in type:
+ command_to_execute.append(single_check)
+ if all_files:
+ command_to_execute.append("--all-files")
+ if show_diff_on_failure:
+ command_to_execute.append("--show-diff-on-failure")
+ if last_commit:
+ command_to_execute.extend(["--from-ref", "HEAD^", "--to-ref", "HEAD"])
+ if commit_ref:
+ command_to_execute.extend(["--from-ref", f"{commit_ref}^", "--to-ref", f"{commit_ref}"])
+ if files:
+ command_to_execute.append("--files")
+ if verbose or dry_run:
+ command_to_execute.append("--verbose")
+ if precommit_args:
+ command_to_execute.extend(precommit_args)
+ env = os.environ.copy()
+ env['GITHUB_REPOSITORY'] = github_repository
+ static_checks_result = run_command(
+ command_to_execute,
+ verbose=verbose,
+ dry_run=dry_run,
+ check=False,
+ no_output_dump_on_exception=True,
+ text=True,
+ env=env,
+ )
+ if static_checks_result.returncode != 0:
+ console.print("[red]There were errors during pre-commit check. They should be fixed[/]")
+ sys.exit(static_checks_result.returncode)
+
+
+@main.command(name="stop", help="Stop running breeze environment.")
+@option_verbose
+@option_dry_run
+@click.option(
+ "-p",
+ "--preserve-volumes",
+ help="Skip removing volumes when stopping Breeze.",
+ is_flag=True,
+)
+def stop(verbose: bool, dry_run: bool, preserve_volumes: bool):
+ command_to_execute = ['docker-compose', 'down', "--remove-orphans"]
+ if not preserve_volumes:
+ command_to_execute.append("--volumes")
+ shell_params = ShellParams(verbose=verbose)
+ env_variables = construct_env_variables_docker_compose_command(shell_params)
+ run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables)
+
+
+@dataclass
+class DocBuildParams:
+ package_filter: Tuple[str]
+ docs_only: bool
+ spellcheck_only: bool
+
+ @property
+ def args_doc_builder(self) -> List[str]:
+ doc_args = []
+ if self.docs_only:
+ doc_args.append("--docs-only")
+ if self.spellcheck_only:
+ doc_args.append("--spellcheck-only")
+ if self.package_filter and len(self.package_filter) > 0:
+ for single_filter in self.package_filter:
+ doc_args.extend(["--package-filter", single_filter])
+ return doc_args
diff --git a/dev/breeze/src/airflow_breeze/commands/main.py b/dev/breeze/src/airflow_breeze/commands/main.py
new file mode 100644
index 0000000000..6434e5a63e
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/main.py
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow_breeze.commands.common_options import (
+ option_answer,
+ option_backend,
+ option_db_reset,
+ option_dry_run,
+ option_force_build,
+ option_forward_credentials,
+ option_github_repository,
+ option_integration,
+ option_mount_sources,
+ option_mssql_version,
+ option_mysql_version,
+ option_postgres_version,
+ option_python,
+ option_use_airflow_version,
+ option_verbose,
+)
+from airflow_breeze.commands.configure_rich_click import click
+from airflow_breeze.utils.path_utils import create_directories
+
+
+@click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']})
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_backend
+@option_postgres_version
+@option_mysql_version
+@option_mssql_version
+@option_forward_credentials
+@option_force_build
+@option_use_airflow_version
+@option_mount_sources
+@option_integration
+@option_db_reset
+@option_answer
+@click.pass_context
+def main(ctx: click.Context, **kwargs):
+ from airflow_breeze.commands.developer_commands import shell
+
+ create_directories()
+ if not ctx.invoked_subcommand:
+ ctx.forward(shell, extra_args={})
diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py b/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
new file mode 100644
index 0000000000..219013ca97
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
@@ -0,0 +1,384 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from typing import Optional, Tuple
+
+import click
+
+from airflow_breeze.build_image.prod.build_prod_image import build_production_image
+from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
+from airflow_breeze.commands.common_options import (
+ option_additional_dev_apt_command,
+ option_additional_dev_apt_deps,
+ option_additional_dev_apt_env,
+ option_additional_extras,
+ option_additional_python_deps,
+ option_additional_runtime_apt_command,
+ option_additional_runtime_apt_deps,
+ option_additional_runtime_apt_env,
+ option_answer,
+ option_build_multiple_images,
+ option_debian_version,
+ option_dev_apt_command,
+ option_dev_apt_deps,
+ option_docker_cache,
+ option_dry_run,
+ option_empty_image,
+ option_github_repository,
+ option_github_token,
+ option_github_username,
+ option_image_name,
+ option_image_tag,
+ option_install_providers_from_sources,
+ option_login_to_github_registry,
+ option_parallelism,
+ option_platform,
+ option_prepare_buildx_cache,
+ option_push_image,
+ option_python,
+ option_python_versions,
+ option_run_in_parallel,
+ option_runtime_apt_command,
+ option_runtime_apt_deps,
+ option_tag_as_latest,
+ option_upgrade_to_newer_dependencies,
+ option_verbose,
+ option_verify_image,
+ option_wait_for_image,
+ option_with_ci_group,
+)
+from airflow_breeze.commands.custom_param_types import BetterChoice
+from airflow_breeze.commands.main import main
+from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS
+from airflow_breeze.utils.ci_group import ci_group
+from airflow_breeze.utils.confirm import set_forced_answer
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
+from airflow_breeze.utils.python_versions import get_python_version_list
+from airflow_breeze.utils.run_tests import verify_an_image
+from airflow_breeze.utils.run_utils import filter_out_none
+
+PRODUCTION_IMAGE_TOOLS_COMMANDS = {
+ "name": "Production Image tools",
+ "commands": [
+ "build-prod-image",
+ "pull-prod-image",
+ "verify-prod-image",
+ ],
+}
+
+PRODUCTION_IMAGE_TOOLS_PARAMETERS = {
+ "breeze build-prod-image": [
+ {
+ "name": "Basic usage",
+ "options": [
+ "--python",
+ "--install-airflow-version",
+ "--upgrade-to-newer-dependencies",
+ "--debian-version",
+ "--image-tag",
+ "--docker-cache",
+ ],
+ },
+ {
+ "name": "Building multiple images",
+ "options": [
+ "--build_multiple_images",
+ "--python-versions",
+ ],
+ },
+ {
+ "name": "Options for customizing images",
+ "options": [
+ "--install-providers-from-sources",
+ "--additional-python-deps",
+ "--additional-extras",
+ "--additional-runtime-apt-deps",
+ "--additional-runtime-apt-env",
+ "--additional-runtime-apt-command",
+ "--additional-dev-apt-deps",
+ "--additional-dev-apt-env",
+ "--additional-dev-apt-command",
+ "--extras",
+ "--runtime-apt-deps",
+ "--runtime-apt-command",
+ "--dev-apt-deps",
+ "--dev-apt-command",
+ ],
+ },
+ {
+ "name": "Customization options (for specific customization needs)",
+ "options": [
+ "--cleanup-docker-context-files",
+ "--install-from-docker-context-files",
+ "--disable-mysql-client-installation",
+ "--disable-mssql-client-installation",
+ "--disable-postgres-client-installation",
+ "--disable-airflow-repo-cache",
+ "--disable-pypi",
+ "--install-airflow-reference",
+ "--installation-method",
+ ],
+ },
+ {
+ "name": "Preparing cache and push (for maintainers and CI)",
+ "options": [
+ "--github-token",
+ "--github-username",
+ "--login-to-github-registry",
+ "--push-image",
+ "--prepare-buildx-cache",
+ "--platform",
+ "--empty-image",
+ ],
+ },
+ ],
+ "breeze pull-prod-image": [
+ {
+ "name": "Pull image flags",
+ "options": [
+ "--image-tag",
+ "--python",
+ "--verify-image",
+ "--wait-for-image",
+ "--tag-as-latest",
+ ],
+ },
+ {
+ "name": "Parallel running",
+ "options": [
+ "--run-in-parallel",
+ "--parallelism",
+ "--python-versions",
+ ],
+ },
+ ],
+ "breeze verify-prod-image": [
+ {
+ "name": "Verify image flags",
+ "options": [
+ "--image-name",
+ "--python",
+ "--image-tag",
+ ],
+ }
+ ],
+}
+
+
+@option_verbose
+@option_dry_run
+@option_answer
+@option_with_ci_group
+@main.command(name='build-prod-image')
+@option_python
+@option_build_multiple_images
+@option_python_versions
+@option_upgrade_to_newer_dependencies
+@option_platform
+@option_debian_version
+@option_github_repository
+@option_github_token
+@option_github_username
+@option_login_to_github_registry
+@option_docker_cache
+@option_image_tag
+@option_prepare_buildx_cache
+@option_push_image
+@option_empty_image
+@click.option(
+ '--installation-method',
+ help="Install Airflow from: sources or PyPI.",
+ type=BetterChoice(ALLOWED_INSTALLATION_METHODS),
+)
+@option_install_providers_from_sources
+@click.option(
+ '--install-from-docker-context-files',
+ help='Install wheels from local docker-context-files when building image.',
+ is_flag=True,
+)
+@click.option(
+ '--cleanup-docker-context-files',
+ help='Clean up docker context files before running build.',
+ is_flag=True,
+)
+@click.option(
+ '--extras', default=",".join(DEFAULT_EXTRAS), show_default=True, help="Extras to install by default."
+)
+@click.option('--disable-mysql-client-installation', help="Do not install MySQL client.", is_flag=True)
+@click.option('--disable-mssql-client-installation', help="Do not install MsSQl client.", is_flag=True)
+@click.option('--disable-postgres-client-installation', help="Do not install Postgres client.", is_flag=True)
+@click.option(
+ '--disable-airflow-repo-cache',
+ help="Disable cache from Airflow repository during building.",
+ is_flag=True,
+)
+@click.option('--disable-pypi', help="Disable PyPI during building.", is_flag=True)
+@click.option(
+ '--install-airflow-reference',
+ help="Install Airflow using GitHub tag or branch.",
+)
+@click.option('-V', '--install-airflow-version', help="Install version of Airflow from PyPI.")
+@option_additional_extras
+@option_additional_dev_apt_deps
+@option_additional_runtime_apt_deps
+@option_additional_python_deps
+@option_additional_dev_apt_command
+@option_additional_dev_apt_env
+@option_additional_runtime_apt_env
+@option_additional_runtime_apt_command
+@option_dev_apt_command
+@option_dev_apt_deps
+@option_runtime_apt_command
+@option_runtime_apt_deps
+def build_prod_image(
+ verbose: bool,
+ dry_run: bool,
+ with_ci_group: bool,
+ build_multiple_images: bool,
+ python_versions: str,
+ answer: Optional[str],
+ **kwargs,
+):
+ """
+ Build Production image. Include building multiple images for all or selected Python versions sequentially.
+ """
+
+ def run_build(prod_image_params: BuildProdParams) -> None:
+ return_code, info = build_production_image(
+ verbose=verbose, dry_run=dry_run, with_ci_group=with_ci_group, prod_image_params=prod_image_params
+ )
+ if return_code != 0:
+ console.print(f"[red]Error when building image! {info}")
+ sys.exit(return_code)
+
+ set_forced_answer(answer)
+ parameters_passed = filter_out_none(**kwargs)
+ if build_multiple_images:
+ with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
+ python_version_list = get_python_version_list(python_versions)
+ for python in python_version_list:
+ params = BuildProdParams(**parameters_passed)
+ params.python = python
+ params.answer = answer
+ run_build(prod_image_params=params)
+ else:
+ params = BuildProdParams(**parameters_passed)
+ run_build(prod_image_params=params)
+
+
+@main.command(name='pull-prod-image')
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_run_in_parallel
+@option_parallelism
+@option_python_versions
+@option_image_tag
+@option_wait_for_image
+@option_tag_as_latest
+@option_verify_image
+@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
+def pull_prod_image(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ run_in_parallel: bool,
+ parallelism: int,
+ python_versions: str,
+ image_tag: Optional[str],
+ wait_for_image: bool,
+ tag_as_latest: bool,
+ verify_image: bool,
+ extra_pytest_args: Tuple,
+):
+ """Pull and optionally verify Production images - possibly in parallel for all Python versions."""
+ if run_in_parallel:
+ python_version_list = get_python_version_list(python_versions)
+ prod_image_params_list = [
+ BuildProdParams(image_tag=image_tag, python=python, github_repository=github_repository)
+ for python in python_version_list
+ ]
+ run_pull_in_parallel(
+ dry_run=dry_run,
+ parallelism=parallelism,
+ image_params_list=prod_image_params_list,
+ python_version_list=python_version_list,
+ verbose=verbose,
+ verify_image=verify_image,
+ wait_for_image=wait_for_image,
+ tag_as_latest=tag_as_latest,
+ extra_pytest_args=extra_pytest_args if extra_pytest_args is not None else (),
+ )
+ else:
+ image_params = BuildProdParams(
+ image_tag=image_tag, python=python, github_repository=github_repository
+ )
+ return_code, info = run_pull_image(
+ image_params=image_params,
+ dry_run=dry_run,
+ verbose=verbose,
+ wait_for_image=wait_for_image,
+ tag_as_latest=tag_as_latest,
+ poll_time=10.0,
+ )
+ if return_code != 0:
+ console.print(f"[red]There was an error when pulling PROD image: {info}[/]")
+ sys.exit(return_code)
+
+
+@main.command(
+ name='verify-prod-image',
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+)
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_image_tag
+@option_image_name
+@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
+def verify_prod_image(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ image_name: str,
+ image_tag: str,
+ extra_pytest_args: Tuple,
+):
+ """Verify Production image."""
+ if image_name is None:
+ build_params = BuildProdParams(
+ python=python, image_tag=image_tag, github_repository=github_repository
+ )
+ image_name = build_params.airflow_image_name_with_tag
+ console.print(f"[bright_blue]Verifying PROD image: {image_name}[/]")
+ return_code, info = verify_an_image(
+ image_name=image_name,
+ verbose=verbose,
+ dry_run=dry_run,
+ image_type='PROD',
+ extra_pytest_args=extra_pytest_args,
+ )
+ sys.exit(return_code)
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management.py b/dev/breeze/src/airflow_breeze/commands/release_management.py
new file mode 100644
index 0000000000..f9fe3a3948
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/release_management.py
@@ -0,0 +1,397 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from typing import IO, List, Optional, Tuple
+
+import click
+
+from airflow_breeze.commands.common_options import (
+ argument_packages,
+ option_answer,
+ option_branch,
+ option_dry_run,
+ option_github_repository,
+ option_image_tag,
+ option_max_age,
+ option_package_format,
+ option_parallelism,
+ option_python,
+ option_python_versions,
+ option_run_in_parallel,
+ option_timezone,
+ option_updated_on_or_after,
+ option_verbose,
+ option_version_suffix_for_pypi,
+ option_with_ci_group,
+)
+from airflow_breeze.commands.custom_param_types import BetterChoice
+from airflow_breeze.commands.main import main
+from airflow_breeze.global_constants import (
+ ALLOWED_GENERATE_CONSTRAINTS_MODES,
+ DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ MOUNT_ALL,
+)
+from airflow_breeze.shell.shell_params import ShellParams
+from airflow_breeze.utils.ci_group import ci_group
+from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.constraints import run_generate_constraints, run_generate_constraints_in_parallel
+from airflow_breeze.utils.docker_command_utils import (
+ construct_env_variables_docker_compose_command,
+ get_extra_docker_flags,
+)
+from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies
+from airflow_breeze.utils.python_versions import get_python_version_list
+from airflow_breeze.utils.run_utils import run_command
+
+RELEASE_MANAGEMENT_PARAMETERS = {
+ "breeze prepare-airflow-package": [
+ {"name": "Package flags", "options": ["--package-format", "--version-suffix-for-pypi"]}
+ ],
+ "breeze prepare-provider-packages": [
+ {
+ "name": "Package flags",
+ "options": [
+ "--package-format",
+ "--version-suffix-for-pypi",
+ "--package-list-file",
+ ],
+ }
+ ],
+ "breeze prepare-provider-documentation": [
+ {"name": "Provider documentation preparation flags", "options": ["--skip-package-verification"]}
+ ],
+ "breeze find-newer-dependencies": [
+ {
+ "name": "Find newer dependencies flags",
+ "options": [
+ "--python",
+ "--timezone",
+ "--constraints-branch",
+ "--updated-on-or-after",
+ "--max-age",
+ ],
+ }
+ ],
+ "breeze generate-constraints": [
+ {
+ "name": "Generate constraints flags",
+ "options": [
+ "--image-tag",
+ "--python",
+ "--generate-constraints-mode",
+ ],
+ },
+ {
+ "name": "Parallel running",
+ "options": [
+ "--run-in-parallel",
+ "--parallelism",
+ "--python-versions",
+ ],
+ },
+ ],
+}
+
+RELEASE_MANAGEMENT_COMMANDS = {
+ "name": "Release management",
+ "commands": [
+ "prepare-provider-documentation",
+ "prepare-provider-packages",
+ "prepare-airflow-package",
+ "generate-constraints",
+ "find-newer-dependencies",
+ ],
+}
+
+
+@main.command(
+ name='prepare-airflow-package',
+ help="Prepare sdist/whl package of Airflow.",
+)
+@option_verbose
+@option_dry_run
+@option_github_repository
+@option_with_ci_group
+@option_package_format
+@option_version_suffix_for_pypi
+def prepare_airflow_packages(
+ verbose: bool,
+ dry_run: bool,
+ github_repository: str,
+ with_ci_group: bool,
+ package_format: str,
+ version_suffix_for_pypi: str,
+):
+ shell_params = ShellParams(
+ verbose=verbose, github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION
+ )
+ env_variables = construct_env_variables_docker_compose_command(shell_params)
+ env_variables['INSTALL_PROVIDERS_FROM_SOURCES'] = "false"
+ extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
+ with ci_group("Prepare Airflow package", enabled=with_ci_group):
+ run_command(
+ [
+ "docker",
+ "run",
+ "-t",
+ *extra_docker_flags,
+ "-e",
+ "SKIP_ENVIRONMENT_INITIALIZATION=true",
+ "-e",
+ f"PACKAGE_FORMAT={package_format}",
+ "-e",
+ f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
+ "--pull",
+ "never",
+ shell_params.airflow_image_name_with_tag,
+ "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh",
+ ],
+ verbose=verbose,
+ dry_run=dry_run,
+ env=env_variables,
+ )
+
+
+@main.command(
+ name='prepare-provider-documentation',
+ help="Prepare CHANGELOG, README and COMMITS information for providers.",
+)
+@option_verbose
+@option_dry_run
+@option_github_repository
+@option_with_ci_group
+@option_answer
+@click.option(
+ '--skip-package-verification',
+ help="Skip Provider package verification.",
+ is_flag=True,
+)
+@argument_packages
+def prepare_provider_documentation(
+ verbose: bool,
+ dry_run: bool,
+ github_repository: str,
+ with_ci_group: bool,
+ answer: Optional[str],
+ skip_package_verification: bool,
+ packages: List[str],
+):
+ set_forced_answer(answer)
+ shell_params = ShellParams(
+ verbose=verbose,
+ mount_sources=MOUNT_ALL,
+ skip_package_verification=skip_package_verification,
+ github_repository=github_repository,
+ python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ answer=answer,
+ )
+ env_variables = construct_env_variables_docker_compose_command(shell_params)
+ extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
+ if answer and answer.lower() in ['y', 'yes']:
+ term_flag = "-t"
+ else:
+ term_flag = "-it"
+ cmd_to_run = [
+ "docker",
+ "run",
+ term_flag,
+ *extra_docker_flags,
+ "-e",
+ "SKIP_ENVIRONMENT_INITIALIZATION=true",
+ "--pull",
+ "never",
+ shell_params.airflow_image_name_with_tag,
+ "/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh",
+ ]
+ if packages:
+ cmd_to_run.extend(packages)
+ with ci_group("Prepare provider documentation", enabled=with_ci_group):
+ run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
+
+
+@main.command(
+ name='prepare-provider-packages',
+ help="Prepare sdist/whl packages of Airflow Providers.",
+)
+@option_verbose
+@option_dry_run
+@option_github_repository
+@option_with_ci_group
+@option_package_format
+@option_version_suffix_for_pypi
+@click.option(
+ '--package-list-file',
+ type=click.File('rt'),
+ help='Read list of packages from text file (one package per line)',
+)
+@argument_packages
+def prepare_provider_packages(
+ verbose: bool,
+ dry_run: bool,
+ github_repository: str,
+ with_ci_group: bool,
+ package_format: str,
+ version_suffix_for_pypi: str,
+ package_list_file: IO,
+ packages: Tuple[str, ...],
+):
+ packages_list = list(packages)
+ if package_list_file:
+ packages_list.extend([package.strip() for package in package_list_file.readlines()])
+ shell_params = ShellParams(
+ verbose=verbose,
+ mount_sources=MOUNT_ALL,
+ github_repository=github_repository,
+ python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ )
+ env_variables = construct_env_variables_docker_compose_command(shell_params)
+ extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
+ cmd_to_run = [
+ "docker",
+ "run",
+ "-t",
+ *extra_docker_flags,
+ "-e",
+ "SKIP_ENVIRONMENT_INITIALIZATION=true",
+ "-e",
+ f"PACKAGE_FORMAT={package_format}",
+ "-e",
+ f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
+ "--pull",
+ "never",
+ shell_params.airflow_image_name_with_tag,
+ "/opt/airflow/scripts/in_container/run_prepare_provider_packages.sh",
+ ]
+ cmd_to_run.extend(packages_list)
+ with ci_group("Prepare provider packages", enabled=with_ci_group):
+ run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
+
+
+@main.command(
+ name='generate-constraints',
+ help="Generates pinned constraint files with all extras from setup.py in parallel.",
+)
+@click.option(
+ '--generate-constraints-mode',
+ type=BetterChoice(ALLOWED_GENERATE_CONSTRAINTS_MODES),
+ default=ALLOWED_GENERATE_CONSTRAINTS_MODES[0],
+ show_default=True,
+ help='Mode of generating constraints',
+)
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_run_in_parallel
+@option_parallelism
+@option_python_versions
+@option_image_tag
+@option_answer
+@option_with_ci_group
+def generate_constraints(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ run_in_parallel: bool,
+ parallelism: int,
+ python_versions: str,
+ image_tag: str,
+ answer: Optional[str],
+ generate_constraints_mode: str,
+ with_ci_group: bool,
+):
+ set_forced_answer(answer)
+ if run_in_parallel:
+ given_answer = user_confirm(
+ f"Did you build all CI images {python_versions} with --upgrade-to-newer-dependencies "
+ f"true flag set?",
+ timeout=None,
+ )
+ else:
+ given_answer = user_confirm(
+ f"Did you build CI image {python} with --upgrade-to-newer-dependencies true flag set?",
+ timeout=None,
+ )
+ if given_answer != Answer.YES:
+ if run_in_parallel:
+ console.print("\n[yellow]Use this command to build the images:[/]\n")
+ console.print(
+ f" breeze build-image --run-in-parallel --python-versions '{python_versions}' "
+ f"--upgrade-to-newer-dependencies true\n"
+ )
+ else:
+ shell_params = ShellParams(
+ image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
+ )
+ console.print("\n[yellow]Use this command to build the image:[/]\n")
+ console.print(
+ f" breeze build-image --python'{shell_params.python}' "
+ f"--upgrade-to-newer-dependencies true\n"
+ )
+ sys.exit(1)
+ if run_in_parallel:
+ python_version_list = get_python_version_list(python_versions)
+ shell_params_list = [
+ ShellParams(
+ image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
+ )
+ for python in python_version_list
+ ]
+ with ci_group(f"Generating constraints with {generate_constraints_mode}", enabled=with_ci_group):
+ run_generate_constraints_in_parallel(
+ shell_params_list=shell_params_list,
+ parallelism=parallelism,
+ dry_run=dry_run,
+ verbose=verbose,
+ python_version_list=python_version_list,
+ generate_constraints_mode=generate_constraints_mode,
+ )
+ else:
+ with ci_group(f"Generating constraints with {generate_constraints_mode}", enabled=with_ci_group):
+ shell_params = ShellParams(
+ image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
+ )
+ return_code, info = run_generate_constraints(
+ shell_params=shell_params,
+ dry_run=dry_run,
+ verbose=verbose,
+ generate_constraints_mode=generate_constraints_mode,
+ )
+ if return_code != 0:
+ console.print(f"[red]There was an error when generating constraints: {info}[/]")
+ sys.exit(return_code)
+
+
+@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.")
+@option_timezone
+@option_branch
+@option_python
+@option_updated_on_or_after
+@option_max_age
+def breeze_find_newer_dependencies(
+ constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int
+):
+ return find_newer_dependencies(
+ constraints_branch=constraints_branch,
+ python=python,
+ timezone=timezone,
+ updated_on_or_after=updated_on_or_after,
+ max_age=max_age,
+ )
diff --git a/dev/breeze/src/airflow_breeze/commands/testing.py b/dev/breeze/src/airflow_breeze/commands/testing.py
new file mode 100644
index 0000000000..df124749c4
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/commands/testing.py
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from typing import Tuple
+
+import click
+
+from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
+from airflow_breeze.commands.common_options import (
+ option_dry_run,
+ option_github_repository,
+ option_image_name,
+ option_image_tag,
+ option_python,
+ option_verbose,
+)
+from airflow_breeze.commands.main import main
+from airflow_breeze.utils.console import console
+from airflow_breeze.utils.run_tests import run_docker_compose_tests
+
+TESTING_COMMANDS = {
+ "name": "Testing",
+ "commands": [
+ "docker-compose-tests",
+ ],
+}
+
+TESTING_PARAMETERS = {
+ "breeze docker-compose-tests": [
+ {
+ "name": "Docker-compose tests flag",
+ "options": [
+ "--image-name",
+ "--python",
+ "--image-tag",
+ ],
+ }
+ ],
+}
+
+
+@main.command(
+ name='docker-compose-tests',
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+)
+@option_verbose
+@option_dry_run
+@option_python
+@option_github_repository
+@option_image_tag
+@option_image_name
+@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
+def docker_compose_tests(
+ verbose: bool,
+ dry_run: bool,
+ python: str,
+ github_repository: str,
+ image_name: str,
+ image_tag: str,
+ extra_pytest_args: Tuple,
+):
+ """Run docker-compose tests."""
+ if image_name is None:
+ build_params = BuildProdParams(
+ python=python, image_tag=image_tag, github_repository=github_repository
+ )
+ image_name = build_params.airflow_image_name_with_tag
+ console.print(f"[bright_blue]Running docker-compose with PROD image: {image_name}[/]")
+ return_code, info = run_docker_compose_tests(
+ image_name=image_name,
+ verbose=verbose,
+ dry_run=dry_run,
+ extra_pytest_args=extra_pytest_args,
+ )
+ sys.exit(return_code)
diff --git a/dev/breeze/src/airflow_breeze/utils/python_versions.py b/dev/breeze/src/airflow_breeze/utils/python_versions.py
new file mode 100644
index 0000000000..e3e7523454
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/utils/python_versions.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from typing import List
+
+from airflow_breeze.global_constants import ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS
+from airflow_breeze.utils.console import console
+
+
+def get_python_version_list(python_versions: str) -> List[str]:
+ """
+ Retrieve and validate space-separated list of Python versions and return them in the form of list.
+ :param python_versions: space separated list of Python versions
+ :return: List of python versions
+ """
+ python_version_list = python_versions.split(" ")
+ errors = False
+ for python in python_version_list:
+ if python not in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS:
+ console.print(f"[red]The Python version {python} passed in {python_versions} is wrong.[/]")
+ errors = True
+ if errors:
+ console.print(
+ f"\nSome of the Python versions passed are not in the "
+ f"list: {ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS}. Quitting.\n"
+ )
+ sys.exit(1)
+ return python_version_list
diff --git a/dev/breeze/src/airflow_breeze/utils/recording.py b/dev/breeze/src/airflow_breeze/utils/recording.py
new file mode 100644
index 0000000000..0ec34edcca
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/utils/recording.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import atexit
+import os
+from typing import Optional
+
+import rich
+from rich.console import Console
+
+from airflow_breeze.utils.path_utils import in_autocomplete
+
+output_file_for_recording = os.environ.get('RECORD_BREEZE_OUTPUT_FILE')
+
+help_console: Optional[Console] = None
+
+DEFAULT_COLUMNS = 129
+
+
+def enable_recording_of_help_output(path: str, title: Optional[str], width: Optional[str]):
+ import rich_click as click
+
+ if not title:
+ title = "Breeze screenshot"
+ if not width:
+ width_int = DEFAULT_COLUMNS
+ else:
+ width_int = int(width)
+
+ def save_ouput_as_svg():
+ if help_console:
+ help_console.save_svg(path=path, title=title)
+
+ class RecordingConsole(rich.console.Console):
+ def __init__(self, **kwargs):
+ super().__init__(record=True, width=width_int, force_terminal=True, **kwargs)
+ global help_console
+ help_console = self
+
+ atexit.register(save_ouput_as_svg)
+ click.rich_click.MAX_WIDTH = width_int
+ click.formatting.FORCED_WIDTH = width_int - 2
+ click.rich_click.COLOR_SYSTEM = "standard"
+ # monkeypatch rich_click console to record help (rich_click does not allow passing extra args to console)
+ click.rich_click.Console = RecordingConsole
+
+
+if output_file_for_recording and not in_autocomplete():
+ enable_recording_of_help_output(
+ path=output_file_for_recording,
+ title=os.environ.get('RECORD_BREEZE_TITLE'),
+ width=os.environ.get('RECORD_BREEZE_WIDTH'),
+ )
+else:
+ try:
+ import click
+
+ columns = os.get_terminal_size().columns
+ click.formatting.FORCED_WIDTH = columns - 2 if columns else DEFAULT_COLUMNS
+ except OSError:
+ pass
diff --git a/images/breeze/output-cleanup.svg b/images/breeze/output-cleanup.svg
index 4d4c22e8d2..ced3f3d578 100644
--- a/images/breeze/output-cleanup.svg
+++ b/images/breeze/output-cleanup.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="670" viewBox="0 0 1720.0 670"
+<svg width="1720.0" height="692" viewBox="0 0 1720.0 692"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -124,10 +124,11 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--all</span><span class="r1"> </span><span class="r1">Also remove currently downloaded Breeze images. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">[default: apache/airflow]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
[airflow] 01/02: nIcer handling of cached Breeze parameters
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e2ea6a7e563621569675b6802726bd245a5ee6e6
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sun Apr 24 00:48:03 2022 +0200
nIcer handling of cached Breeze parameters
The cached parameters in Python Breeze were largely based on
Bash implementation. They did the job but required pretty
cumbersome synchronization of cached values with parameters
passed and it was easy to forget about this as you had to
do it sepearately in each method that had potentially
cacheable parameters.
In this PR we take advantage of the Click class hiarchy and their
extendability. We've already extended Click Choice parameter
to be much better formatted for long list of choices but we take
this a bit further now with adding new type of parameter that
can cache the values between runs.
This has multiple advantages:
* we do not have to remember about synchroniation - parameters
automatically read/write their values from cache as they are
used
* we can automatically set parameters to default when wrong
value is passed. This is nice as user does not have to
re-run the command because the values are corrected
on-the-flight.
* the parameters can have their default values displayed in
help screen (we use sentinel default that we can use to
detect if value was passed from parameter or taken from default)
* the parameters can also have their <current> values marked in
list of choices - so the user in the help screen can see not
only the default but also which value is currently selected
and will be used if you do not pass any parameter.
---
BREEZE.rst | 9 +-
dev/breeze/src/airflow_breeze/breeze.py | 167 +++++++++++++++------
.../build_image/ci/build_ci_image.py | 19 +--
.../build_image/prod/build_prod_image.py | 18 +--
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 11 +-
dev/breeze/src/airflow_breeze/utils/cache.py | 32 +---
dev/breeze/tests/test_build_image.py | 60 --------
dev/breeze/tests/test_prod_image.py | 94 ------------
images/breeze/output-build-image.svg | 7 +-
images/breeze/output-build-prod-image.svg | 7 +-
images/breeze/output-commands.svg | 12 +-
images/breeze/output-config.svg | 15 +-
images/breeze/output-docker-compose-tests.svg | 5 +-
images/breeze/output-find-newer-dependencies.svg | 4 +-
images/breeze/output-generate-constraints.svg | 7 +-
images/breeze/output-pull-image.svg | 4 +-
images/breeze/output-pull-prod-image.svg | 4 +-
images/breeze/output-shell.svg | 12 +-
images/breeze/output-start-airflow.svg | 15 +-
images/breeze/output-verify-image.svg | 5 +-
images/breeze/output-verify-prod-image.svg | 5 +-
21 files changed, 200 insertions(+), 312 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 25f6c7c85f..f164e7fdff 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -363,19 +363,20 @@ You can use additional ``breeze`` flags to choose your environment. You can spec
version to use, and backend (the meta-data database). Thanks to that, with Breeze, you can recreate the same
environments as we have in matrix builds in the CI.
-For example, you can choose to run Python 3.7 tests with MySQL as backend and in the Docker environment as
-follows:
+For example, you can choose to run Python 3.7 tests with MySQL as backend and with mysql version 8
+as follows:
.. code-block:: bash
- breeze --python 3.7 --backend mysql
+ breeze --python 3.7 --backend mysql --mysql-version 8
The choices you make are persisted in the ``./.build/`` cache directory so that next time when you use the
``breeze`` script, it could use the values that were used previously. This way you do not have to specify
them when you run the script. You can delete the ``.build/`` directory in case you want to restore the
default settings.
-The defaults when you run the Breeze environment are Python 3.7 version and SQLite database.
+You can see which value of the parameters that can be stored persistently in cache marked with >VALUE<
+in the help of the commands.
Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy``
command but it is very similar to current ``breeze`` command):
diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py
index 4e522c7120..850e70a0cd 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -23,7 +23,7 @@ import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
-from typing import IO, List, Optional, Tuple
+from typing import IO, Any, List, Optional, Tuple
import rich
@@ -490,12 +490,9 @@ except ImportError:
from click import Context, IntRange
-from airflow_breeze.build_image.ci.build_ci_image import build_ci_image, get_ci_image_build_params
+from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
-from airflow_breeze.build_image.prod.build_prod_image import (
- build_production_image,
- get_prod_image_build_params,
-)
+from airflow_breeze.build_image.prod.build_prod_image import build_production_image
from airflow_breeze.global_constants import (
ALLOWED_BACKENDS,
ALLOWED_BUILD_CACHE,
@@ -521,9 +518,10 @@ from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
from airflow_breeze.shell.enter_shell import enter_shell
from airflow_breeze.utils.cache import (
check_if_cache_exists,
+ check_if_values_allowed,
delete_cache,
+ read_and_validate_value_from_cache,
read_from_cache_file,
- synchronize_parameters_with_cache,
touch_cache_file,
write_to_cache_file,
)
@@ -560,6 +558,8 @@ class BetterChoice(click.Choice):
that when the long list of choices does not wrap on words.
"""
+ name = "BetterChoice"
+
def get_metavar(self, param) -> str:
choices_str = " | ".join(self.choices)
# Use curly braces to indicate a required argument.
@@ -574,6 +574,68 @@ class BetterChoice(click.Choice):
return f"[{choices_str}]"
+@dataclass
+class CacheableDefault:
+ value: Any
+
+ def __repr__(self):
+ return self.value
+
+
+class CacheableChoice(click.Choice):
+ """
+ This class implements caching of values from the last use.
+ """
+
+ def convert(self, value, param, ctx):
+ param_name = param.envvar if param.envvar else param.name.upper()
+ if isinstance(value, CacheableDefault):
+ is_cached, new_value = read_and_validate_value_from_cache(param_name, value.value)
+ if not is_cached:
+ console.print(
+ f"\n[bright_blue]Default value of {param.name} " f"parameter {new_value} used.[/]\n"
+ )
+ else:
+ allowed, allowed_values = check_if_values_allowed(param_name, value)
+ if allowed:
+ new_value = value
+ write_to_cache_file(param_name, new_value, check_allowed_values=False)
+ else:
+ new_value = allowed_values[0]
+ console.print(
+ f"\n[yellow]The value {value} is not allowed for parameter {param.name}. "
+ f"Setting default value to {new_value}"
+ )
+ write_to_cache_file(param_name, new_value, check_allowed_values=False)
+ return super().convert(new_value, param, ctx)
+
+ def get_metavar(self, param) -> str:
+ param_name = param.envvar if param.envvar else param.name.upper()
+ current_value = (
+ read_from_cache_file(param_name) if not output_file_for_recording else param.default.value
+ )
+ if not current_value:
+ current_choices = self.choices
+ else:
+ current_choices = [
+ f">{choice}<" if choice == current_value else choice for choice in self.choices
+ ]
+ choices_str = " | ".join(current_choices)
+ # Use curly braces to indicate a required argument.
+ if param.required and param.param_type_name == "argument":
+ return f"{{{choices_str}}}"
+
+ if param.param_type_name == "argument" and param.nargs == -1:
+ # avoid double [[ for multiple args
+ return f"{choices_str}"
+
+ # Use square braces to indicate an option or optional argument.
+ return f"[{choices_str}]"
+
+ def __init__(self, choices, case_sensitive: bool = True) -> None:
+ super().__init__(choices=choices, case_sensitive=case_sensitive)
+
+
option_verbose = click.option(
"-v", "--verbose", is_flag=True, help="Print verbose information about performed steps.", envvar='VERBOSE'
)
@@ -597,16 +659,21 @@ option_answer = click.option(
option_python = click.option(
'-p',
'--python',
- type=BetterChoice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS),
- help='Python major/minor version used in Airflow image for PROD/CI images.',
+ type=CacheableChoice(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS),
+ default=CacheableDefault(value=ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0]),
+ show_default=True,
+ help='Python major/minor version used in Airflow image for images.',
envvar='PYTHON_MAJOR_MINOR_VERSION',
)
option_backend = click.option(
'-b',
'--backend',
+ type=CacheableChoice(ALLOWED_BACKENDS),
+ default=CacheableDefault(value=ALLOWED_BACKENDS[0]),
+ show_default=True,
help="Database backend to use.",
- type=BetterChoice(ALLOWED_BACKENDS),
+ envvar='BACKEND',
)
option_integration = click.option(
@@ -617,15 +684,30 @@ option_integration = click.option(
)
option_postgres_version = click.option(
- '-P', '--postgres-version', help="Version of Postgres.", type=BetterChoice(ALLOWED_POSTGRES_VERSIONS)
+ '-P',
+ '--postgres-version',
+ type=CacheableChoice(ALLOWED_POSTGRES_VERSIONS),
+ default=CacheableDefault(ALLOWED_POSTGRES_VERSIONS[0]),
+ show_default=True,
+ help="Version of Postgres used.",
)
option_mysql_version = click.option(
- '-M', '--mysql-version', help="Version of MySQL.", type=BetterChoice(ALLOWED_MYSQL_VERSIONS)
+ '-M',
+ '--mysql-version',
+ help="Version of MySQL used.",
+ type=CacheableChoice(ALLOWED_MYSQL_VERSIONS),
+ default=CacheableDefault(ALLOWED_MYSQL_VERSIONS[0]),
+ show_default=True,
)
option_mssql_version = click.option(
- '-S', '--mssql-version', help="Version of MsSQL.", type=BetterChoice(ALLOWED_MSSQL_VERSIONS)
+ '-S',
+ '--mssql-version',
+ help="Version of MsSQL used.",
+ type=CacheableChoice(ALLOWED_MSSQL_VERSIONS),
+ default=CacheableDefault(ALLOWED_MSSQL_VERSIONS[0]),
+ show_default=True,
)
option_executor = click.option(
@@ -967,7 +1049,7 @@ argument_packages = click.argument(
@option_verbose
@main.command()
-def version(verbose: bool):
+def version(verbose: bool, python: str):
"""Print information about version of apache-airflow-breeze."""
console.print(ASCIIART, style=ASCIIART_STYLE)
console.print(f"\n[bright_blue]Breeze version: {VERSION}[/]")
@@ -1176,13 +1258,12 @@ def build_image(
with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
python_version_list = get_python_version_list(python_versions)
for python in python_version_list:
- params = get_ci_image_build_params(parameters_passed)
+ params = BuildCiParams(**parameters_passed)
params.python = python
params.answer = answer
run_build(ci_image_params=params)
else:
- params = get_ci_image_build_params(parameters_passed)
- synchronize_parameters_with_cache(params, parameters_passed)
+ params = BuildCiParams(**parameters_passed)
run_build(ci_image_params=params)
@@ -1254,7 +1335,6 @@ def pull_image(
)
else:
image_params = BuildCiParams(image_tag=image_tag, python=python, github_repository=github_repository)
- synchronize_parameters_with_cache(image_params, {"python": python})
return_code, info = run_pull_image(
image_params=image_params,
dry_run=dry_run,
@@ -1293,9 +1373,7 @@ def verify_image(
):
"""Verify CI image."""
if image_name is None:
- build_params = get_ci_image_build_params(
- {"python": python, "image_tag": image_tag, "github_repository": github_repository}
- )
+ build_params = BuildCiParams(python=python, image_tag=image_tag, github_repository=github_repository)
image_name = build_params.airflow_image_name_with_tag
console.print(f"[bright_blue]Verifying CI image: {image_name}[/]")
return_code, info = verify_an_image(
@@ -1400,13 +1478,12 @@ def build_prod_image(
with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
python_version_list = get_python_version_list(python_versions)
for python in python_version_list:
- params = get_prod_image_build_params(parameters_passed)
+ params = BuildProdParams(**parameters_passed)
params.python = python
params.answer = answer
run_build(prod_image_params=params)
else:
- params = get_prod_image_build_params(parameters_passed)
- synchronize_parameters_with_cache(params, parameters_passed)
+ params = BuildProdParams(**parameters_passed)
run_build(prod_image_params=params)
@@ -1459,7 +1536,6 @@ def pull_prod_image(
image_params = BuildProdParams(
image_tag=image_tag, python=python, github_repository=github_repository
)
- synchronize_parameters_with_cache(image_params, {"python": python})
return_code, info = run_pull_image(
image_params=image_params,
dry_run=dry_run,
@@ -1498,8 +1574,8 @@ def verify_prod_image(
):
"""Verify Production image."""
if image_name is None:
- build_params = get_prod_image_build_params(
- {"python": python, "image_tag": image_tag, "github_repository": github_repository}
+ build_params = BuildProdParams(
+ python=python, image_tag=image_tag, github_repository=github_repository
)
image_name = build_params.airflow_image_name_with_tag
console.print(f"[bright_blue]Verifying PROD image: {image_name}[/]")
@@ -1538,8 +1614,8 @@ def docker_compose_tests(
):
"""Run docker-compose tests."""
if image_name is None:
- build_params = get_prod_image_build_params(
- {"python": python, "image_tag": image_tag, "github_repository": github_repository}
+ build_params = BuildProdParams(
+ python=python, image_tag=image_tag, github_repository=github_repository
)
image_name = build_params.airflow_image_name_with_tag
console.print(f"[bright_blue]Running docker-compose with PROD image: {image_name}[/]")
@@ -1689,16 +1765,26 @@ def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Option
@main.command(name='config')
@option_python
@option_backend
+@option_postgres_version
+@option_mysql_version
+@option_mssql_version
@click.option('-C/-c', '--cheatsheet/--no-cheatsheet', help="Enable/disable cheatsheet.", default=None)
@click.option('-A/-a', '--asciiart/--no-asciiart', help="Enable/disable ASCIIart.", default=None)
-def change_config(python, backend, cheatsheet, asciiart):
+def change_config(
+ python: str,
+ backend: str,
+ postgres_version: str,
+ mysql_version: str,
+ mssql_version: str,
+ cheatsheet: bool,
+ asciiart: bool,
+):
"""
Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).
"""
asciiart_file = "suppress_asciiart"
cheatsheet_file = "suppress_cheatsheet"
- python_file = 'PYTHON_MAJOR_MINOR_VERSION'
- backend_file = 'BACKEND'
+
if asciiart is not None:
if asciiart:
delete_cache(asciiart_file)
@@ -1713,12 +1799,6 @@ def change_config(python, backend, cheatsheet, asciiart):
elif cheatsheet is not None:
touch_cache_file(cheatsheet_file)
console.print('[bright_blue]Disable Cheatsheet[/]')
- if python is not None:
- write_to_cache_file(python_file, python)
- console.print(f'[bright_blue]Python default value set to: {python}[/]')
- if backend is not None:
- write_to_cache_file(backend_file, backend)
- console.print(f'[bright_blue]Backend default value set to: {backend}[/]')
def get_status(file: str):
return "disabled" if check_if_cache_exists(file) else "enabled"
@@ -1726,8 +1806,13 @@ def change_config(python, backend, cheatsheet, asciiart):
console.print()
console.print("[bright_blue]Current configuration:[/]")
console.print()
- console.print(f"[bright_blue]* Python: {read_from_cache_file(python_file)}[/]")
- console.print(f"[bright_blue]* Backend: {read_from_cache_file(backend_file)}[/]")
+ console.print(f"[bright_blue]* Python: {python}[/]")
+ console.print(f"[bright_blue]* Backend: {backend}[/]")
+ console.print()
+ console.print(f"[bright_blue]* Postgres version: {postgres_version}[/]")
+ console.print(f"[bright_blue]* MySQL version: {mysql_version}[/]")
+ console.print(f"[bright_blue]* MsSQL version: {mssql_version}[/]")
+ console.print()
console.print(f"[bright_blue]* ASCIIART: {get_status(asciiart_file)}[/]")
console.print(f"[bright_blue]* Cheatsheet: {get_status(cheatsheet_file)}[/]")
console.print()
@@ -2162,7 +2247,6 @@ def generate_constraints(
shell_params = ShellParams(
image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
)
- synchronize_parameters_with_cache(shell_params, {"python": python})
console.print("\n[yellow]Use this command to build the image:[/]\n")
console.print(
f" breeze build-image --python'{shell_params.python}' "
@@ -2191,7 +2275,6 @@ def generate_constraints(
shell_params = ShellParams(
image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
)
- synchronize_parameters_with_cache(shell_params, {"python": python})
return_code, info = run_generate_constraints(
shell_params=shell_params,
dry_run=dry_run,
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
index 9740da54b2..b1922521d3 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
@@ -17,14 +17,14 @@
import multiprocessing as mp
import sys
-from typing import Dict, List, Tuple
+from typing import List, Tuple
from airflow_breeze.build_image.ci.build_ci_params import (
OPTIONAL_CI_IMAGE_ARGS,
REQUIRED_CI_IMAGE_ARGS,
BuildCiParams,
)
-from airflow_breeze.utils.cache import synchronize_parameters_with_cache, touch_cache_file
+from airflow_breeze.utils.cache import touch_cache_file
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.confirm import Answer, user_confirm
from airflow_breeze.utils.console import console
@@ -189,21 +189,6 @@ def build_ci_image(
return build_result.returncode, f"Image build: {ci_image_params.python}"
-def get_ci_image_build_params(parameters_passed: Dict) -> BuildCiParams:
- """
- Converts parameters received as dict into BuildCiParams. In case cacheable
- parameters are missing, it reads the last used value for that parameter
- from the cache and if it is not found, it uses default value for that parameter.
-
- This method updates cached based on parameters passed via Dict.
-
- :param parameters_passed: parameters to use when constructing BuildCiParams
- """
- ci_image_params = BuildCiParams(**parameters_passed)
- synchronize_parameters_with_cache(ci_image_params, parameters_passed)
- return ci_image_params
-
-
def build_ci_image_in_parallel(
verbose: bool, dry_run: bool, parallelism: int, python_version_list: List[str], **kwargs
):
diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
index 87383c455b..8aea74ce82 100644
--- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
@@ -17,10 +17,9 @@
"""Command to build PROD image."""
import contextlib
import sys
-from typing import Dict, Tuple
+from typing import Tuple
from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.utils.cache import synchronize_parameters_with_cache
from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import console
from airflow_breeze.utils.docker_command_utils import (
@@ -120,21 +119,6 @@ def check_docker_context_files(install_from_docker_context_files: bool):
sys.exit(1)
-def get_prod_image_build_params(parameters_passed: Dict) -> BuildProdParams:
- """
- Converts parameters received as dict into BuildProdParams. In case cacheable
- parameters are missing, it reads the last used value for that parameter
- from the cache and if it is not found, it uses default value for that parameter.
-
- This method updates cached based on parameters passed via Dict.
-
- :param parameters_passed: parameters to use when constructing BuildCiParams
- """
- prod_image_params = BuildProdParams(**parameters_passed)
- synchronize_parameters_with_cache(prod_image_params, parameters_passed)
- return prod_image_params
-
-
def build_production_image(
verbose: bool, dry_run: bool, with_ci_group: bool, prod_image_params: BuildProdParams
) -> Tuple[int, str]:
diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
index 309cf316ea..933f85ca4d 100644
--- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
+++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
@@ -21,10 +21,11 @@ from pathlib import Path
from typing import Dict, Union
from airflow_breeze import global_constants
-from airflow_breeze.build_image.ci.build_ci_image import build_ci_image, get_ci_image_build_params
+from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
+from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.cache import (
- check_cached_value_is_allowed,
+ read_and_validate_value_from_cache,
read_from_cache_file,
write_to_cache_file,
)
@@ -62,7 +63,7 @@ def synchronize_cached_params(parameters_passed_by_the_user: Dict[str, str]) ->
write_to_cache_file(param_name, user_param_value)
else:
param_value = getattr(global_constants, SOURCE_OF_DEFAULT_VALUES_FOR_VARIABLES[param])
- _, user_param_value = check_cached_value_is_allowed(param_name, param_value)
+ _, user_param_value = read_and_validate_value_from_cache(param_name, param_value)
updated_params[param] = user_param_value
return updated_params
@@ -119,9 +120,7 @@ def run_shell_with_build_image_checks(
build_ci_image_check_cache = Path(
BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python}"
)
- ci_image_params = get_ci_image_build_params(
- {"python": shell_params.python, "upgrade_to_newer_dependencies": "false"}
- )
+ ci_image_params = BuildCiParams(python=shell_params.python, upgrade_to_newer_dependencies="false")
if build_ci_image_check_cache.exists():
console.print(f'[bright_blue]{shell_params.the_image_type} image already built locally.[/]')
else:
diff --git a/dev/breeze/src/airflow_breeze/utils/cache.py b/dev/breeze/src/airflow_breeze/utils/cache.py
index d2e1117ab3..2319ee4505 100644
--- a/dev/breeze/src/airflow_breeze/utils/cache.py
+++ b/dev/breeze/src/airflow_breeze/utils/cache.py
@@ -24,7 +24,7 @@ the shell with the same version as the "previous run".
import sys
from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, List, Optional, Tuple
from airflow_breeze import global_constants
from airflow_breeze.utils.console import console
@@ -70,9 +70,11 @@ def write_to_cache_file(param_name: str, param_value: str, check_allowed_values:
sys.exit(1)
-def check_cached_value_is_allowed(param_name: str, default_param_value: str) -> Tuple[bool, Optional[str]]:
+def read_and_validate_value_from_cache(
+ param_name: str, default_param_value: str
+) -> Tuple[bool, Optional[str]]:
"""
- Checks if the cache is present and whether its value is valid according to current rules.
+ Reads and validates value from cache is present and whether its value is valid according to current rules.
It could happen that the allowed values have been modified since the last time cached value was set,
so this check is crucial to check outdated values.
If the value is not set or in case the cached value stored is not currently allowed,
@@ -114,27 +116,3 @@ def delete_cache(param_name: str) -> bool:
(Path(BUILD_CACHE_DIR) / f".{param_name}").unlink()
deleted = True
return deleted
-
-
-def synchronize_parameters_with_cache(
- image_params: Any, parameters_passed_via_command_line: Dict[str, str]
-) -> None:
- """
- Synchronizes cacheable parameters between executions. It reads values from cache and updates
- them wen done. It is only done for parameters that are relevant for image build.
-
- :param image_params: parameters of the build
- :param parameters_passed_via_command_line: parameters that were passed by command line
- """
- cacheable_parameters = {
- 'python': 'PYTHON_MAJOR_MINOR_VERSION',
- }
- for parameter, cache_key in cacheable_parameters.items():
- value_from_parameter = parameters_passed_via_command_line.get(parameter)
- if value_from_parameter:
- write_to_cache_file(cache_key, value_from_parameter, check_allowed_values=True)
- setattr(image_params, parameter, value_from_parameter)
- else:
- is_cached, value = check_cached_value_is_allowed(cache_key, getattr(image_params, parameter))
- if is_cached:
- setattr(image_params, parameter, value)
diff --git a/dev/breeze/tests/test_build_image.py b/dev/breeze/tests/test_build_image.py
deleted file mode 100644
index 77c9c7759d..0000000000
--- a/dev/breeze/tests/test_build_image.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from unittest.mock import patch
-
-import pytest
-
-from airflow_breeze.build_image.ci.build_ci_image import get_ci_image_build_params
-
-
-@pytest.mark.parametrize(
- 'parameters, expected_build_params, cached_values, written_cache_version',
- [
- ({}, {"python": "3.7"}, {}, False), # default value no params
- ({"python": "3.8"}, {"python": "3.8"}, {}, "3.8"), # default value override params
- ({}, {"python": "3.8"}, {'PYTHON_MAJOR_MINOR_VERSION': "3.8"}, False), # value from cache
- (
- {"python": "3.9"},
- {"python": "3.9"},
- {'PYTHON_MAJOR_MINOR_VERSION': "3.8"},
- "3.9",
- ), # override cache with passed param
- ],
-)
-def test_get_image_params(parameters, expected_build_params, cached_values, written_cache_version):
- with patch('airflow_breeze.utils.cache.read_from_cache_file') as read_from_cache_mock, patch(
- 'airflow_breeze.utils.cache.check_if_cache_exists'
- ) as check_if_cache_exists_mock, patch(
- 'airflow_breeze.utils.cache.write_to_cache_file'
- ) as write_to_cache_file_mock, patch(
- 'airflow_breeze.utils.cache.check_cached_value_is_allowed'
- ) as check_cache_and_write_mock:
- check_if_cache_exists_mock.return_value = True
- check_cache_and_write_mock.side_effect = lambda cache_key, default_value: (
- cache_key in cached_values,
- cached_values[cache_key] if cache_key in cached_values else default_value,
- )
- read_from_cache_mock.side_effect = lambda param_name: cached_values.get(param_name)
- build_parameters = get_ci_image_build_params(parameters)
- for param, param_value in expected_build_params.items():
- assert getattr(build_parameters, param) == param_value
- if written_cache_version:
- write_to_cache_file_mock.assert_called_once_with(
- "PYTHON_MAJOR_MINOR_VERSION", written_cache_version, check_allowed_values=True
- )
- else:
- write_to_cache_file_mock.assert_not_called()
diff --git a/dev/breeze/tests/test_prod_image.py b/dev/breeze/tests/test_prod_image.py
deleted file mode 100644
index a93e3ecf04..0000000000
--- a/dev/breeze/tests/test_prod_image.py
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from typing import Dict, Union
-from unittest.mock import patch
-
-import pytest
-
-from airflow_breeze.build_image.prod.build_prod_image import get_prod_image_build_params
-
-default_params: Dict[str, Union[str, bool]] = {
- 'docker_cache': "pulled",
- 'disable_mysql_client_installation': False,
- 'disable_mssql_client_installation': False,
- 'disable_postgres_client_installation': False,
- 'install_from_docker_context_files': False,
- 'disable_airflow_repo_cache': False,
- 'upgrade_to_newer_dependencies': "false",
- 'install_providers_from_sources': False,
- 'cleanup_docker_context_files': False,
- 'prepare_buildx_cache': False,
-}
-
-params_python8 = {**default_params, "python": "3.8"} # type: Dict[str, Union[str, bool]]
-
-params_python9 = {**default_params, "python": "3.9"} # type: Dict[str, Union[str, bool]]
-
-
-@pytest.mark.parametrize(
- 'description, parameters, expected_build_params, cached_values, written_cache_version, check_if_allowed',
- [
- ("default value no cache", default_params, {"python": "3.7"}, {}, "3.7", False),
- ("passed value different no cache", params_python8, {"python": "3.8"}, {}, "3.8", True),
- (
- "passed value same as cache",
- params_python8,
- {"python": "3.8"},
- {'PYTHON_MAJOR_MINOR_VERSION': "3.8"},
- "3.8",
- True,
- ),
- (
- "passed value different than cache",
- params_python9,
- {"python": "3.9"},
- {'PYTHON_MAJOR_MINOR_VERSION': "3.8"},
- "3.9",
- True,
- ),
- ],
-)
-def test_get_image_params(
- description, parameters, expected_build_params, cached_values, written_cache_version, check_if_allowed
-):
- with patch('airflow_breeze.utils.cache.read_from_cache_file') as read_from_cache_mock, patch(
- 'airflow_breeze.utils.cache.check_if_cache_exists'
- ) as check_if_cache_exists_mock, patch(
- 'airflow_breeze.utils.cache.write_to_cache_file'
- ) as write_to_cache_file_mock, patch(
- 'airflow_breeze.utils.cache.read_from_cache_file'
- ) as read_from_cache_file:
- check_if_cache_exists_mock.return_value = True
- read_from_cache_file.side_effect = lambda cache_key: (
- cache_key in cached_values,
- cached_values[cache_key] if cache_key in cached_values else None,
- )
- read_from_cache_mock.side_effect = lambda param_name: cached_values.get(param_name)
- build_parameters = get_prod_image_build_params(parameters)
- for param, param_value in expected_build_params.items():
- assert getattr(build_parameters, param) == param_value
- if written_cache_version:
- if check_if_allowed:
- write_to_cache_file_mock.assert_called_once_with(
- "PYTHON_MAJOR_MINOR_VERSION", written_cache_version, check_allowed_values=True
- )
- else:
- write_to_cache_file_mock.assert_called_once_with(
- "PYTHON_MAJOR_MINOR_VERSION", written_cache_version
- )
- else:
- write_to_cache_file_mock.assert_not_called()
diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg
index d2693055b4..0eceba7182 100644
--- a/images/breeze/output-build-image.svg
+++ b/images/breeze/output-build-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1506" viewBox="0 0 1720.0 1506"
+<svg width="1720.0" height="1528" viewBox="0 0 1720.0 1528"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -121,8 +121,9 @@
<div><span class="r1"> </span><span class="r1">Build CI image. Include building multiple images for all python versions (sequentially). </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--upgrade-to-newer-dependencies</span><span class="r1"> </span><span class="r6">-u</span><span class="r1"> </span><span class="r1">When other than "false", upgrade all PIP packages to latest.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: false] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg
index 1a0471b2af..16f732b769 100644
--- a/images/breeze/output-build-prod-image.svg
+++ b/images/breeze/output-build-prod-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1836" viewBox="0 0 1720.0 1836"
+<svg width="1720.0" height="1858" viewBox="0 0 1720.0 1858"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -121,8 +121,9 @@
<div><span class="r1"> </span><span class="r1">Build Production image. Include building multiple images for all or selected Python versions sequentially. </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--install-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Install version of Airflow from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--upgrade-to-newer-dependencies</span><span class="r1"> </span><span class="r6">-u</span><span class="r1"> </span><span class="r1">When other than "false", upgrade all PIP packages to latest.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: false] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg
index 78dcf803f3..e5cdec13fb 100644
--- a/images/breeze/output-commands.svg
+++ b/images/breeze/output-commands.svg
@@ -119,13 +119,13 @@
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze [OPTIONS] COMMAND [ARGS]... </span></div>
<div><span class="r2"> </span></div>
<div><span class="r4">╭─ Basic flags for the default (shell) command ────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(sqlite | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r4">[default: sqlite]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Use (reinstall at entry) Airflow version from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres.</span><span class="r1"> </span><span class="r7">(10 | 11 | 12 | 13)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL.</span><span class="r1"> </span><span class="r7">(5.7 | 8)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL.</span><span class="r1"> </span><span class="r7">(2017-latest | 2019-latest)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--forward-credentials</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Forward local credentials to container when running. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--db-reset</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Reset DB when entering the container. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
diff --git a/images/breeze/output-config.svg b/images/breeze/output-config.svg
index 92624fa654..5d20779549 100644
--- a/images/breeze/output-config.svg
+++ b/images/breeze/output-config.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="692" viewBox="0 0 1720.0 692"
+<svg width="1720.0" height="802" viewBox="0 0 1720.0 802"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -121,14 +121,19 @@
<div><span class="r1"> </span><span class="r1">Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Config flags ───────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(sqlite | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: sqlite] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--cheatsheet</span><span class="r1">/</span><span class="r5">--no-cheatsheet</span><span class="r1"> </span><span class="r6">-C</span><span class="r1">/</span><span class="r6">-c</span><span class="r1"> </span><span class="r1">Enable/disable cheatsheet. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--asciiart</span><span class="r1">/</span><span class="r5">--no-asciiart</span><span class="r1"> </span><span class="r6">-A</span><span class="r1">/</span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Enable/disable ASCIIart. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
diff --git a/images/breeze/output-docker-compose-tests.svg b/images/breeze/output-docker-compose-tests.svg
index 7dc3510792..573b17bc55 100644
--- a/images/breeze/output-docker-compose-tests.svg
+++ b/images/breeze/output-docker-compose-tests.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="714" viewBox="0 0 1720.0 714"
+<svg width="1720.0" height="736" viewBox="0 0 1720.0 736"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -122,7 +122,8 @@
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-name</span><span class="r1"> </span><span class="r6">-n</span><span class="r1"> Name of the image to verify (overrides </span><span class="r5">--python</span><span class="r1"> and </span><span class="r5">--image-tag</span><span class="r1">). </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images.</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
diff --git a/images/breeze/output-find-newer-dependencies.svg b/images/breeze/output-find-newer-dependencies.svg
index cab8045a1b..0fbf3aa534 100644
--- a/images/breeze/output-find-newer-dependencies.svg
+++ b/images/breeze/output-find-newer-dependencies.svg
@@ -121,8 +121,8 @@
<div><span class="r1"> </span><span class="r1">Finds which dependencies are being upgraded. </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--timezone</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Timezone to use during the check</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--constraints-branch</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Constraint branch to use to find newer dependencies</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--updated-on-or-after</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Date when the release was updated after</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-generate-constraints.svg b/images/breeze/output-generate-constraints.svg
index 7f92500124..298da74edc 100644
--- a/images/breeze/output-generate-constraints.svg
+++ b/images/breeze/output-generate-constraints.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="934" viewBox="0 0 1720.0 934"
+<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -122,8 +122,9 @@
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--generate-constraints-mode</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Mode of generating constraints</span><span class="r1"> </span><span class="r7">(source-providers | pypi-providers | no-providers)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: source-providers] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
diff --git a/images/breeze/output-pull-image.svg b/images/breeze/output-pull-image.svg
index e77af30370..c5ea6d91ae 100644
--- a/images/breeze/output-pull-image.svg
+++ b/images/breeze/output-pull-image.svg
@@ -122,8 +122,8 @@
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verify-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Verify image. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--wait-for-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Wait until image is available. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Tags the image as latest after pulling. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-pull-prod-image.svg b/images/breeze/output-pull-prod-image.svg
index c432345694..a25ea871ba 100644
--- a/images/breeze/output-pull-prod-image.svg
+++ b/images/breeze/output-pull-prod-image.svg
@@ -122,8 +122,8 @@
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verify-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Verify image. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--wait-for-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Wait until image is available. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Tags the image as latest after pulling. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-shell.svg b/images/breeze/output-shell.svg
index 51af37d741..4ffd4c8b21 100644
--- a/images/breeze/output-shell.svg
+++ b/images/breeze/output-shell.svg
@@ -121,13 +121,13 @@
<div><span class="r1"> </span><span class="r1">Enter breeze.py environment. this is the default command use when no other is selected. </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(sqlite | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r4">[default: sqlite]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Use (reinstall at entry) Airflow version from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres.</span><span class="r1"> </span><span class="r7">(10 | 11 | 12 | 13)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL.</span><span class="r1"> </span><span class="r7">(5.7 | 8)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL.</span><span class="r1"> </span><span class="r7">(2017-latest | 2019-latest)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--forward-credentials</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Forward local credentials to container when running. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--db-reset</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Reset DB when entering the container. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
diff --git a/images/breeze/output-start-airflow.svg b/images/breeze/output-start-airflow.svg
index 83101d09e4..5a338cb49e 100644
--- a/images/breeze/output-start-airflow.svg
+++ b/images/breeze/output-start-airflow.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1066" viewBox="0 0 1720.0 1066"
+<svg width="1720.0" height="1088" viewBox="0 0 1720.0 1088"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -121,13 +121,14 @@
<div><span class="r1"> </span><span class="r1">Enter breeze.py environment and starts all Airflow components in the tmux session. </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(sqlite | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r4">[default: sqlite]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Use (reinstall at entry) Airflow version from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres.</span><span class="r1"> </span><span class="r7">(10 | 11 | 12 | 13)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL.</span><span class="r1"> </span><span class="r7">(5.7 | 8)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL.</span><span class="r1"> </span><span class="r7">(2017-latest | 2019-latest)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--load-example-dags</span><span class="r1"> </span><span class="r6">-e</span><span class="r1"> </span><span class="r1">Enable configuration to load example DAGs when starting Airflow. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--load-default-connections</span><span class="r1"> </span><span class="r6">-c</span><span class="r1"> </span><span class="r1">Enable configuration to load default connections when starting Airflow. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--forward-credentials</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Forward local credentials to container when running. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-verify-image.svg b/images/breeze/output-verify-image.svg
index 1d8262869a..2f07f67ad7 100644
--- a/images/breeze/output-verify-image.svg
+++ b/images/breeze/output-verify-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="714" viewBox="0 0 1720.0 714"
+<svg width="1720.0" height="736" viewBox="0 0 1720.0 736"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -122,7 +122,8 @@
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-name</span><span class="r1"> </span><span class="r6">-n</span><span class="r1"> Name of the image to verify (overrides </span><span class="r5">--python</span><span class="r1"> and </span><span class="r5">--image-tag</span><span class="r1">). </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images.</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
diff --git a/images/breeze/output-verify-prod-image.svg b/images/breeze/output-verify-prod-image.svg
index e84c671b42..4560031a3b 100644
--- a/images/breeze/output-verify-prod-image.svg
+++ b/images/breeze/output-verify-prod-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="714" viewBox="0 0 1720.0 714"
+<svg width="1720.0" height="736" viewBox="0 0 1720.0 736"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -122,7 +122,8 @@
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-name</span><span class="r1"> </span><span class="r6">-n</span><span class="r1"> Name of the image to verify (overrides </span><span class="r5">--python</span><span class="r1"> and </span><span class="r5">--image-tag</span><span class="r1">). </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for PROD/CI images.</span><span class="r1"> </span><span class="r7">(3.7 | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>