You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/01/05 17:38:43 UTC
[airflow] branch main updated: rebase conflict resolved (#20338)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 95740a8 rebase conflict resolved (#20338)
95740a8 is described below
commit 95740a87083c703968ce3da45b15113851ef09f7
Author: Bowrna <ma...@gmail.com>
AuthorDate: Wed Jan 5 23:08:05 2022 +0530
rebase conflict resolved (#20338)
---
dev/breeze/src/airflow_breeze/branch_defaults.py | 19 +++
dev/breeze/src/airflow_breeze/breeze.py | 106 ++++++++++++-
dev/breeze/src/airflow_breeze/cache.py | 79 ++++++++++
dev/breeze/src/airflow_breeze/ci/build_image.py | 94 ++++++++++++
dev/breeze/src/airflow_breeze/ci/build_params.py | 123 +++++++++++++++
dev/breeze/src/airflow_breeze/console.py | 21 +++
dev/breeze/src/airflow_breeze/global_constants.py | 179 ++++++++++++++++++++++
dev/breeze/src/airflow_breeze/utils.py | 51 ++++++
8 files changed, 665 insertions(+), 7 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py
new file mode 100644
index 0000000..153981c
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/branch_defaults.py
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+AIRFLOW_BRANCH = "main"
+DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main"
diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py
index c854d32..01b2517 100755
--- a/dev/breeze/src/airflow_breeze/breeze.py
+++ b/dev/breeze/src/airflow_breeze/breeze.py
@@ -21,8 +21,8 @@ from typing import Optional
import click
from click import ClickException
-from rich.console import Console
+from airflow_breeze.console import console
from airflow_breeze.visuals import ASCIIART, ASCIIART_STYLE
NAME = "Breeze2"
@@ -32,8 +32,6 @@ __AIRFLOW_SOURCES_ROOT = Path.cwd()
__AIRFLOW_CFG_FILE = "setup.cfg"
-console = Console(force_terminal=True, color_system="standard", width=180)
-
def get_airflow_sources_root():
return __AIRFLOW_SOURCES_ROOT
@@ -95,13 +93,107 @@ def shell(verbose: bool):
@option_verbose
-@main.command()
-def build_ci_image(verbose: bool):
- """Builds breeze.ci image for breeze.py."""
+@main.command(name='build-ci-image')
+@click.option(
+ '--additional-extras',
+ help='This installs additional extra package while installing airflow in the image.',
+)
+@click.option('-p', '--python', help='Choose your python version')
+@click.option(
+ '--additional-dev-apt-deps', help='Additional apt dev dependencies to use when building the images.'
+)
+@click.option(
+ '--additional-runtime-apt-deps',
+ help='Additional apt runtime dependencies to use when building the images.',
+)
+@click.option(
+ '--additional-python-deps', help='Additional python dependencies to use when building the images.'
+)
+@click.option(
+ '--additional_dev_apt_command', help='Additional command executed before dev apt deps are installed.'
+)
+@click.option(
+ '--additional_runtime_apt_command',
+ help='Additional command executed before runtime apt deps are installed.',
+)
+@click.option(
+ '--additional_dev_apt_env', help='Additional environment variables set when adding dev dependencies.'
+)
+@click.option(
+ '--additional_runtime_apt_env',
+ help='Additional environment variables set when adding runtime dependencies.',
+)
+@click.option('--dev-apt-command', help='The basic command executed before dev apt deps are installed.')
+@click.option(
+ '--dev-apt-deps',
+ help='The basic apt dev dependencies to use when building the images.',
+)
+@click.option(
+ '--runtime-apt-command', help='The basic command executed before runtime apt deps are installed.'
+)
+@click.option(
+ '--runtime-apt-deps',
+ help='The basic apt runtime dependencies to use when building the images.',
+)
+@click.option('--github-repository', help='Choose repository to push/pull image.')
+@click.option('--build-cache', help='Cache option')
+@click.option('--upgrade-to-newer-dependencies', is_flag=True)
+def build_ci_image(
+ verbose: bool,
+ additional_extras: Optional[str],
+ python: Optional[float],
+ additional_dev_apt_deps: Optional[str],
+ additional_runtime_apt_deps: Optional[str],
+ additional_python_deps: Optional[str],
+ additional_dev_apt_command: Optional[str],
+ additional_runtime_apt_command: Optional[str],
+ additional_dev_apt_env: Optional[str],
+ additional_runtime_apt_env: Optional[str],
+ dev_apt_command: Optional[str],
+ dev_apt_deps: Optional[str],
+ runtime_apt_command: Optional[str],
+ runtime_apt_deps: Optional[str],
+ github_repository: Optional[str],
+ build_cache: Optional[str],
+ upgrade_to_newer_dependencies: bool,
+):
+ """Builds docker CI image without entering the container."""
+ from airflow_breeze.ci.build_image import build_image
+
if verbose:
console.print(f"\n[blue]Building image of airflow from {__AIRFLOW_SOURCES_ROOT}[/]\n")
- raise ClickException("\nPlease implement building the CI image\n")
+ build_image(
+ verbose,
+ additional_extras=additional_extras,
+ python_version=python,
+ additional_dev_apt_deps=additional_dev_apt_deps,
+ additional_runtime_apt_deps=additional_runtime_apt_deps,
+ additional_python_deps=additional_python_deps,
+ additional_runtime_apt_command=additional_runtime_apt_command,
+ additional_dev_apt_command=additional_dev_apt_command,
+ additional_dev_apt_env=additional_dev_apt_env,
+ additional_runtime_apt_env=additional_runtime_apt_env,
+ dev_apt_command=dev_apt_command,
+ dev_apt_deps=dev_apt_deps,
+ runtime_apt_command=runtime_apt_command,
+ runtime_apt_deps=runtime_apt_deps,
+ github_repository=github_repository,
+ docker_cache=build_cache,
+ upgrade_to_newer_dependencies=upgrade_to_newer_dependencies,
+ )
+
+
+@option_verbose
+@main.command(name='build-prod-image')
+def build_prod_image(verbose: bool):
+ """Builds docker Production image without entering the container."""
+ if verbose:
+ console.print("\n[blue]Building image[/]\n")
+ raise ClickException("\nPlease implement building the Production image\n")
if __name__ == '__main__':
+ from airflow_breeze.cache import BUILD_CACHE_DIR
+
+ BUILD_CACHE_DIR.mkdir(parents=True, exist_ok=True)
main()
diff --git a/dev/breeze/src/airflow_breeze/cache.py b/dev/breeze/src/airflow_breeze/cache.py
new file mode 100644
index 0000000..2930020
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/cache.py
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+from pathlib import Path
+from typing import Any, List, Optional, Tuple
+
+from airflow_breeze import global_constants
+from airflow_breeze.breeze import get_airflow_sources_root
+from airflow_breeze.console import console
+
+BUILD_CACHE_DIR = Path(get_airflow_sources_root(), '.build')
+
+
+def check_if_cache_exists(param_name: str) -> bool:
+ return (Path(BUILD_CACHE_DIR) / f".{param_name}").exists()
+
+
+def read_from_cache_file(param_name: str) -> Optional[str]:
+ cache_exists = check_if_cache_exists(param_name)
+ if cache_exists:
+ return (Path(BUILD_CACHE_DIR) / f".{param_name}").read_text().strip()
+ else:
+ return None
+
+
+def write_to_cache_file(param_name: str, param_value: str, check_allowed_values: bool = True) -> None:
+ allowed = False
+ if check_allowed_values:
+ allowed, allowed_values = check_if_values_allowed(param_name, param_value)
+ if allowed or not check_allowed_values:
+ Path(BUILD_CACHE_DIR, f".{param_name}").write_text(param_value)
+ else:
+ console.print(f'[cyan]You have sent the {param_value} for {param_name}')
+ console.print(f'[cyan]Allowed value for the {param_name} are {allowed_values}')
+ console.print('[cyan]Provide one of the supported params. Write to cache dir failed')
+ sys.exit()
+
+
+def check_cache_and_write_if_not_cached(
+ param_name: str, default_param_value: str
+) -> Tuple[bool, Optional[str]]:
+ is_cached = False
+ allowed = False
+ cached_value = read_from_cache_file(param_name)
+ if cached_value is None:
+ write_to_cache_file(param_name, default_param_value)
+ cached_value = default_param_value
+ else:
+ allowed, allowed_values = check_if_values_allowed(param_name, cached_value)
+ if allowed:
+ is_cached = True
+ else:
+ write_to_cache_file(param_name, default_param_value)
+ cached_value = default_param_value
+ return is_cached, cached_value
+
+
+def check_if_values_allowed(param_name: str, param_value: str) -> Tuple[bool, List[Any]]:
+ allowed = False
+ allowed_values: List[Any] = []
+ allowed_values = getattr(global_constants, f'ALLOWED_{param_name.upper()}')
+ if param_value in allowed_values:
+ allowed = True
+ return allowed, allowed_values
diff --git a/dev/breeze/src/airflow_breeze/ci/build_image.py b/dev/breeze/src/airflow_breeze/ci/build_image.py
new file mode 100644
index 0000000..ed6e3dd
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/ci/build_image.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# from airflow_breeze.utils import run_command
+from pathlib import Path
+from typing import List
+
+from airflow_breeze.breeze import get_airflow_sources_root
+from airflow_breeze.cache import check_cache_and_write_if_not_cached
+from airflow_breeze.ci.build_params import BuildParams
+from airflow_breeze.console import console
+from airflow_breeze.utils import filter_out_none, run_command
+
+PARAMS_CI_IMAGE = [
+ "python_base_image",
+ "airflow_version",
+ "airflow_branch",
+ "airflow_extras",
+ "airflow_pre_cached_pip_packages",
+ "additional_airflow_extras",
+ "additional_python_deps",
+ "additional_dev_apt_command",
+ "additional_dev_apt_deps",
+ "additional_dev_apt_env",
+ "additional_runtime_apt_command",
+ "additional_runtime_apt_deps",
+ "additional_runtime_apt_env",
+ "upgrade_to_newer_dependencies",
+ "constraints_github_repository",
+ "airflow_constraints_reference",
+ "airflow_constraints",
+ "airflow_image_repository",
+ "airflow_image_date_created",
+ "build_id",
+ "commit_sha",
+]
+
+PARAMS_TO_VERIFY_CI_IMAGE = [
+ "dev_apt_command",
+ "dev_apt_deps",
+ "runtime_apt_command",
+ "runtime_apt_deps",
+]
+
+
+def construct_arguments_docker_command(ci_image: BuildParams) -> List[str]:
+ args_command = []
+ for param in PARAMS_CI_IMAGE:
+ args_command.append("--build-arg")
+ args_command.append(param.upper() + "=" + str(getattr(ci_image, param)))
+ for verify_param in PARAMS_TO_VERIFY_CI_IMAGE:
+ param_value = str(getattr(ci_image, verify_param))
+ if len(param_value) > 0:
+ args_command.append("--build-arg")
+ args_command.append(verify_param.upper() + "=" + param_value)
+ docker_cache = ci_image.docker_cache_ci_directive
+ if len(docker_cache) > 0:
+ args_command.extend(ci_image.docker_cache_ci_directive)
+ return args_command
+
+
+def construct_docker_command(ci_image: BuildParams) -> List[str]:
+ arguments = construct_arguments_docker_command(ci_image)
+ final_command = []
+ final_command.extend(["docker", "build"])
+ final_command.extend(arguments)
+ final_command.extend(["-t", ci_image.airflow_ci_image_name, "--target", "main", "."])
+ final_command.extend(["-f", str(Path(get_airflow_sources_root(), 'Dockerfile.ci').resolve())])
+ return final_command
+
+
+def build_image(verbose, **kwargs):
+ ci_image_params = BuildParams(filter_out_none(**kwargs))
+ is_cached, value = check_cache_and_write_if_not_cached(
+ "PYTHON_MAJOR_MINOR_VERSION", ci_image_params.python_version
+ )
+ if is_cached:
+ ci_image_params.python_version = value
+ cmd = construct_docker_command(ci_image_params)
+ output = run_command(cmd, verbose=verbose, text=True)
+ console.print(f"[blue]{output}")
diff --git a/dev/breeze/src/airflow_breeze/ci/build_params.py b/dev/breeze/src/airflow_breeze/ci/build_params.py
new file mode 100644
index 0000000..7fc00b0
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/ci/build_params.py
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
+from airflow_breeze.breeze import get_airflow_sources_root
+from airflow_breeze.utils import run_command
+
+
+@dataclass
+class BuildParams:
+ # To construct ci_image_name
+ python_version: str = "3.7"
+ airflow_branch: str = AIRFLOW_BRANCH
+ build_id: int = 0
+ # To construct docker cache ci directive
+ docker_cache: str = "pulled"
+ airflow_extras: str = "devel_ci"
+ additional_airflow_extras: str = ""
+ additional_python_deps: str = ""
+ # To construct ci_image_name
+ tag: str = "latest"
+ # To construct airflow_image_repository
+ github_repository: str = "apache/airflow"
+ constraints_github_repository: str = "apache/airflow"
+ # Not sure if defaultConstraintsBranch and airflow_constraints_reference are different
+ default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
+ airflow_constraints: str = "constraints-source-providers"
+ airflow_constraints_reference: Optional[str] = "constraints-main"
+ airflow_constraints_location: Optional[str] = ""
+ airflow_pre_cached_pip_packages: str = "true"
+ dev_apt_command: str = ""
+ dev_apt_deps: str = ""
+ additional_dev_apt_command: str = ""
+ additional_dev_apt_deps: str = ""
+ additional_dev_apt_env: str = ""
+ runtime_apt_command: str = ""
+ runtime_apt_deps: str = ""
+ additional_runtime_apt_command: str = ""
+ additional_runtime_apt_deps: str = ""
+ additional_runtime_apt_env: str = ""
+ upgrade_to_newer_dependencies: str = "true"
+
+ @property
+ def airflow_image_name(self):
+ image = f'ghcr.io/{self.github_repository.lower()}'
+ return image
+
+ @property
+ def airflow_ci_image_name(self):
+ """Construct CI image link"""
+ image = f'{self.airflow_image_name}/{self.airflow_branch}/ci/python{self.python_version}'
+ return image if not self.tag else image + f":{self.tag}"
+
+ @property
+ def airflow_image_repository(self):
+ return f'https://github.com/{self.github_repository}'
+
+ @property
+ def python_base_image(self):
+ """Construct Python Base Image"""
+ # ghcr.io/apache/airflow/main/python:3.8-slim-buster
+ return f'{self.airflow_image_name}/{self.airflow_branch}/python:{self.python_version}-slim-buster'
+
+ @property
+ def airflow_ci_local_manifest_image(self):
+ """Construct CI Local Manifest Image"""
+ return f'local-airflow-ci-manifest/{self.airflow_branch}/python{self.python_version}'
+
+ @property
+ def airflow_ci_remote_manifest_image(self):
+ """Construct CI Remote Manifest Image"""
+ return f'{self.airflow_ci_image_name}/{self.airflow_branch}/ci-manifest//python:{self.python_version}'
+
+ @property
+ def airflow_image_date_created(self):
+ # 2021-12-18T15:19:25Z '%Y-%m-%dT%H:%M:%SZ'
+ # Set date in above format and return
+ now = datetime.now()
+ return now.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+ @property
+ def commit_sha(self):
+ output = run_command(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True)
+ return output.stdout.strip()
+
+ @property
+ def docker_cache_ci_directive(self) -> List:
+ docker_cache_ci_directive = []
+ if self.docker_cache == "pulled":
+ docker_cache_ci_directive.append("--cache-from")
+ docker_cache_ci_directive.append(self.airflow_ci_image_name)
+ elif self.docker_cache == "disabled":
+ docker_cache_ci_directive.append("--no-cache")
+ else:
+ pass
+ return docker_cache_ci_directive
+
+ @property
+ def airflow_version(self):
+ airflow_setup_file = Path(get_airflow_sources_root()) / 'setup.py'
+ with open(airflow_setup_file) as setup_file:
+ for line in setup_file.readlines():
+ if "version =" in line:
+ return line.split()[2][1:-1]
diff --git a/dev/breeze/src/airflow_breeze/console.py b/dev/breeze/src/airflow_breeze/console.py
new file mode 100644
index 0000000..175fb70
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/console.py
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from rich.console import Console
+from rich.theme import Theme
+
+custom_theme = Theme({"info": "blue", "warning": "magenta", "error": "red"})
+console = Console(force_terminal=True, color_system="standard", width=180, theme=custom_theme)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
new file mode 100644
index 0000000..77df241
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -0,0 +1,179 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+AIRFLOW_SOURCES = ""
+DEFAULT_PYTHON_MAJOR_MINOR_VERSION = 3.7
+BUILD_CACHE_DIR = "${AIRFLOW_SOURCES}/.build"
+
+FORCE_PULL_IMAGES = False
+CHECK_IF_BASE_PYTHON_IMAGE_UPDATED = False
+FORCE_BUILD_IMAGES = False
+LAST_FORCE_ANSWER_FILE = f"{BUILD_CACHE_DIR}/last_force_answer.sh"
+FORCE_ANSWER_TO_QUESTION = ""
+SKIP_CHECK_REMOTE_IMAGE = False
+PUSH_PYTHON_BASE_IMAGE = False
+
+ALLOWED_PYTHON_MAJOR_MINOR_VERSION = ['3.6', '3.7', '3.8', '3.9']
+ALLOWED_BACKENDS = ['sqlite', 'mysql', 'postgres', 'mssql']
+ALLOWED_STATIC_CHECKS = [
+ "all",
+ "airflow-config-yaml",
+ "airflow-providers-available",
+ "airflow-provider-yaml-files-ok",
+ "base-operator",
+ "bats-tests",
+ "bats-in-container-tests",
+ "black",
+ "blacken-docs",
+ "boring-cyborg",
+ "build",
+ "build-providers-dependencies",
+ "chart-schema-lint",
+ "capitalized-breeze",
+ "changelog-duplicates",
+ "check-apache-license",
+ "check-builtin-literals",
+ "check-executables-have-shebangs",
+ "check-extras-order",
+ "check-hooks-apply",
+ "check-integrations",
+ "check-merge-conflict",
+ "check-xml",
+ "daysago-import-check",
+ "debug-statements",
+ "detect-private-key",
+ "doctoc",
+ "dont-use-safe-filter",
+ "end-of-file-fixer",
+ "fix-encoding-pragma",
+ "flake8",
+ "flynt",
+ "codespell",
+ "forbid-tabs",
+ "helm-lint",
+ "identity",
+ "incorrect-use-of-LoggingMixin",
+ "insert-license",
+ "isort",
+ "json-schema",
+ "language-matters",
+ "lint-dockerfile",
+ "lint-openapi",
+ "markdownlint",
+ "mermaid",
+ "mixed-line-ending",
+ "mypy",
+ "mypy-helm",
+ "no-providers-in-core-examples",
+ "no-relative-imports",
+ "pre-commit-descriptions",
+ "pre-commit-hook-names",
+ "pretty-format-json",
+ "provide-create-sessions",
+ "providers-changelogs",
+ "providers-init-file",
+ "providers-subpackages-init-file",
+ "provider-yamls",
+ "pydevd",
+ "pydocstyle",
+ "python-no-log-warn",
+ "pyupgrade",
+ "restrict-start_date",
+ "rst-backticks",
+ "setup-order",
+ "setup-extra-packages",
+ "shellcheck",
+ "sort-in-the-wild",
+ "sort-spelling-wordlist",
+ "stylelint",
+ "trailing-whitespace",
+ "ui-lint",
+ "update-breeze-file",
+ "update-extras",
+ "update-local-yml-file",
+ "update-setup-cfg-file",
+ "update-versions",
+ "verify-db-migrations-documented",
+ "version-sync",
+ "www-lint",
+ "yamllint",
+ "yesqa",
+]
+ALLOWED_INTEGRATIONS = [
+ 'cassandra',
+ 'kerberos',
+ 'mongo',
+ 'openldap',
+ 'pinot',
+ 'rabbitmq',
+ 'redis',
+ 'statsd',
+ 'trino',
+ 'all',
+]
+ALLOWED_KUBERNETES_MODES = ['image']
+ALLOWED_KUBERNETES_VERSIONS = ['v1.21.1', 'v1.20.2']
+ALLOWED_KIND_VERSIONS = ['v0.11.1']
+ALLOWED_HELM_VERSIONS = ['v3.6.3']
+ALLOWED_EXECUTORS = ['KubernetesExecutor', 'CeleryExecutor', 'LocalExecutor', 'CeleryKubernetesExecutor']
+ALLOWED_KIND_OPERATIONS = ['start', 'stop', 'restart', 'status', 'deploy', 'test', 'shell', 'k9s']
+ALLOWED_INSTALL_AIRFLOW_VERSIONS = ['2.0.2', '2.0.1', '2.0.0', 'wheel', 'sdist']
+ALLOWED_GENERATE_CONSTRAINTS_MODES = ['source-providers', 'pypi-providers', 'no-providers']
+ALLOWED_POSTGRES_VERSIONS = ['10', '11', '12', '13']
+ALLOWED_MYSQL_VERSIONS = ['5.7', '8']
+ALLOWED_MSSQL_VERSIONS = ['2017-latest', '2019-latest']
+ALLOWED_TEST_TYPES = [
+ 'All',
+ 'Always',
+ 'Core',
+ 'Providers',
+ 'API',
+ 'CLI',
+ 'Integration',
+ 'Other',
+ 'WWW',
+ 'Postgres',
+ 'MySQL',
+ 'Helm',
+ 'Quarantined',
+]
+ALLOWED_PACKAGE_FORMATS = ['both', 'sdist', 'wheel']
+ALLOWED_USE_AIRFLOW_VERSION = ['.', 'apache-airflow']
+
+PARAM_NAME_DESCRIPTION = {
+ "BACKEND": "backend",
+ "MYSQL_VERSION": "Mysql version",
+ "KUBERNETES_MODE": "Kubernetes mode",
+ "KUBERNETES_VERSION": "Kubernetes version",
+ "KIND_VERSION": "KinD version",
+ "HELM_VERSION": "Helm version",
+ "EXECUTOR": "Executors",
+ "POSTGRES_VERSION": "Postgres version",
+ "MSSQL_VERSION": "MSSql version",
+}
+
+PARAM_NAME_FLAG = {
+ "BACKEND": "--backend",
+ "MYSQL_VERSION": "--mysql-version",
+ "KUBERNETES_MODE": "--kubernetes-mode",
+ "KUBERNETES_VERSION": "--kubernetes-version",
+ "KIND_VERSION": "--kind-version",
+ "HELM_VERSION": "--helm-version",
+ "EXECUTOR": "--executor",
+ "POSTGRES_VERSION": "--postgres-version",
+ "MSSQL_VERSION": "--mssql-version",
+}
diff --git a/dev/breeze/src/airflow_breeze/utils.py b/dev/breeze/src/airflow_breeze/utils.py
new file mode 100644
index 0000000..5ccf01a
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/utils.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import hashlib
+import shlex
+import subprocess
+from typing import Dict, List
+
+from airflow_breeze.console import console
+
+
+def run_command(cmd: List[str], *, check: bool = True, verbose: bool = False, **kwargs):
+ if verbose:
+ console.print(f"[blue]$ {' '.join(shlex.quote(c) for c in cmd)}")
+ try:
+ return subprocess.run(cmd, check=check, **kwargs)
+ except subprocess.CalledProcessError as ex:
+ print("========================= OUTPUT start ============================")
+ print(ex.stderr)
+ print(ex.stdout)
+ print("========================= OUTPUT end ============================")
+ raise
+
+
+def generate_md5(filename, file_size: int = 65536):
+ hash_md5 = hashlib.md5()
+ with open(filename, "rb") as f:
+ for file_chunk in iter(lambda: f.read(file_size), b""):
+ hash_md5.update(file_chunk)
+ return hash_md5.hexdigest()
+
+
+def filter_out_none(**kwargs) -> Dict:
+ for key in list(kwargs):
+ if kwargs[key] is None:
+ kwargs.pop(key)
+ return kwargs