You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/05/07 09:36:00 UTC
[airflow] branch v2-3-test updated (37486c91aa -> 0ac5701c9d)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 37486c91aa Improve handling of entry and exit to common Breeze commands (#23395)
new 2a835cd134 Mark image as refreshed when pulled on CI (#23410)
new dcce81713b Improve verbose output of Breeze (#23446)
new ffe5526c6a Unify approach for user questions asked in Breeze (#23335)
new 6a5c04ce25 Move non-opencontainer labeling of the image to breeze from Dockerfile (#23379)
new 47daedc9a0 Move tests command in new breeze (#23445)
new c0da48e871 Seperate provider verification as standalone breeze command (#23454)
new 0ac5701c9d Fix accidental including of providers in airflow package (#23552)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build-images.yml | 1 -
.github/workflows/ci.yml | 45 +-
BREEZE.rst | 49 +-
CONTRIBUTORS_QUICK_START.rst | 102 +--
Dockerfile | 5 +-
Dockerfile.ci | 383 +--------
TESTING.rst | 6 +-
airflow/www/ask_for_recompile_assets_if_needed.sh | 8 +-
breeze | 14 +-
breeze-legacy | 1 -
dev/PROVIDER_PACKAGE_DETAILS.md | 16 +
dev/README_RELEASE_PROVIDER_PACKAGES.md | 51 +-
.../doc/adr/0012-asking-user-for-confirmation.md | 99 +++
.../build_image/ci/build_ci_image.py | 122 +--
.../build_image/ci/build_ci_params.py | 3 +-
.../build_image/prod/build_prod_image.py | 87 +-
.../build_image/prod/build_prod_params.py | 38 +-
.../src/airflow_breeze/commands/ci_image_tools.py | 23 +-
.../src/airflow_breeze/commands/common_options.py | 64 +-
.../commands/configuration_and_maintenance.py | 30 +-
.../airflow_breeze/commands/custom_param_types.py | 35 +-
.../airflow_breeze/commands/developer_commands.py | 80 +-
dev/breeze/src/airflow_breeze/commands/main.py | 6 +
.../commands/production_image_tools.py | 21 +-
.../airflow_breeze/commands/release_management.py | 391 ++++++---
dev/breeze/src/airflow_breeze/commands/testing.py | 76 +-
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 17 +-
.../src/airflow_breeze/shell/shell_params.py | 21 +-
dev/breeze/src/airflow_breeze/utils/ci_group.py | 12 +-
dev/breeze/src/airflow_breeze/utils/confirm.py | 42 +-
dev/breeze/src/airflow_breeze/utils/constraints.py | 79 --
.../airflow_breeze/utils/docker_command_utils.py | 81 +-
.../utils/find_newer_dependencies.py | 2 +-
.../src/airflow_breeze/utils/md5_build_check.py | 11 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 13 +
dev/breeze/src/airflow_breeze/utils/pulll_image.py | 10 +-
.../utils/rebuild_image_if_needed.py | 2 +-
dev/breeze/src/airflow_breeze/utils/reinstall.py | 11 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 48 +-
dev/breeze/tests/test_commands.py | 20 +-
dev/import_all_classes.py | 141 ----
dev/provider_packages/README.md | 356 --------
dev/provider_packages/prepare_provider_packages.py | 766 ++---------------
.../howto/create-update-providers.rst | 20 -
images/breeze/output-build-image.svg | 54 +-
images/breeze/output-build-prod-image.svg | 52 +-
images/breeze/output-commands.svg | 81 +-
images/breeze/output-find-newer-dependencies.svg | 58 +-
images/breeze/output-generate-constraints.svg | 53 +-
images/breeze/output-prepare-airflow-package.svg | 53 +-
.../output-prepare-provider-documentation.svg | 52 +-
images/breeze/output-prepare-provider-packages.svg | 53 +-
images/breeze/output-pull-image.svg | 55 +-
images/breeze/output-pull-prod-image.svg | 55 +-
images/breeze/output-shell.svg | 91 +-
images/breeze/output-start-airflow.svg | 93 ++-
.../breeze/{output-stop.svg => output-tests.svg} | 70 +-
...age.svg => output-verify-provider-packages.svg} | 94 +--
scripts/ci/docker-compose/_docker.env | 2 +-
scripts/ci/docker-compose/base.yml | 2 +-
scripts/ci/docker-compose/devcontainer.env | 1 -
scripts/ci/libraries/_initialization.sh | 3 -
.../ci_install_and_test_provider_packages.sh | 48 --
.../ci_make_providers_2_1_compliant.sh | 27 -
scripts/docker/entrypoint_ci.sh | 32 +-
scripts/in_container/_in_container_utils.sh | 59 +-
.../run_install_and_test_provider_packages.sh | 344 --------
.../in_container/run_prepare_airflow_packages.sh | 4 +
.../run_prepare_provider_documentation.sh | 29 -
.../in_container/run_prepare_provider_packages.sh | 2 +
scripts/in_container/verify_providers.py | 916 +++++++++++++++++++++
72 files changed, 2642 insertions(+), 3153 deletions(-)
create mode 100644 dev/breeze/doc/adr/0012-asking-user-for-confirmation.md
delete mode 100644 dev/breeze/src/airflow_breeze/utils/constraints.py
delete mode 100755 dev/import_all_classes.py
delete mode 100644 dev/provider_packages/README.md
copy images/breeze/{output-stop.svg => output-tests.svg} (61%)
copy images/breeze/{output-pull-image.svg => output-verify-provider-packages.svg} (54%)
delete mode 100755 scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
delete mode 100755 scripts/ci/provider_packages/ci_make_providers_2_1_compliant.sh
delete mode 100755 scripts/in_container/run_install_and_test_provider_packages.sh
create mode 100755 scripts/in_container/verify_providers.py
[airflow] 02/07: Improve verbose output of Breeze (#23446)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit dcce81713b4a289b2679813e2d3530ab2f85a1c2
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue May 3 15:03:01 2022 +0200
Improve verbose output of Breeze (#23446)
When you add --verbose or --dry-run options to breeze it will
print the commands it is executing (or is supposed to in dry-run
mode). The output contains environment variables as they
often contain crucial information to execute the command (for
example in docker-compose run commands it contains COMPOSE_FILE
variable which is the list of compose files that are used). This
is done in a fashion that you can copy the whole command and
execute it, but it very unfriendly for visual inspection as
all the variables were printed in one line and in semi-random
order and also the variables contained often all system variables
set by the shell before.
This change keeps the proerty of "we can copy&paste the command
and run it" but it improves the visual aspect of it:
1) each env variable is kept in one line
2) first all system variables are printed and then variables that
were specifically added for this command
3) variables in each group are sorted alphabetically which helps
in finding the variable you are looking for when you visually
inspect the output.
(cherry picked from commit 6f146e721c81e9304bf7c0af66fc3d203d902dab)
---
dev/breeze/src/airflow_breeze/utils/run_utils.py | 24 +++++++++++++++++++-----
1 file changed, 19 insertions(+), 5 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 92cf8c8c76..b215cb523b 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -24,7 +24,7 @@ import sys
from distutils.version import StrictVersion
from functools import lru_cache
from pathlib import Path
-from typing import List, Mapping, Optional, Union
+from typing import Dict, List, Mapping, Optional, Union
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
@@ -66,10 +66,7 @@ def run_command(
workdir: str = str(cwd) if cwd else os.getcwd()
if verbose or dry_run:
command_to_print = ' '.join(shlex.quote(c) for c in cmd)
- # if we pass environment variables to execute, then
- env_to_print = ' '.join(f'{key}="{val}"' for (key, val) in env.items()) if env else ''
- if env_to_print:
- env_to_print += ' '
+ env_to_print = get_environments_to_print(env)
get_console().print(f"\n[info]Working directory {workdir} [/]\n")
# Soft wrap allows to copy&paste and run resulting output as it has no hard EOL
get_console().print(f"\n[info]{env_to_print}{command_to_print}[/]\n", soft_wrap=True)
@@ -103,6 +100,23 @@ def run_command(
return ex
+def get_environments_to_print(env: Optional[Mapping[str, str]]):
+ if not env:
+ return "# No environment variables \\\n"
+ system_env: Dict[str, str] = {}
+ my_env: Dict[str, str] = {}
+ for key, val in env.items():
+ if os.environ.get(key) == val:
+ system_env[key] = val
+ else:
+ my_env[key] = val
+ env_to_print = ''.join(f'{key}="{val}" \\\n' for (key, val) in sorted(system_env.items()))
+ env_to_print += r"""\
+"""
+ env_to_print += ''.join(f'{key}="{val}" \\\n' for (key, val) in sorted(my_env.items()))
+ return env_to_print
+
+
def assert_pre_commit_installed(verbose: bool):
"""
Check if pre-commit is installed in the right version.
[airflow] 05/07: Move tests command in new breeze (#23445)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 47daedc9a0a4c6830b084c91e1123fb2a698fe1f
Author: Joppe Vos <44...@users.noreply.github.com>
AuthorDate: Fri May 6 11:03:05 2022 +0200
Move tests command in new breeze (#23445)
(cherry picked from commit 7ba4e35a9d1b65b4c1a318ba4abdf521f98421a2)
---
BREEZE.rst | 9 +-
TESTING.rst | 6 +-
dev/breeze/src/airflow_breeze/commands/testing.py | 76 ++++++++++-
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 11 +-
.../airflow_breeze/utils/docker_command_utils.py | 11 +-
images/breeze/output-commands.svg | 55 ++++----
images/breeze/output-tests.svg | 142 +++++++++++++++++++++
7 files changed, 264 insertions(+), 46 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index c9f44c90c5..862bfeceea 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -471,11 +471,9 @@ Those are commands mostly used by contributors:
* Build documentation with ``breeze build-docs`` command
* Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command
* Run static checks with autocomplete support ``breeze static-checks`` command
-* Run test specified with ``./breeze-legacy tests`` command
+* Run test specified with ``breeze tests`` command
* Build CI docker image with ``breeze build-image`` command
* Cleanup breeze with ``breeze cleanup`` command
-* Run static checks with autocomplete support ``breeze static-checks`` command
-* Run test specified with ``./breeze-legacy tests`` command
Additional management tasks:
@@ -487,6 +485,11 @@ Tests
-----
* Run docker-compose tests with ``breeze docker-compose-tests`` command.
+* Run test specified with ``breeze tests`` command.
+
+.. image:: ./images/breeze/output-tests.svg
+ :width: 100%
+ :alt: Breeze tests
Kubernetes tests
----------------
diff --git a/TESTING.rst b/TESTING.rst
index 215875534a..eb1eb4cb98 100644
--- a/TESTING.rst
+++ b/TESTING.rst
@@ -168,19 +168,19 @@ to breeze.
.. code-block:: bash
- ./breeze-legacy tests tests/providers/http/hooks/test_http.py tests/core/test_core.py --db-reset -- --log-cli-level=DEBUG
+ breeze tests tests/providers/http/hooks/test_http.py tests/core/test_core.py --db-reset --log-cli-level=DEBUG
You can run the whole test suite without adding the test target:
.. code-block:: bash
- ./breeze-legacy tests --db-reset
+ breeze tests --db-reset
You can also specify individual tests or a group of tests:
.. code-block:: bash
- ./breeze-legacy tests --db-reset tests/core/test_core.py::TestCore
+ breeze tests --db-reset tests/core/test_core.py::TestCore
Running Tests of a specified type from the Host
diff --git a/dev/breeze/src/airflow_breeze/commands/testing.py b/dev/breeze/src/airflow_breeze/commands/testing.py
index 456544b57f..e4a1ec1eb5 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+import os
import sys
from typing import Tuple
@@ -22,22 +23,28 @@ import click
from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
from airflow_breeze.commands.common_options import (
+ option_db_reset,
option_dry_run,
option_github_repository,
option_image_name,
option_image_tag,
+ option_integration,
option_python,
option_verbose,
)
+from airflow_breeze.commands.custom_param_types import BetterChoice
from airflow_breeze.commands.main import main
+from airflow_breeze.global_constants import ALLOWED_TEST_TYPES
+from airflow_breeze.shell.enter_shell import check_docker_is_running, check_docker_resources
+from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.docker_command_utils import construct_env_variables_docker_compose_command
from airflow_breeze.utils.run_tests import run_docker_compose_tests
+from airflow_breeze.utils.run_utils import run_command
TESTING_COMMANDS = {
"name": "Testing",
- "commands": [
- "docker-compose-tests",
- ],
+ "commands": ["docker-compose-tests", "tests"],
}
TESTING_PARAMETERS = {
@@ -51,6 +58,16 @@ TESTING_PARAMETERS = {
],
}
],
+ "breeze tests": [
+ {
+ "name": "Basic flag for tests command",
+ "options": [
+ "--integration",
+ "--test-type",
+ "--db-reset",
+ ],
+ }
+ ],
}
@@ -91,3 +108,56 @@ def docker_compose_tests(
extra_pytest_args=extra_pytest_args,
)
sys.exit(return_code)
+
+
+@main.command(
+ name='tests',
+ help="Run the specified unit test targets. Multiple targets may be specified separated by spaces.",
+ context_settings=dict(
+ ignore_unknown_options=True,
+ allow_extra_args=True,
+ ),
+)
+@option_dry_run
+@option_verbose
+@option_integration
+@click.argument('extra_pytest_args', nargs=-1, type=click.UNPROCESSED)
+@click.option(
+ "-tt",
+ "--test-type",
+ help="Type of test to run.",
+ default="All",
+ type=BetterChoice(ALLOWED_TEST_TYPES),
+)
+@option_db_reset
+def tests(
+ dry_run: bool,
+ verbose: bool,
+ integration: Tuple,
+ extra_pytest_args: Tuple,
+ test_type: str,
+ db_reset: bool,
+):
+ os.environ["RUN_TESTS"] = "true"
+ if test_type:
+ os.environ["TEST_TYPE"] = test_type
+ if integration:
+ if "trino" in integration:
+ integration = integration + ("kerberos",)
+ os.environ["LIST_OF_INTEGRATION_TESTS_TO_RUN"] = ' '.join(list(integration))
+ if db_reset:
+ os.environ["DB_RESET"] = "true"
+
+ exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
+ env_variables = construct_env_variables_docker_compose_command(exec_shell_params)
+ check_docker_is_running(verbose)
+ check_docker_resources(exec_shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run)
+
+ cmd = ['docker-compose', 'run', '--service-ports', '--rm', 'airflow']
+ cmd.extend(list(extra_pytest_args))
+ run_command(
+ cmd,
+ verbose=verbose,
+ dry_run=dry_run,
+ env=env_variables,
+ )
diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
index 951d36220a..e73153b60e 100644
--- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
+++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
@@ -23,8 +23,10 @@ from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.cache import read_from_cache_file
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
+ check_docker_compose_version,
check_docker_is_running,
check_docker_resources,
+ check_docker_version,
construct_env_variables_docker_compose_command,
)
from airflow_breeze.utils.rebuild_image_if_needed import rebuild_ci_image_if_needed
@@ -46,12 +48,9 @@ def enter_shell(**kwargs) -> Union[subprocess.CompletedProcess, subprocess.Calle
"""
verbose = kwargs['verbose']
dry_run = kwargs['dry_run']
- if not check_docker_is_running(verbose):
- get_console().print(
- '[error]Docker is not running.[/]\n'
- '[warning]Please make sure Docker is installed and running.[/]'
- )
- sys.exit(1)
+ check_docker_is_running(verbose)
+ check_docker_version(verbose)
+ check_docker_compose_version(verbose)
if read_from_cache_file('suppress_asciiart') is None:
get_console().print(ASCIIART, style=ASCIIART_STYLE)
if read_from_cache_file('suppress_cheatsheet') is None:
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index c7c54630fa..9d30a86345 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -18,6 +18,7 @@
import os
import re
import subprocess
+import sys
from random import randint
from typing import Dict, List, Tuple, Union
@@ -174,11 +175,10 @@ def compare_version(current_version: str, min_version: str) -> bool:
return version.parse(current_version) >= version.parse(min_version)
-def check_docker_is_running(verbose: bool) -> bool:
+def check_docker_is_running(verbose: bool):
"""
Checks if docker is running. Suppressed Dockers stdout and stderr output.
:param verbose: print commands when running
- :return: False if docker is not running.
"""
response = run_command(
["docker", "info"],
@@ -190,8 +190,11 @@ def check_docker_is_running(verbose: bool) -> bool:
check=False,
)
if response.returncode != 0:
- return False
- return True
+ get_console().print(
+ '[error]Docker is not running.[/]\n'
+ '[warning]Please make sure Docker is installed and running.[/]'
+ )
+ sys.exit(1)
def check_docker_version(verbose: bool):
diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg
index 66b560b308..59d7de7f66 100644
--- a/images/breeze/output-commands.svg
+++ b/images/breeze/output-commands.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1858" viewBox="0 0 1720.0 1858"
+<svg width="1720.0" height="1880" viewBox="0 0 1720.0 1880"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-3448604350-terminal-wrapper span {
+ .rich-svg-1735457778-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-3448604350-terminal-wrapper a {
+ .rich-svg-1735457778-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-3448604350-terminal-body .blink {
- animation: rich-svg-3448604350-blinker 1s infinite;
+ .rich-svg-1735457778-terminal-body .blink {
+ animation: rich-svg-1735457778-blinker 1s infinite;
}
- @keyframes rich-svg-3448604350-blinker {
+ @keyframes rich-svg-1735457778-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-3448604350-terminal-wrapper {
+ .rich-svg-1735457778-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-3448604350-terminal {
+ .rich-svg-1735457778-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-3448604350-terminal:after {
+ .rich-svg-1735457778-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-3448604350-terminal-header {
+ .rich-svg-1735457778-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-3448604350-terminal-title-tab {
+ .rich-svg-1735457778-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-3448604350-terminal-traffic-lights {
+ .rich-svg-1735457778-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-3448604350-terminal-body {
+ .rich-svg-1735457778-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-3448604350-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-3448604350-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3448604350-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3448604350-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3448604350-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3448604350-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3448604350-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-1735457778-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1735457778-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1735457778-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1735457778-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1735457778-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1735457778-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1735457778-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-3448604350-terminal-wrapper">
- <div class="rich-svg-3448604350-terminal">
- <div class="rich-svg-3448604350-terminal-header">
- <svg class="rich-svg-3448604350-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-1735457778-terminal-wrapper">
+ <div class="rich-svg-1735457778-terminal">
+ <div class="rich-svg-1735457778-terminal-header">
+ <svg class="rich-svg-1735457778-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-3448604350-terminal-title-tab">Breeze commands</div>
+ <div class="rich-svg-1735457778-terminal-title-tab">Breeze commands</div>
</div>
- <div class="rich-svg-3448604350-terminal-body">
+ <div class="rich-svg-1735457778-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze [OPTIONS] COMMAND [ARGS]... </span></div>
<div><span class="r2"> </span></div>
@@ -153,7 +153,8 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> static-checks </span><span class="r1"> </span><span class="r1">Run static checks. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> docker-compose-tests </span><span class="r1"> </span><span class="r1">Run docker-compose tests.</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> docker-compose-tests </span><span class="r1"> </span><span class="r1">Run docker-compose tests. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> tests </span><span class="r1"> </span><span class="r1">Run the specified unit test targets. Multiple targets may be specified separated by spaces.</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Configuration & maintenance ────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> cleanup </span><span class="r1"> </span><span class="r1">Cleans the cache of parameters, docker cache and optionally - currently downloaded images.</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-tests.svg b/images/breeze/output-tests.svg
new file mode 100644
index 0000000000..ca27be92e1
--- /dev/null
+++ b/images/breeze/output-tests.svg
@@ -0,0 +1,142 @@
+<svg width="1720.0" height="758" viewBox="0 0 1720.0 758"
+ xmlns="http://www.w3.org/2000/svg">
+ <style>
+ @font-face {
+ font-family: "Rich Fira Code";
+ src: local("FiraCode-Regular"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff") format("woff");
+ font-style: normal;
+ font-weight: 400;
+ }
+ @font-face {
+ font-family: "Rich Fira Code";
+ src: local("FiraCode-Bold"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff") format("woff");
+ font-style: bold;
+ font-weight: 700;
+ }
+ .rich-svg-356577593-terminal-wrapper span {
+ display: inline-block;
+ white-space: pre;
+ vertical-align: top;
+ font-size: 18px;
+ font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+ }
+ .rich-svg-356577593-terminal-wrapper a {
+ text-decoration: none;
+ color: inherit;
+ }
+ .rich-svg-356577593-terminal-body .blink {
+ animation: rich-svg-356577593-blinker 1s infinite;
+ }
+ @keyframes rich-svg-356577593-blinker {
+ from { opacity: 1.0; }
+ 50% { opacity: 0.3; }
+ to { opacity: 1.0; }
+ }
+ .rich-svg-356577593-terminal-wrapper {
+ padding: 140px;
+ padding-top: 100px;
+ }
+ .rich-svg-356577593-terminal {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ background-color: #0c0c0c;
+ border-radius: 14px;
+ box-shadow: 0 0 0 1px #484848;
+ }
+ .rich-svg-356577593-terminal:after {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ content: '';
+ border-radius: 14px;
+ background: rgb(71,77,102);
+ background: linear-gradient(90deg, #804D69 0%, #4E4B89 100%);
+ transform: rotate(-4.5deg);
+ z-index: -1;
+ }
+ .rich-svg-356577593-terminal-header {
+ position: relative;
+ width: 100%;
+ background-color: #2e2e2e;
+ margin-bottom: 12px;
+ font-weight: bold;
+ border-radius: 14px 14px 0 0;
+ color: #f2f2f2;
+ font-size: 18px;
+ box-shadow: inset 0px -1px 0px 0px #4e4e4e,
+ inset 0px -4px 8px 0px #1a1a1a;
+ }
+ .rich-svg-356577593-terminal-title-tab {
+ display: inline-block;
+ margin-top: 14px;
+ margin-left: 124px;
+ font-family: sans-serif;
+ padding: 14px 28px;
+ border-radius: 6px 6px 0 0;
+ background-color: #0c0c0c;
+ box-shadow: inset 0px 1px 0px 0px #4e4e4e,
+ 0px -4px 4px 0px #1e1e1e,
+ inset 1px 0px 0px 0px #4e4e4e,
+ inset -1px 0px 0px 0px #4e4e4e;
+ }
+ .rich-svg-356577593-terminal-traffic-lights {
+ position: absolute;
+ top: 24px;
+ left: 20px;
+ }
+ .rich-svg-356577593-terminal-body {
+ line-height: 22px;
+ padding: 14px;
+ }
+ .rich-svg-356577593-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-356577593-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-356577593-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-356577593-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-356577593-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-356577593-terminal-body .r6 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+.rich-svg-356577593-terminal-body .r7 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+ </style>
+ <foreignObject x="0" y="0" width="100%" height="100%">
+ <body xmlns="http://www.w3.org/1999/xhtml">
+ <div class="rich-svg-356577593-terminal-wrapper">
+ <div class="rich-svg-356577593-terminal">
+ <div class="rich-svg-356577593-terminal-header">
+ <svg class="rich-svg-356577593-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="14" cy="8" r="8" fill="#ff6159"/>
+ <circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
+ <circle cx="62" cy="8" r="8" fill="#28c941"/>
+ </svg>
+ <div class="rich-svg-356577593-terminal-title-tab">Command: tests</div>
+ </div>
+ <div class="rich-svg-356577593-terminal-body">
+ <div><span class="r2"> </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze tests [OPTIONS] [EXTRA_PYTEST_ARGS]... </span></div>
+<div><span class="r2"> </span></div>
+<div><span class="r1"> </span><span class="r1">Run the specified unit test targets. Multiple targets may be specified separated by spaces. </span><span class="r1"> </span></div>
+<div><span class="r1"> </span></div>
+<div><span class="r4">╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r6">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--test-type</span><span class="r1"> </span><span class="r7">-tt</span><span class="r1"> </span><span class="r1">Type of test to run. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r6">(All | Always | Core | Providers | API | CLI | Integration | Other | WWW | Postgres | MySQL | </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r6">Helm | Quarantined) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--db-reset</span><span class="r1"> </span><span class="r7">-d</span><span class="r1"> </span><span class="r1">Reset DB when entering the container. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r7">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r7">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r7">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r1"></span><span class="r1"> </span></div>
+ </div>
+ </div>
+ </div>
+ </body>
+ </foreignObject>
+</svg>
[airflow] 07/07: Fix accidental including of providers in airflow package (#23552)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0ac5701c9de38b783233e22fce7b94f42609d66c
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat May 7 08:26:04 2022 +0200
Fix accidental including of providers in airflow package (#23552)
The change #23454 accidentally remove INSTALL_PROVIDERS_FROM_SOURCES
setting to "false" which resulted in airflow package containing all
providers. This has been caught by our tests (but it was only
visible after merging)
This PR brings the variable back.
(cherry picked from commit 29ec8d0a06f0f548083bc4f31d0bfe61286a3342)
---
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 52539cefd3..069c0a3c9d 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -506,6 +506,7 @@ DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = {
"ENABLED_INTEGRATIONS": "enabled_integrations",
"GITHUB_ACTIONS": "github_actions",
"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
+ "INSTALL_PROVIDERS_FROM_SOURCES": "install_providers_from_sources",
"ISSUE_ID": "issue_id",
"LOAD_EXAMPLES": "load_example_dags",
"LOAD_DEFAULT_CONNECTIONS": "load_default_connections",
[airflow] 06/07: Seperate provider verification as standalone breeze command (#23454)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c0da48e871977ac5a087be052ad9089afe42be9e
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat May 7 00:47:00 2022 +0200
Seperate provider verification as standalone breeze command (#23454)
This is another step in simplifying and converting to Python all of
the CI/local development tooling.
This PR separates out verification of providers as a separate
breeze command `verify-provider-packages`. It was previously part of
"prepare_provider_packages.py" but it has been now
extracted to a separate in-container python file and it was
wrapped with breeze's `verify-provider-packages` command.
No longer provider verification is run with "preparing provider docs"
nor "preparing provider packages" - it's a standaline command.
This command is also used in CI now to run the tests:
* all provider packages are built and created on CI together with
airflow version
* the packages are installed inside the CI image and providers are
verified
* the 2.1 version of Airflow is installed together with all 2.1
- compatible providers and provider verification is run there too.
This all is much simpler now - we got rediof some 500 lines of bash
code again in favour of breeze python code.
Fixes: #23430
(cherry picked from commit 3ed07474649b1e202f9b106105fef21f7b2cfddc)
---
.github/workflows/build-images.yml | 1 -
.github/workflows/ci.yml | 45 +-
BREEZE.rst | 40 +-
CONTRIBUTORS_QUICK_START.rst | 102 +--
Dockerfile.ci | 383 +--------
airflow/www/ask_for_recompile_assets_if_needed.sh | 8 +-
breeze-legacy | 1 -
dev/PROVIDER_PACKAGE_DETAILS.md | 16 +
dev/README_RELEASE_PROVIDER_PACKAGES.md | 51 +-
.../build_image/ci/build_ci_image.py | 100 ++-
.../build_image/ci/build_ci_params.py | 3 +-
.../build_image/prod/build_prod_image.py | 87 +-
.../build_image/prod/build_prod_params.py | 3 +-
.../src/airflow_breeze/commands/ci_image_tools.py | 21 +-
.../src/airflow_breeze/commands/common_options.py | 54 +-
.../commands/configuration_and_maintenance.py | 9 +-
.../airflow_breeze/commands/custom_param_types.py | 22 +-
.../airflow_breeze/commands/developer_commands.py | 77 +-
dev/breeze/src/airflow_breeze/commands/main.py | 6 +
.../commands/production_image_tools.py | 19 +-
.../airflow_breeze/commands/release_management.py | 385 ++++++---
dev/breeze/src/airflow_breeze/commands/testing.py | 4 +-
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 6 +-
.../src/airflow_breeze/shell/shell_params.py | 21 +-
dev/breeze/src/airflow_breeze/utils/ci_group.py | 12 +-
dev/breeze/src/airflow_breeze/utils/constraints.py | 79 --
.../airflow_breeze/utils/docker_command_utils.py | 69 +-
.../utils/find_newer_dependencies.py | 2 +-
.../src/airflow_breeze/utils/md5_build_check.py | 8 +-
dev/breeze/src/airflow_breeze/utils/pulll_image.py | 5 +-
.../utils/rebuild_image_if_needed.py | 2 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 22 +-
dev/breeze/tests/test_commands.py | 20 +-
dev/import_all_classes.py | 141 ----
dev/provider_packages/README.md | 356 --------
dev/provider_packages/prepare_provider_packages.py | 766 ++---------------
.../howto/create-update-providers.rst | 20 -
images/breeze/output-build-image.svg | 54 +-
images/breeze/output-build-prod-image.svg | 52 +-
images/breeze/output-commands.svg | 78 +-
images/breeze/output-find-newer-dependencies.svg | 58 +-
images/breeze/output-generate-constraints.svg | 53 +-
images/breeze/output-prepare-airflow-package.svg | 53 +-
.../output-prepare-provider-documentation.svg | 52 +-
images/breeze/output-prepare-provider-packages.svg | 53 +-
images/breeze/output-shell.svg | 91 +-
images/breeze/output-start-airflow.svg | 93 ++-
...nts.svg => output-verify-provider-packages.svg} | 95 +--
scripts/ci/docker-compose/_docker.env | 2 +-
scripts/ci/docker-compose/base.yml | 2 +-
scripts/ci/docker-compose/devcontainer.env | 1 -
scripts/ci/libraries/_initialization.sh | 3 -
.../ci_install_and_test_provider_packages.sh | 48 --
.../ci_make_providers_2_1_compliant.sh | 27 -
scripts/docker/entrypoint_ci.sh | 32 +-
scripts/in_container/_in_container_utils.sh | 59 +-
.../run_install_and_test_provider_packages.sh | 344 --------
.../in_container/run_prepare_airflow_packages.sh | 4 +
.../run_prepare_provider_documentation.sh | 29 -
.../in_container/run_prepare_provider_packages.sh | 2 +
scripts/in_container/verify_providers.py | 916 +++++++++++++++++++++
62 files changed, 2185 insertions(+), 2986 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 950becbf90..5db6a15770 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -41,7 +41,6 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LOGIN_TO_GITHUB_REGISTRY: "true"
IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}"
- WITH_CI_GROUPS: "true"
concurrency:
group: build-${{ github.event.pull_request.number || github.ref }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e85684107a..a5a28dcf3b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,7 +47,6 @@ env:
LOGIN_TO_GITHUB_REGISTRY: "true"
ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}"
IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}"
- WITH_CI_GROUPS: "true"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
@@ -792,17 +791,35 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Prepare airflow package: wheel"
run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
- name: "Install and test provider packages and airflow via wheel files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "wheel"
- PACKAGE_FORMAT: "wheel"
- - name: "Replace non-compliant providers with their 2.1-compliant versions"
- run: ./scripts/ci/provider_packages/ci_make_providers_2_1_compliant.sh
+ run: >
+ breeze verify-provider-packages --use-airflow-version wheel --use-packages-from-dist
+ --package-format wheel
+ - name: "Remove airflow package and replace providers with 2.1-compliant versions"
+ run: |
+ rm -vf dist/apache_airflow-*.whl \
+ dist/apache_airflow_providers_cncf_kubernetes*.whl \
+ dist/apache_airflow_providers_celery*.whl
+ pip download --no-deps --dest dist \
+ apache-airflow-providers-cncf-kubernetes==3.0.0 \
+ apache-airflow-providers-celery==2.1.3
- name: "Install and test provider packages and airflow on Airflow 2.1 files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+ run: >
+ breeze verify-provider-packages --use-airflow-version 2.1.0
+ --use-packages-from-dist --package-format wheel --airflow-constraints-reference constraints-2.1.0
env:
- USE_AIRFLOW_VERSION: "2.1.0"
- PACKAGE_FORMAT: "wheel"
+ # The extras below are all extras that should be installed with Airflow 2.1.0
+ AIRFLOW_EXTRAS: "airbyte,alibaba,amazon,apache.atlas.apache.beam,apache.cassandra,apache.drill,\
+ apache.druid,apache.hdfs,apache.hive,apache.kylin,apache.livy,apache.pig,apache.pinot,\
+ apache.spark,apache.sqoop,apache.webhdfs,arangodb,asana,async,\
+ celery,cgroups,cloudant,cncf.kubernetes,dask,databricks,datadog,dbt.cloud,\
+ deprecated_api,dingding,discord,docker,\
+ elasticsearch,exasol,facebook,ftp,github,github_enterprise,google,google_auth,\
+ grpc,hashicorp,http,imap,influxdb,jdbc,jenkins,jira,kerberos,ldap,\
+ leveldb,microsoft.azure,microsoft.mssql,microsoft.psrp,microsoft.winrm,mongo,mysql,\
+ neo4j,odbc,openfaas,opsgenie,oracle,pagerduty,pandas,papermill,password,plexus,\
+ postgres,presto,qubole,rabbitmq,redis,salesforce,samba,segment,sendgrid,sentry,\
+ sftp,singularity,slack,snowflake,sqlite,ssh,statsd,tableau,telegram,trino,vertica,\
+ virtualenv,yandex,zendesk"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
@@ -851,11 +868,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
path: "./dist/apache-airflow-providers-*.tar.gz"
retention-days: 1
- name: "Install and test provider packages and airflow via sdist files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- USE_AIRFLOW_VERSION: "sdist"
- PACKAGE_FORMAT: "sdist"
- AIRFLOW_EXTRAS: "all"
+ run: >
+ breeze verify-provider-packages --use-airflow-version sdist --use-packages-from-dist
+ --package-format sdist
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
diff --git a/BREEZE.rst b/BREEZE.rst
index 862bfeceea..b7d5e5ef62 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -312,14 +312,23 @@ Those are all available flags of ``setup-autocomplete`` command:
Customize your environment
--------------------------
When you enter the Breeze environment, automatically an environment file is sourced from
-``files/airflow-breeze-config/variables.env``. The ``files`` folder from your local sources is
-automatically mounted to the container under ``/files`` path and you can put there any files you want
-to make available for the Breeze container.
+``files/airflow-breeze-config/variables.env``.
+
+You can also add ``files/airflow-breeze-config/init.sh`` and the script will be sourced always
+when you enter Breeze. For example you can add ``pip install`` commands if you want to install
+custom dependencies - but there are no limits to add your own customizations.
+
+The ``files`` folder from your local sources is automatically mounted to the container under
+``/files`` path and you can put there any files you want to make available for the Breeze container.
+
+You can also copy any .whl or .sdist packages to dist and when you pass ``--use-packages-from-dist`` flag
+as ``wheel`` or ``sdist`` line parameter, breeze will automatically install the packages found there
+when you enter Breeze.
You can also add your local tmux configuration in ``files/airflow-breeze-config/.tmux.conf`` and
these configurations will be available for your tmux environment.
-there is a symlink between ``files/airflow-breeze-config/.tmux.conf`` and ``~/.tmux.conf`` in the container,
+There is a symlink between ``files/airflow-breeze-config/.tmux.conf`` and ``~/.tmux.conf`` in the container,
so you can change it at any place, and run
.. code-block:: bash
@@ -540,6 +549,8 @@ do not need or have no access to run). Those are usually connected with releasin
``breeze build-prod image --prepare-build-cache``(needs buildx plugin and write access to registry ghcr.io)
* Generate constraints with ``breeze generate-constraints`` (needed when conflicting changes are merged)
* Prepare airflow packages: ``breeze prepare-airflow-package`` (when releasing Airflow)
+* Verify providers: ``breeze verify-provider-packages`` (when releasing provider packages) - including importing
+ the providers in an earlier airflow version.
* Prepare provider documentation ``breeze prepare-provider-documentation`` and prepare provider packages
``breeze prepare-provider-packages`` (when releasing provider packages)
* Finding the updated dependencies since the last successful build when we have conflict with
@@ -744,6 +755,27 @@ Those are all available flags of ``verify-image`` command:
:width: 100%
:alt: Breeze verify-image
+Verifying providers
+-------------------
+
+Breeze can also be used to verify if provider classes are importable and if they are following the
+right naming conventions. This happens automatically on CI but you can also run it manually.
+
+.. code-block:: bash
+
+ breeze verify-provider-packages
+
+You can also run the verification with an earlier airflow version to check for compatibility.
+
+.. code-block:: bash
+
+ breeze verify-provider-packages --use-airflow-version 2.1.0
+
+All the command parameters are here:
+
+.. image:: ./images/breeze/output-verify-provider-packages.svg
+ :width: 100%
+ :alt: Breeze verify-provider-packages
Preparing packages
------------------
diff --git a/CONTRIBUTORS_QUICK_START.rst b/CONTRIBUTORS_QUICK_START.rst
index 772fce02af..dd5c4bed8a 100644
--- a/CONTRIBUTORS_QUICK_START.rst
+++ b/CONTRIBUTORS_QUICK_START.rst
@@ -645,27 +645,6 @@ All Tests are inside ./tests directory.
$ breeze --backend mysql --mysql-version 5.7 --python 3.8 --db-reset --test-type All tests
-
-- Running specific test in container using shell scripts. Testing in container scripts are located in
- ``./scripts/in_container`` directory.
-
-.. code-block:: bash
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/
- bin/ run_flake8.sh*
- check_environment.sh* run_generate_constraints.sh*
- entrypoint_ci.sh* run_init_script.sh*
- entrypoint_exec.sh* run_install_and_test_provider_packages.sh*
- _in_container_script_init.sh* run_mypy.sh*
- prod/ run_prepare_provider_packages.sh*
- run_ci_tests.sh* run_prepare_provider_documentation.sh*
- run_clear_tmp.sh* run_system_tests.sh*
- run_docs_build.sh* run_tmux_welcome.sh*
- run_extract_tests.sh* stop_tmux_airflow.sh*
- run_fix_ownership.sh* update_quarantined_test_status.py*
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/run_docs_build.sh
-
- Running specific type of test
- Types of tests
@@ -834,32 +813,7 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r
-
-7. Running specific checks in container using shell scripts. Scripts are located in ``./scripts/in_container``
- directory.
-
-.. code-block:: bash
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/
- bin/ run_flake8.sh*
- check_environment.sh* run_generate_constraints.sh*
- entrypoint_ci.sh* run_init_script.sh*
- entrypoint_exec.sh* run_install_and_test_provider_packages.sh*
- _in_container_script_init.sh* run_mypy.sh*
- prod/ run_prepare_provider_packages.sh*
- run_ci_tests.sh* run_prepare_provider_documentation.sh*
- run_clear_tmp.sh* run_system_tests.sh*
- run_docs_build.sh* run_tmux_welcome.sh*
- run_extract_tests.sh* stop_tmux_airflow.sh*
- run_fix_ownership.sh* update_quarantined_test_status.py*
-
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/run_docs_build.sh
-
-
-
-
-8. Enabling Pre-commit check before push. It will run pre-commit automatically before committing and stops the commit
+7. Enabling Pre-commit check before push. It will run pre-commit automatically before committing and stops the commit
.. code-block:: bash
@@ -867,7 +821,7 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r
$ pre-commit install
$ git commit -m "Added xyz"
-9. To disable Pre-commit
+8. To disable Pre-commit
.. code-block:: bash
@@ -1456,26 +1410,6 @@ All Tests are inside ./tests directory.
$ ./breeze-legacy --backend mysql --mysql-version 5.7 --python 3.8 --db-reset --test-type All tests
-- Running specific test in container using shell scripts. Testing in container scripts are located in
- ``./scripts/in_container`` directory.
-
-.. code-block:: bash
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/
- bin/ run_flake8.sh*
- check_environment.sh* run_generate_constraints.sh*
- entrypoint_ci.sh* run_init_script.sh*
- entrypoint_exec.sh* run_install_and_test_provider_packages.sh*
- _in_container_script_init.sh* run_mypy.sh*
- prod/ run_prepare_provider_packages.sh*
- run_ci_tests.sh* run_prepare_provider_documentation.sh*
- run_clear_tmp.sh* run_system_tests.sh*
- run_docs_build.sh* run_tmux_welcome.sh*
- run_extract_tests.sh* stop_tmux_airflow.sh*
- run_fix_ownership.sh* update_quarantined_test_status.py*
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/run_docs_build.sh
-
- Running specific type of test
- Types of tests
@@ -1642,31 +1576,7 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r
Run flake8..........................................................Passed
-7. Running specific checks in container using shell scripts. Scripts are located in ``./scripts/in_container``
- directory.
-
-.. code-block:: bash
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/
- bin/ run_flake8.sh*
- check_environment.sh* run_generate_constraints.sh*
- entrypoint_ci.sh* run_init_script.sh*
- entrypoint_exec.sh* run_install_and_test_provider_packages.sh*
- _in_container_script_init.sh* run_mypy.sh*
- prod/ run_prepare_provider_packages.sh*
- run_ci_tests.sh* run_prepare_provider_documentation.sh*
- run_clear_tmp.sh* run_system_tests.sh*
- run_docs_build.sh* run_tmux_welcome.sh*
- run_extract_tests.sh* stop_tmux_airflow.sh*
- run_fix_ownership.sh* update_quarantined_test_status.py*
-
-
- root@df8927308887:/opt/airflow# ./scripts/in_container/run_docs_build.sh
-
-
-
-
-8. Enabling Pre-commit check before push. It will run pre-commit automatically before committing and stops the commit
+7. Enabling Pre-commit check before push. It will run pre-commit automatically before committing and stops the commit
.. code-block:: bash
@@ -1674,7 +1584,7 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r
$ pre-commit install
$ git commit -m "Added xyz"
-9. To disable Pre-commit
+8. To disable Pre-commit
.. code-block:: bash
@@ -1705,10 +1615,6 @@ To avoid burden on CI infrastructure and to save time, Pre-commit hooks can be r
<a href="https://github.com/apache/airflow/blob/main/STATIC_CODE_CHECKS.rst#running-static-code-checks-via-breeze"
target="_blank">Running Static Code Checks via Breeze</a>
-
-
-
-
Contribution guide
~~~~~~~~~~~~~~~~~~
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 2cef0e4136..920edff152 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -670,391 +670,44 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
echo
echo "${COLOR_BLUE}Skip installing airflow - only install wheel/tar.gz packages that are present locally.${COLOR_RESET}"
echo
- uninstall_airflow_and_providers
- elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
echo
- echo "${COLOR_BLUE}Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
echo
uninstall_airflow_and_providers
- install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]"
- uninstall_providers
- elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then
+ elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
echo
- echo "${COLOR_BLUE}Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
echo
uninstall_airflow_and_providers
- install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]"
- uninstall_providers
- else
- echo
- echo "${COLOR_BLUE}Install airflow from PyPI without extras"
- echo
- install_released_airflow_version "${USE_AIRFLOW_VERSION}"
- fi
- if [[ ${USE_PACKAGES_FROM_DIST=} == "true" ]]; then
- echo
- echo "${COLOR_BLUE}Install all packages from dist folder"
- if [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
- echo "(except apache-airflow)"
- fi
- if [[ ${PACKAGE_FORMAT} == "both" ]]; then
- echo
- echo "${COLOR_RED}ERROR:You can only specify 'wheel' or 'sdist' as PACKAGE_FORMAT not 'both'.${COLOR_RESET}"
- echo
- exit 1
- fi
- echo
- installable_files=()
- for file in /dist/*.{whl,tar.gz}
- do
- if [[ ${USE_AIRFLOW_VERSION} == "wheel" && ${file} == "apache?airflow-[0-9]"* ]]; then
- # Skip Apache Airflow package - it's just been installed above with extras
- echo "Skipping ${file}"
- continue
- fi
- if [[ ${PACKAGE_FORMAT} == "wheel" && ${file} == *".whl" ]]; then
- echo "Adding ${file} to install"
- installable_files+=( "${file}" )
- fi
- if [[ ${PACKAGE_FORMAT} == "sdist" && ${file} == *".tar.gz" ]]; then
- echo "Adding ${file} to install"
- installable_files+=( "${file}" )
- fi
- done
- if (( ${#installable_files[@]} )); then
- pip install "${installable_files[@]}" --no-deps
- fi
- fi
-
- # Added to have run-tests on path
- export PATH=${PATH}:${AIRFLOW_SOURCES}
-
- # This is now set in conftest.py - only for pytest tests
- unset AIRFLOW__CORE__UNIT_TEST_MODE
-
- mkdir -pv "${AIRFLOW_HOME}/logs/"
- cp -f "${IN_CONTAINER_DIR}/airflow_ci.cfg" "${AIRFLOW_HOME}/unittests.cfg"
-
- # Change the default worker_concurrency for tests
- export AIRFLOW__CELERY__WORKER_CONCURRENCY=8
-
- set +e
-
- "${IN_CONTAINER_DIR}/check_environment.sh"
- ENVIRONMENT_EXIT_CODE=$?
- set -e
- if [[ ${ENVIRONMENT_EXIT_CODE} != 0 ]]; then
- echo
- echo "Error: check_environment returned ${ENVIRONMENT_EXIT_CODE}. Exiting."
+ echo "${COLOR_BLUE}Install airflow from wheel package with extras: '${AIRFLOW_EXTRAS}' and constraints reference ${AIRFLOW_CONSTRAINTS_REFERENCE}.${COLOR_RESET}"
echo
- exit ${ENVIRONMENT_EXIT_CODE}
- fi
- # Create symbolic link to fix possible issues with kubectl config cmd-path
- mkdir -p /usr/lib/google-cloud-sdk/bin
- touch /usr/lib/google-cloud-sdk/bin/gcloud
- ln -s -f /usr/bin/gcloud /usr/lib/google-cloud-sdk/bin/gcloud
-
- if [[ ${SKIP_SSH_SETUP="false"} == "false" ]]; then
- # Set up ssh keys
- echo 'yes' | ssh-keygen -t rsa -C your_email@youremail.com -m PEM -P '' -f ~/.ssh/id_rsa \
- >"${AIRFLOW_HOME}/logs/ssh-keygen.log" 2>&1
-
- cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
- ln -s -f ~/.ssh/authorized_keys ~/.ssh/authorized_keys2
- chmod 600 ~/.ssh/*
-
- # SSH Service
- sudo service ssh restart >/dev/null 2>&1
-
- # Sometimes the server is not quick enough to load the keys!
- while [[ $(ssh-keyscan -H localhost 2>/dev/null | wc -l) != "3" ]] ; do
- echo "Not all keys yet loaded by the server"
- sleep 0.05
- done
-
- ssh-keyscan -H localhost >> ~/.ssh/known_hosts 2>/dev/null
- fi
-
- # shellcheck source=scripts/in_container/configure_environment.sh
- . "${IN_CONTAINER_DIR}/configure_environment.sh"
-
- # shellcheck source=scripts/in_container/run_init_script.sh
- . "${IN_CONTAINER_DIR}/run_init_script.sh"
-
- cd "${AIRFLOW_SOURCES}"
-
- if [[ ${START_AIRFLOW:="false"} == "true" || ${START_AIRFLOW} == "True" ]]; then
- export AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=${LOAD_DEFAULT_CONNECTIONS}
- export AIRFLOW__CORE__LOAD_EXAMPLES=${LOAD_EXAMPLES}
- # shellcheck source=scripts/in_container/bin/run_tmux
- exec run_tmux
- fi
-fi
-
-set +u
-if [[ "${RUN_TESTS}" != "true" ]]; then
- exec /bin/bash "${@}"
-fi
-set -u
-
-export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}-${BACKEND}.xml"
-
-EXTRA_PYTEST_ARGS=(
- "--verbosity=0"
- "--strict-markers"
- "--durations=100"
- "--maxfail=50"
- "--color=yes"
- "--junitxml=${RESULT_LOG_FILE}"
- # timeouts in seconds for individual tests
- "--timeouts-order"
- "moi"
- "--setup-timeout=60"
- "--execution-timeout=60"
- "--teardown-timeout=60"
- # Only display summary for non-expected case
- # f - failed
- # E - error
- # X - xpassed (passed even if expected to fail)
- # The following cases are not displayed:
- # s - skipped
- # x - xfailed (expected to fail and failed)
- # p - passed
- # P - passed with output
- "-rfEX"
-)
-
-if [[ "${TEST_TYPE}" == "Helm" ]]; then
- # Enable parallelism
- EXTRA_PYTEST_ARGS+=(
- "-n" "auto"
- )
-else
- EXTRA_PYTEST_ARGS+=(
- "--with-db-init"
- )
-fi
-
-if [[ ${ENABLE_TEST_COVERAGE:="false"} == "true" ]]; then
- EXTRA_PYTEST_ARGS+=(
- "--cov=airflow/"
- "--cov-config=.coveragerc"
- "--cov-report=xml:/files/coverage-${TEST_TYPE}-${BACKEND}.xml"
- )
-fi
-
-declare -a SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
- ALL_TESTS ALL_PRESELECTED_TESTS ALL_OTHER_TESTS
-
-function find_all_other_tests() {
- local all_tests_dirs
- all_tests_dirs=$(find "tests" -type d)
- all_tests_dirs=$(echo "${all_tests_dirs}" | sed "/tests$/d" )
- all_tests_dirs=$(echo "${all_tests_dirs}" | sed "/tests\/dags/d" )
- local path
- for path in "${ALL_PRESELECTED_TESTS[@]}"
- do
- escaped_path="${path//\//\\\/}"
- all_tests_dirs=$(echo "${all_tests_dirs}" | sed "/${escaped_path}/d" )
- done
- for path in ${all_tests_dirs}
- do
- ALL_OTHER_TESTS+=("${path}")
- done
-}
-
-if [[ ${#@} -gt 0 && -n "$1" ]]; then
- SELECTED_TESTS=("${@}")
-else
- CLI_TESTS=("tests/cli")
- API_TESTS=("tests/api" "tests/api_connexion")
- PROVIDERS_TESTS=("tests/providers")
- ALWAYS_TESTS=("tests/always")
- CORE_TESTS=(
- "tests/core"
- "tests/executors"
- "tests/jobs"
- "tests/models"
- "tests/serialization"
- "tests/ti_deps"
- "tests/utils"
- )
- WWW_TESTS=("tests/www")
- HELM_CHART_TESTS=("tests/charts")
- ALL_TESTS=("tests")
- ALL_PRESELECTED_TESTS=(
- "${CLI_TESTS[@]}"
- "${API_TESTS[@]}"
- "${HELM_CHART_TESTS[@]}"
- "${PROVIDERS_TESTS[@]}"
- "${CORE_TESTS[@]}"
- "${ALWAYS_TESTS[@]}"
- "${WWW_TESTS[@]}"
- )
-
- if [[ ${TEST_TYPE:=""} == "CLI" ]]; then
- SELECTED_TESTS=("${CLI_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "API" ]]; then
- SELECTED_TESTS=("${API_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "Providers" ]]; then
- SELECTED_TESTS=("${PROVIDERS_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "Core" ]]; then
- SELECTED_TESTS=("${CORE_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "Always" ]]; then
- SELECTED_TESTS=("${ALWAYS_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "WWW" ]]; then
- SELECTED_TESTS=("${WWW_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "Helm" ]]; then
- SELECTED_TESTS=("${HELM_CHART_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "Other" ]]; then
- find_all_other_tests
- SELECTED_TESTS=("${ALL_OTHER_TESTS[@]}")
- elif [[ ${TEST_TYPE:=""} == "All" || ${TEST_TYPE} == "Quarantined" || \
- ${TEST_TYPE} == "Always" || \
- ${TEST_TYPE} == "Postgres" || ${TEST_TYPE} == "MySQL" || \
- ${TEST_TYPE} == "Long" || \
- ${TEST_TYPE} == "Integration" ]]; then
- SELECTED_TESTS=("${ALL_TESTS[@]}")
- else
- echo
- echo "${COLOR_RED}ERROR: Wrong test type ${TEST_TYPE} ${COLOR_RESET}"
- echo
- exit 1
- fi
-
-fi
-readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \
- ALL_TESTS ALL_PRESELECTED_TESTS
-
-if [[ -n ${LIST_OF_INTEGRATION_TESTS_TO_RUN=} ]]; then
- # Integration tests
- for INT in ${LIST_OF_INTEGRATION_TESTS_TO_RUN}
- do
- EXTRA_PYTEST_ARGS+=("--integration" "${INT}")
- done
-elif [[ ${TEST_TYPE:=""} == "Long" ]]; then
- EXTRA_PYTEST_ARGS+=(
- "-m" "long_running"
- "--include-long-running"
- )
-elif [[ ${TEST_TYPE:=""} == "Postgres" ]]; then
- EXTRA_PYTEST_ARGS+=(
- "--backend"
- "postgres"
- )
-elif [[ ${TEST_TYPE:=""} == "MySQL" ]]; then
- EXTRA_PYTEST_ARGS+=(
- "--backend"
- "mysql"
- )
-elif [[ ${TEST_TYPE:=""} == "Quarantined" ]]; then
- EXTRA_PYTEST_ARGS+=(
- "-m" "quarantined"
- "--include-quarantined"
- )
-fi
-
-echo
-echo "Running tests ${SELECTED_TESTS[*]}"
-echo
-
-ARGS=("${EXTRA_PYTEST_ARGS[@]}" "${SELECTED_TESTS[@]}")
-
-if [[ ${RUN_SYSTEM_TESTS:="false"} == "true" ]]; then
- "${IN_CONTAINER_DIR}/run_system_tests.sh" "${ARGS[@]}"
-else
- "${IN_CONTAINER_DIR}/run_ci_tests.sh" "${ARGS[@]}"
-fi
-EOF
-
-# The content below is automatically copied from scripts/docker/entrypoint_ci.sh
-COPY <<"EOF" /entrypoint_ci.sh
-#!/usr/bin/env bash
-if [[ ${VERBOSE_COMMANDS:="false"} == "true" ]]; then
- set -x
-fi
-
-. /opt/airflow/scripts/in_container/_in_container_script_init.sh
-
-LD_PRELOAD="/usr/lib/$(uname -m)-linux-gnu/libstdc++.so.6"
-export LD_PRELOAD
-
-chmod 1777 /tmp
-
-AIRFLOW_SOURCES=$(cd "${IN_CONTAINER_DIR}/../.." || exit 1; pwd)
-
-PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION:=3.6}
-
-export AIRFLOW_HOME=${AIRFLOW_HOME:=${HOME}}
-
-: "${AIRFLOW_SOURCES:?"ERROR: AIRFLOW_SOURCES not set !!!!"}"
-
-if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
-
- if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
- if [[ ${BACKEND:=} == "mysql" || ${BACKEND} == "mssql" ]]; then
- echo "${COLOR_RED}ARM platform is not supported for ${BACKEND} backend. Exiting.${COLOR_RESET}"
- exit 1
- fi
- fi
-
- echo
- echo "${COLOR_BLUE}Running Initialization. Your basic configuration is:${COLOR_RESET}"
- echo
- echo " * ${COLOR_BLUE}Airflow home:${COLOR_RESET} ${AIRFLOW_HOME}"
- echo " * ${COLOR_BLUE}Airflow sources:${COLOR_RESET} ${AIRFLOW_SOURCES}"
- echo " * ${COLOR_BLUE}Airflow core SQL connection:${COLOR_RESET} ${AIRFLOW__CORE__SQL_ALCHEMY_CONN:=}"
- echo
-
- RUN_TESTS=${RUN_TESTS:="false"}
- CI=${CI:="false"}
- USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION:=""}"
-
- if [[ ${USE_AIRFLOW_VERSION} == "" ]]; then
- export PYTHONPATH=${AIRFLOW_SOURCES}
- echo
- echo "${COLOR_BLUE}Using airflow version from current sources${COLOR_RESET}"
- echo
- if [[ -d "${AIRFLOW_SOURCES}/airflow/www/" ]]; then
- pushd "${AIRFLOW_SOURCES}/airflow/www/" >/dev/null
- ./ask_for_recompile_assets_if_needed.sh
- popd >/dev/null
- fi
- # Cleanup the logs, tmp when entering the environment
- sudo rm -rf "${AIRFLOW_SOURCES}"/logs/*
- sudo rm -rf "${AIRFLOW_SOURCES}"/tmp/*
- mkdir -p "${AIRFLOW_SOURCES}"/logs/
- mkdir -p "${AIRFLOW_SOURCES}"/tmp/
- elif [[ ${USE_AIRFLOW_VERSION} == "none" ]]; then
+ install_airflow_from_wheel "${AIRFLOW_EXTRAS}" "${AIRFLOW_CONSTRAINTS_REFERENCE}"
+ uninstall_providers
+ elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then
echo
- echo "${COLOR_BLUE}Skip installing airflow - only install wheel/tar.gz packages that are present locally.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
echo
uninstall_airflow_and_providers
- elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
echo
- echo "${COLOR_BLUE}Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Install airflow from sdist package with extras: '${AIRFLOW_EXTRAS}' and constraints reference ${AIRFLOW_CONSTRAINTS_REFERENCE}.${COLOR_RESET}"
echo
- uninstall_airflow_and_providers
- install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]"
+ install_airflow_from_sdist "${AIRFLOW_EXTRAS}" "${AIRFLOW_CONSTRAINTS_REFERENCE}"
uninstall_providers
- elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then
+ else
echo
- echo "${COLOR_BLUE}Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
echo
uninstall_airflow_and_providers
- install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]"
- uninstall_providers
- else
echo
- echo "${COLOR_BLUE}Install airflow from PyPI without extras"
+ echo "${COLOR_BLUE}Install released airflow from PyPI with extras: '${AIRFLOW_EXTRAS}' and constraints reference ${AIRFLOW_CONSTRAINTS_REFERENCE}.${COLOR_RESET}"
echo
- install_released_airflow_version "${USE_AIRFLOW_VERSION}"
+ install_released_airflow_version "${USE_AIRFLOW_VERSION}" "${AIRFLOW_CONSTRAINTS_REFERENCE}"
fi
if [[ ${USE_PACKAGES_FROM_DIST=} == "true" ]]; then
echo
- echo "${COLOR_BLUE}Install all packages from dist folder"
+ echo "${COLOR_BLUE}Install all packages from dist folder${COLOR_RESET}"
if [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
- echo "(except apache-airflow)"
+ echo "${COLOR_BLUE}(except apache-airflow)${COLOR_RESET}"
fi
if [[ ${PACKAGE_FORMAT} == "both" ]]; then
echo
@@ -1066,7 +719,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
installable_files=()
for file in /dist/*.{whl,tar.gz}
do
- if [[ ${USE_AIRFLOW_VERSION} == "wheel" && ${file} == "apache?airflow-[0-9]"* ]]; then
+ if [[ ${USE_AIRFLOW_VERSION} == "wheel" && ${file} == "/dist/apache?airflow-[0-9]"* ]]; then
# Skip Apache Airflow package - it's just been installed above with extras
echo "Skipping ${file}"
continue
@@ -1081,7 +734,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
fi
done
if (( ${#installable_files[@]} )); then
- pip install "${installable_files[@]}" --no-deps
+ pip install "${installable_files[@]}"
fi
fi
diff --git a/airflow/www/ask_for_recompile_assets_if_needed.sh b/airflow/www/ask_for_recompile_assets_if_needed.sh
index a50502e777..8342ae7631 100755
--- a/airflow/www/ask_for_recompile_assets_if_needed.sh
+++ b/airflow/www/ask_for_recompile_assets_if_needed.sh
@@ -15,9 +15,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
set -e
+if [[ ${GITHUB_ACTIONS:="false"} == "true" ]]; then
+ echo
+ echo -e "${BLUE}Skipping asset compilation check in CI.${NO_COLOR}"
+ echo
+ exit 0
+fi
+
cd "$( dirname "${BASH_SOURCE[0]}" )"
MD5SUM_FILE="static/dist/sum.md5"
diff --git a/breeze-legacy b/breeze-legacy
index 4b8bfe388a..d5ca8b8b4f 100755
--- a/breeze-legacy
+++ b/breeze-legacy
@@ -647,7 +647,6 @@ export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}"
export AIRFLOW_IMAGE_KUBERNETES="${AIRFLOW_IMAGE_KUBERNETES}"
export SQLITE_URL="${SQLITE_URL}"
export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
-export SKIP_PACKAGE_VERIFICATION="${SKIP_PACKAGE_VERIFICATION}"
export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
export EXECUTOR="${EXECUTOR}"
export START_AIRFLOW="${START_AIRFLOW}"
diff --git a/dev/PROVIDER_PACKAGE_DETAILS.md b/dev/PROVIDER_PACKAGE_DETAILS.md
index 1137f9dead..34abe8f86e 100644
--- a/dev/PROVIDER_PACKAGE_DETAILS.md
+++ b/dev/PROVIDER_PACKAGE_DETAILS.md
@@ -23,6 +23,7 @@
- [Overview](#overview)
- [Deciding when to release](#deciding-when-to-release)
+- [Verifying providers](#verifying-providers)
- [Generating provider documentation](#generating-provider-documentation)
- [Content of the release notes](#content-of-the-release-notes)
- [Preparing packages](#preparing-packages)
@@ -57,6 +58,21 @@ marking those as patches/features/backwards incompatible changes.
Details to be hashed out in [the related issue](https://github.com/apache/airflow/issues/11425)
+# Verifying providers
+
+You can verify if all providers are properly named and importable.
+
+```bash
+breeze verify-provider-packages
+```
+
+You can also run the verification with an earlier airflow version to check for compatibility.
+
+```bash
+breeze verify-provider-packages --use-airflow-version 2.1.0
+```
+
+
# Generating provider documentation
When you want to prepare release notes for a package, you need to run:
diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md
index 7669f15345..74c06a0693 100644
--- a/dev/README_RELEASE_PROVIDER_PACKAGES.md
+++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md
@@ -20,10 +20,12 @@
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
**Table of contents**
+- [What the provider packages are](#what-the-provider-packages-are)
- [Provider packages](#provider-packages)
- [Decide when to release](#decide-when-to-release)
- [Provider packages versioning](#provider-packages-versioning)
- [Prepare Regular Provider packages (RC)](#prepare-regular-provider-packages-rc)
+ - [Increasing version number](#increasing-version-number)
- [Generate release notes](#generate-release-notes)
- [Build provider packages for SVN apache upload](#build-provider-packages-for-svn-apache-upload)
- [Build and sign the source and convenience packages](#build-and-sign-the-source-and-convenience-packages)
@@ -47,6 +49,19 @@
------------------------------------------------------------------------------------------------------------
+# What the provider packages are
+
+The Provider packages are separate packages (one package per provider) that implement
+integrations with external services for Airflow in the form of installable Python packages.
+
+The Release Manager prepares packages separately from the main Airflow Release, using
+`breeze` commands and accompanying scripts. This document provides an overview of the command line tools
+needed to prepare the packages.
+
+NOTE!! When you have problems with any of those commands that run inside `breeze` docker image, you
+can run the command with `--debug` flag that will drop you in the shell inside the image and will
+print the command that you should run.
+
# Provider packages
The prerequisites to release Apache Airflow are described in [README.md](README.md).
@@ -72,11 +87,35 @@ Details about maintaining the SEMVER version are going to be discussed and imple
# Prepare Regular Provider packages (RC)
+## Increasing version number
+
+First thing that release manager has to do is to change version of the provider to a target
+version. Each provider has a `provider.yaml` file that, among others, stores information
+about provider versions. When you attempt to release a provider you should update that
+information based on the changes for the provider, and it's `CHANGELOG.rst`. It might be that
+`CHANGELOG.rst` already contains the right target version. This will be especially true if some
+changes in the provider add new features (then minor version is increased) or when the changes
+introduce backwards-incompatible, breaking change in the provider (then major version is
+incremented). Committers, when approving and merging changes to the providers, should pay attention
+that the `CHANGELOG.rst` is updated whenever anything other than bugfix is added.
+
+If there are no new features or breaking changes, the release manager should simply increase the
+patch-level version for the provider.
+
+The new version should be first on the list.
+
## Generate release notes
-Prepare release notes for all the packages you plan to release. When the provider package version
-has not been updated since the latest version, the release notes are not generated. Release notes
-are only generated, when the latest version of the package does not yet have a corresponding TAG.
+Each of the provider packages contains Release notes in the form of the `CHANGELOG.rst` file that is
+automatically generated from history of the changes and code of the provider.
+They are stored in the documentation directory. The `README.md` file generated during package
+preparation is not stored anywhere in the repository - it contains however link to the Changelog
+generated.
+
+When the provider package version has not been updated since the latest version, the release notes
+are not generated. Release notes are only generated, when the latest version of the package does not
+yet have a corresponding TAG.
+
The tags for providers is of the form ``providers-<PROVIDER_ID>/<VERSION>`` for example
``providers-amazon/1.0.0``. During releasing, the RC1/RC2 tags are created (for example
``providers-amazon/1.0.0rc1``).
@@ -84,7 +123,6 @@ The tags for providers is of the form ``providers-<PROVIDER_ID>/<VERSION>`` for
Details about maintaining the SEMVER version are going to be discussed and implemented in
[the related issue](https://github.com/apache/airflow/issues/11425)
-
```shell script
breeze prepare-provider-documentation [packages]
```
@@ -92,14 +130,15 @@ breeze prepare-provider-documentation [packages]
This command will not only prepare documentation but will also help the release manager to review
changes implemented in all providers, and determine which of the providers should be released. For each
provider details will be printed on what changes were implemented since the last release including
-links to particular commits. This should help to determine which version of provider should be released:
+links to particular commits.
+
+This should help to determine which version of provider should be released:
* increased patch-level for bugfix-only change
* increased minor version if new features are added
* increased major version if breaking changes are added
It also helps the release manager to update CHANGELOG.rst where high-level overview of the changes should be documented for the providers released.
-
You should iterate and re-generate the same content after any change as many times as you want.
The generated files should be added and committed to the repository.
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
index 3b09d61602..c962027ddf 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
@@ -26,7 +26,6 @@ from airflow_breeze.build_image.ci.build_ci_params import (
BuildCiParams,
)
from airflow_breeze.utils.cache import touch_cache_file
-from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
@@ -109,9 +108,7 @@ def should_we_run_the_build(build_ci_params: BuildCiParams, verbose: bool) -> bo
sys.exit(1)
-def build_ci_image(
- verbose: bool, dry_run: bool, with_ci_group: bool, ci_image_params: BuildCiParams
-) -> Tuple[int, str]:
+def build_ci_image(verbose: bool, dry_run: bool, ci_image_params: BuildCiParams) -> Tuple[int, str]:
"""
Builds CI image:
@@ -127,7 +124,6 @@ def build_ci_image(
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
- :param with_ci_group: whether to wrap the build in CI logging group
:param ci_image_params: CI image parameters
"""
fix_group_permissions(verbose=verbose)
@@ -136,63 +132,59 @@ def build_ci_image(
f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} "
f"python version: {ci_image_params.python}[/]\n"
)
- with ci_group(
- f"Build CI image for Python {ci_image_params.python} " f"with tag: {ci_image_params.image_tag}",
- enabled=with_ci_group,
- ):
- if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies:
- if not should_we_run_the_build(build_ci_params=ci_image_params, verbose=verbose):
- return 0, f"Image build: {ci_image_params.python}"
- run_command(
- ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_image_name],
+ if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies:
+ if not should_we_run_the_build(build_ci_params=ci_image_params, verbose=verbose):
+ return 0, f"Image build: {ci_image_params.python}"
+ run_command(
+ ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_image_name],
+ verbose=verbose,
+ dry_run=dry_run,
+ cwd=AIRFLOW_SOURCES_ROOT,
+ text=True,
+ check=False,
+ )
+ if ci_image_params.prepare_buildx_cache:
+ login_to_docker_registry(ci_image_params, dry_run=dry_run)
+ cmd = construct_docker_build_command(
+ image_params=ci_image_params,
+ verbose=verbose,
+ required_args=REQUIRED_CI_IMAGE_ARGS,
+ optional_args=OPTIONAL_CI_IMAGE_ARGS,
+ production_image=False,
+ )
+ if ci_image_params.empty_image:
+ env = os.environ.copy()
+ env['DOCKER_BUILDKIT'] = "1"
+ get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n")
+ cmd = construct_empty_docker_build_command(image_params=ci_image_params)
+ build_result = run_command(
+ cmd,
+ input="FROM scratch\n",
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
text=True,
- check=False,
+ env=env,
)
- if ci_image_params.prepare_buildx_cache:
- login_to_docker_registry(ci_image_params, dry_run=dry_run)
- cmd = construct_docker_build_command(
- image_params=ci_image_params,
- verbose=verbose,
- required_args=REQUIRED_CI_IMAGE_ARGS,
- optional_args=OPTIONAL_CI_IMAGE_ARGS,
- production_image=False,
+ else:
+ get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n")
+ build_result = run_command(
+ cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False
)
- if ci_image_params.empty_image:
- env = os.environ.copy()
- env['DOCKER_BUILDKIT'] = "1"
- get_console().print(f"\n[info]Building empty CI Image for Python {ci_image_params.python}\n")
- cmd = construct_empty_docker_build_command(image_params=ci_image_params)
- build_result = run_command(
- cmd,
- input="FROM scratch\n",
- verbose=verbose,
- dry_run=dry_run,
- cwd=AIRFLOW_SOURCES_ROOT,
- text=True,
- env=env,
- )
+ if not dry_run:
+ if build_result.returncode == 0:
+ mark_image_as_refreshed(ci_image_params)
else:
- get_console().print(f"\n[info]Building CI Image for Python {ci_image_params.python}\n")
- build_result = run_command(
- cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, text=True, check=False
+ get_console().print("[error]Error when building image![/]")
+ return (
+ build_result.returncode,
+ f"Image build: {ci_image_params.python}",
)
- if not dry_run:
- if build_result.returncode == 0:
- mark_image_as_refreshed(ci_image_params)
- else:
- get_console().print("[error]Error when building image![/]")
- return (
- build_result.returncode,
- f"Image build: {ci_image_params.python}",
- )
- else:
- get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]")
- if ci_image_params.push_image:
- return tag_and_push_image(image_params=ci_image_params, dry_run=dry_run, verbose=verbose)
- return build_result.returncode, f"Image build: {ci_image_params.python}"
+ else:
+ get_console().print("[info]Not updating build cache because we are in `dry_run` mode.[/]")
+ if ci_image_params.push_image:
+ return tag_and_push_image(image_params=ci_image_params, dry_run=dry_run, verbose=verbose)
+ return build_result.returncode, f"Image build: {ci_image_params.python}"
def mark_image_as_refreshed(ci_image_params: BuildCiParams):
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py
index 0619fb59f7..b744ce288d 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py
+++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_params.py
@@ -53,7 +53,8 @@ class BuildCiParams:
dev_apt_command: str = ""
dev_apt_deps: str = ""
image_tag: Optional[str] = None
- github_token: str = ""
+ github_token: str = os.environ.get('GITHUB_TOKEN', "")
+ github_actions: str = os.environ.get('GITHUB_ACTIONS', "false")
additional_dev_apt_command: str = ""
additional_dev_apt_deps: str = ""
additional_dev_apt_env: str = ""
diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
index 0e589eca72..abdc873628 100644
--- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_image.py
@@ -21,7 +21,6 @@ import sys
from typing import Tuple
from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
-from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
construct_docker_build_command,
@@ -121,7 +120,7 @@ def check_docker_context_files(install_packages_from_context: bool):
def build_production_image(
- verbose: bool, dry_run: bool, with_ci_group: bool, prod_image_params: BuildProdParams
+ verbose: bool, dry_run: bool, prod_image_params: BuildProdParams
) -> Tuple[int, str]:
"""
Builds PROD image:
@@ -140,7 +139,6 @@ def build_production_image(
:param verbose: print commands when running
:param dry_run: do not execute "write" commands - just print what would happen
- :param with_ci_group: whether to wrap the build in CI logging group
:param prod_image_params: PROD image parameters
"""
fix_group_permissions(verbose=verbose)
@@ -149,52 +147,47 @@ def build_production_image(
f"\n[info]Building PROD image of airflow from {AIRFLOW_SOURCES_ROOT} "
f"python version: {prod_image_params.python}[/]\n"
)
- with ci_group(
- f"Build Production image for Python {prod_image_params.python} "
- f"with tag: {prod_image_params.image_tag}",
- enabled=with_ci_group,
- ):
- if prod_image_params.cleanup_context:
- clean_docker_context_files(verbose=verbose, dry_run=dry_run)
- check_docker_context_files(prod_image_params.install_packages_from_context)
- if prod_image_params.prepare_buildx_cache:
- login_to_docker_registry(prod_image_params, dry_run=dry_run)
- run_command(
- ["docker", "rmi", "--no-prune", "--force", prod_image_params.airflow_image_name],
+ if prod_image_params.cleanup_context:
+ clean_docker_context_files(verbose=verbose, dry_run=dry_run)
+ check_docker_context_files(prod_image_params.install_packages_from_context)
+ if prod_image_params.prepare_buildx_cache:
+ login_to_docker_registry(prod_image_params, dry_run=dry_run)
+ run_command(
+ ["docker", "rmi", "--no-prune", "--force", prod_image_params.airflow_image_name],
+ verbose=verbose,
+ dry_run=dry_run,
+ cwd=AIRFLOW_SOURCES_ROOT,
+ text=True,
+ check=False,
+ )
+ get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n")
+ if prod_image_params.empty_image:
+ env = os.environ.copy()
+ env['DOCKER_BUILDKIT'] = "1"
+ get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n")
+ cmd = construct_empty_docker_build_command(image_params=prod_image_params)
+ build_command_result = run_command(
+ cmd,
+ input="FROM scratch\n",
verbose=verbose,
dry_run=dry_run,
cwd=AIRFLOW_SOURCES_ROOT,
- text=True,
check=False,
+ text=True,
+ env=env,
)
- get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n")
- if prod_image_params.empty_image:
- env = os.environ.copy()
- env['DOCKER_BUILDKIT'] = "1"
- get_console().print(f"\n[info]Building empty PROD Image for Python {prod_image_params.python}\n")
- cmd = construct_empty_docker_build_command(image_params=prod_image_params)
- build_command_result = run_command(
- cmd,
- input="FROM scratch\n",
- verbose=verbose,
- dry_run=dry_run,
- cwd=AIRFLOW_SOURCES_ROOT,
- check=False,
- text=True,
- env=env,
- )
- else:
- cmd = construct_docker_build_command(
- image_params=prod_image_params,
- verbose=verbose,
- required_args=REQUIRED_PROD_IMAGE_ARGS,
- optional_args=OPTIONAL_PROD_IMAGE_ARGS,
- production_image=True,
- )
- build_command_result = run_command(
- cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True
- )
- if build_command_result.returncode == 0:
- if prod_image_params.push_image:
- return tag_and_push_image(image_params=prod_image_params, dry_run=dry_run, verbose=verbose)
- return build_command_result.returncode, f"Image build: {prod_image_params.python}"
+ else:
+ cmd = construct_docker_build_command(
+ image_params=prod_image_params,
+ verbose=verbose,
+ required_args=REQUIRED_PROD_IMAGE_ARGS,
+ optional_args=OPTIONAL_PROD_IMAGE_ARGS,
+ production_image=True,
+ )
+ build_command_result = run_command(
+ cmd, verbose=verbose, dry_run=dry_run, cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True
+ )
+ if build_command_result.returncode == 0:
+ if prod_image_params.push_image:
+ return tag_and_push_image(image_params=prod_image_params, dry_run=dry_run, verbose=verbose)
+ return build_command_result.returncode, f"Image build: {prod_image_params.python}"
diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
index 0260907fbd..8372af7233 100644
--- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
+++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
@@ -77,7 +77,8 @@ class BuildProdParams:
image_tag: Optional[str] = None
extras: str = ""
additional_airflow_extras: str = ""
- github_token: str = ""
+ github_token: str = os.environ.get('GITHUB_TOKEN', "")
+ github_actions: str = os.environ.get('GITHUB_ACTIONS', "false")
login_to_github_registry: str = "false"
github_username: str = ""
platform: str = f"linux/{os.uname().machine}"
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py b/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
index 90761417ea..f6f01d3dba 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
@@ -61,10 +61,8 @@ from airflow_breeze.commands.common_options import (
option_verbose,
option_verify_image,
option_wait_for_image,
- option_with_ci_group,
)
from airflow_breeze.commands.main import main
-from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
from airflow_breeze.utils.python_versions import get_python_version_list
@@ -168,7 +166,6 @@ CI_IMAGE_TOOLS_PARAMETERS = {
@option_github_repository
@option_verbose
@option_dry_run
-@option_with_ci_group
@option_answer
@option_python
@option_build_multiple_images
@@ -202,7 +199,6 @@ CI_IMAGE_TOOLS_PARAMETERS = {
def build_image(
verbose: bool,
dry_run: bool,
- with_ci_group: bool,
build_multiple_images: bool,
python_versions: str,
answer: str,
@@ -211,22 +207,19 @@ def build_image(
"""Build CI image. Include building multiple images for all python versions (sequentially)."""
def run_build(ci_image_params: BuildCiParams) -> None:
- return_code, info = build_ci_image(
- verbose=verbose, dry_run=dry_run, with_ci_group=with_ci_group, ci_image_params=ci_image_params
- )
+ return_code, info = build_ci_image(verbose=verbose, dry_run=dry_run, ci_image_params=ci_image_params)
if return_code != 0:
get_console().print(f"[error]Error when building image! {info}")
sys.exit(return_code)
parameters_passed = filter_out_none(**kwargs)
if build_multiple_images:
- with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
- python_version_list = get_python_version_list(python_versions)
- for python in python_version_list:
- params = BuildCiParams(**parameters_passed)
- params.python = python
- params.answer = answer
- run_build(ci_image_params=params)
+ python_version_list = get_python_version_list(python_versions)
+ for python in python_version_list:
+ params = BuildCiParams(**parameters_passed)
+ params.python = python
+ params.answer = answer
+ run_build(ci_image_params=params)
else:
params = BuildCiParams(**parameters_passed)
run_build(ci_image_params=params)
diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py
index 7193743fad..474ba81b1c 100644
--- a/dev/breeze/src/airflow_breeze/commands/common_options.py
+++ b/dev/breeze/src/airflow_breeze/commands/common_options.py
@@ -19,17 +19,20 @@ import multiprocessing as mp
import click
+from airflow_breeze.branch_defaults import DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
from airflow_breeze.commands.custom_param_types import (
AnswerChoice,
BetterChoice,
CacheableChoice,
CacheableDefault,
+ UseAirflowVersionType,
)
from airflow_breeze.global_constants import (
ALLOWED_BACKENDS,
ALLOWED_BUILD_CACHE,
ALLOWED_DEBIAN_VERSIONS,
ALLOWED_EXECUTORS,
+ ALLOWED_INSTALLATION_PACKAGE_FORMATS,
ALLOWED_INTEGRATIONS,
ALLOWED_MOUNT_OPTIONS,
ALLOWED_MSSQL_VERSIONS,
@@ -38,6 +41,7 @@ from airflow_breeze.global_constants import (
ALLOWED_PLATFORMS,
ALLOWED_POSTGRES_VERSIONS,
ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+ ALLOWED_USE_AIRFLOW_VERSIONS,
get_available_packages,
)
from airflow_breeze.utils.recording import output_file_for_recording
@@ -55,7 +59,7 @@ option_dry_run = click.option(
option_answer = click.option(
"-a",
"--answer",
- type=AnswerChoice(choices=['y', 'n', 'q', 'yes', 'no', 'quit']),
+ type=AnswerChoice(['y', 'n', 'q', 'yes', 'no', 'quit']),
help="Force answer to questions.",
envvar='ANSWER',
)
@@ -124,11 +128,20 @@ option_forward_credentials = click.option(
'-f', '--forward-credentials', help="Forward local credentials to container when running.", is_flag=True
)
option_use_airflow_version = click.option(
- '-V',
'--use-airflow-version',
- help="Use (reinstall at entry) Airflow version from PyPI.",
+ help="Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or `sdist`"
+ " if Airflow should be removed, installed from wheel packages or sdist packages available in dist "
+ "folder respectively. Important! Using it Implies --mount-sources `none`.",
+ type=UseAirflowVersionType(ALLOWED_USE_AIRFLOW_VERSIONS),
envvar='USE_AIRFLOW_VERSION',
)
+option_airflow_extras = click.option(
+ '--airflow-extras',
+ help="Airflow extras to install when --use-airflow-version is used",
+ default="",
+ show_default=True,
+ envvar='AIRFLOW_EXTRAS',
+)
option_mount_sources = click.option(
'--mount-sources',
type=BetterChoice(ALLOWED_MOUNT_OPTIONS),
@@ -137,7 +150,7 @@ option_mount_sources = click.option(
help="Choose scope of local sources should be mounted (default = selected).",
)
option_force_build = click.option(
- '-f', '--force-build', help="Force image build no matter if it is determined as needed.", is_flag=True
+ '--force-build', help="Force image build no matter if it is determined as needed.", is_flag=True
)
option_db_reset = click.option(
'-d',
@@ -146,6 +159,13 @@ option_db_reset = click.option(
is_flag=True,
envvar='DB_RESET',
)
+option_use_packages_from_dist = click.option(
+ '--use-packages-from-dist',
+ is_flag=True,
+ help="Install all found packages (--package-format determines type) from 'dist' folder "
+ "when entering breeze.",
+ envvar='USE_PACKAGES_FROM_DIST',
+)
option_docker_cache = click.option(
'-c',
'--docker-cache',
@@ -191,7 +211,6 @@ option_platform = click.option(
type=BetterChoice(ALLOWED_PLATFORMS),
)
option_debian_version = click.option(
- '-d',
'--debian-version',
help='Debian version used for the image.',
type=BetterChoice(ALLOWED_DEBIAN_VERSIONS),
@@ -344,6 +363,14 @@ option_package_format = click.option(
show_default=True,
envvar='PACKAGE_FORMAT',
)
+option_installation_package_format = click.option(
+ '--package-format',
+ type=BetterChoice(ALLOWED_INSTALLATION_PACKAGE_FORMATS),
+ help='Format of packages that should be installed from dist.',
+ default=ALLOWED_INSTALLATION_PACKAGE_FORMATS[0],
+ show_default=True,
+ envvar='PACKAGE_FORMAT',
+)
option_python_versions = click.option(
'--python-versions',
help="Space separated list of python versions used for build with multiple versions.",
@@ -371,13 +398,6 @@ option_build_multiple_images = click.option(
is_flag=True,
envvar='BUILD_MULTIPLE_IMAGES',
)
-option_with_ci_group = click.option(
- '-g',
- '--with-ci-group',
- is_flag=True,
- help="Uses CI group for the command to fold long logs in logical groups.",
- envvar='WITH_CI_GROUP',
-)
argument_packages = click.argument(
"packages",
nargs=-1,
@@ -401,8 +421,10 @@ option_max_age = click.option(
default=3,
help="Max age of the last release (used if no updated-on-or-after if specified)",
)
-option_branch = click.option(
- "--constraints-branch",
- default='constraints-main',
- help="Constraint branch to use to find newer dependencies",
+option_airflow_constraints_reference = click.option(
+ "--airflow-constraints-reference",
+ default=DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH,
+ help="Constraint reference to use. Useful with --use-airflow-version parameter to specify "
+ "constraints for the installed version and to find newer dependencies",
+ envvar='AIRFLOW_CONSTRAINTS_REFERENCE',
)
diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
index 5f4d12c83b..39b150b8f4 100644
--- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
+++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
@@ -44,7 +44,7 @@ from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
check_docker_resources,
- construct_env_variables_docker_compose_command,
+ get_env_variables_for_docker_commands,
get_extra_docker_flags,
)
from airflow_breeze.utils.path_utils import (
@@ -415,18 +415,15 @@ def fix_ownership(verbose: bool, dry_run: bool):
verbose=verbose,
mount_sources=MOUNT_ALL,
python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ skip_environment_initialization=True,
)
extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
- env = construct_env_variables_docker_compose_command(shell_params)
+ env = get_env_variables_for_docker_commands(shell_params)
cmd = [
"docker",
"run",
"-t",
*extra_docker_flags,
- "-e",
- "GITHUB_ACTIONS=",
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
"--pull",
"never",
shell_params.airflow_image_name_with_tag,
diff --git a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py b/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
index 45c4ee0ff8..1851a95467 100644
--- a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
+++ b/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
@@ -16,7 +16,8 @@
# under the License.
from dataclasses import dataclass
-from typing import Any
+from re import match
+from typing import Any, Sequence
import click
@@ -40,8 +41,12 @@ class BetterChoice(click.Choice):
name = "BetterChoice"
+ def __init__(self, *args):
+ super().__init__(*args)
+ self.all_choices: Sequence[str] = self.choices
+
def get_metavar(self, param) -> str:
- choices_str = " | ".join(self.choices)
+ choices_str = " | ".join(self.all_choices)
# Use curly braces to indicate a required argument.
if param.required and param.param_type_name == "argument":
return f"{{{choices_str}}}"
@@ -126,3 +131,16 @@ class CacheableChoice(click.Choice):
def __init__(self, choices, case_sensitive: bool = True) -> None:
super().__init__(choices=choices, case_sensitive=case_sensitive)
+
+
+class UseAirflowVersionType(BetterChoice):
+ """Extends choice with dynamic version number."""
+
+ def __init__(self, *args):
+ super().__init__(*args)
+ self.all_choices = [*self.choices, "<airflow_version>"]
+
+ def convert(self, value, param, ctx):
+ if match(r"^\d*\.\d*\.\d*\S*$", value):
+ return value
+ return super().convert(value, param, ctx)
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 598190d5d9..8404337f62 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -24,6 +24,8 @@ import rich_click as click
from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
from airflow_breeze.commands.common_options import (
+ option_airflow_constraints_reference,
+ option_airflow_extras,
option_answer,
option_backend,
option_db_reset,
@@ -32,6 +34,7 @@ from airflow_breeze.commands.common_options import (
option_force_build,
option_forward_credentials,
option_github_repository,
+ option_installation_package_format,
option_integration,
option_load_default_connection,
option_load_example_dags,
@@ -41,6 +44,7 @@ from airflow_breeze.commands.common_options import (
option_postgres_version,
option_python,
option_use_airflow_version,
+ option_use_packages_from_dist,
option_verbose,
)
from airflow_breeze.commands.custom_param_types import BetterChoice
@@ -55,7 +59,7 @@ from airflow_breeze.shell.enter_shell import enter_shell, find_airflow_container
from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
- construct_env_variables_docker_compose_command,
+ get_env_variables_for_docker_commands,
get_extra_docker_flags,
)
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
@@ -81,11 +85,10 @@ DEVELOPER_PARAMETERS = {
"options": [
"--python",
"--backend",
- "--use-airflow-version",
"--postgres-version",
"--mysql-version",
"--mssql-version",
- "--debian-version",
+ "--integration",
"--forward-credentials",
"--db-reset",
],
@@ -93,9 +96,14 @@ DEVELOPER_PARAMETERS = {
{
"name": "Advanced flags for the default (shell) command",
"options": [
+ "--use-airflow-version",
+ "--constraints-reference",
+ "--airflow-extras",
+ "--use-packages-from-dist",
+ "--package-format",
"--force-build",
"--mount-sources",
- "--integration",
+ "--debian-version",
],
},
],
@@ -105,11 +113,10 @@ DEVELOPER_PARAMETERS = {
"options": [
"--python",
"--backend",
- "--use-airflow-version",
"--postgres-version",
"--mysql-version",
"--mssql-version",
- "--debian-version",
+ "--integration",
"--forward-credentials",
"--db-reset",
],
@@ -117,9 +124,14 @@ DEVELOPER_PARAMETERS = {
{
"name": "Advanced flag for running",
"options": [
+ "--use-airflow-version",
+ "--constraints-reference",
+ "--airflow-extras",
+ "--use-packages-from-dist",
+ "--package-format",
"--force-build",
"--mount-sources",
- "--integration",
+ "--debian-version",
],
},
],
@@ -128,13 +140,13 @@ DEVELOPER_PARAMETERS = {
"name": "Basic flags",
"options": [
"--python",
+ "--load-example-dags",
+ "--load-default-connections",
"--backend",
- "--use-airflow-version",
"--postgres-version",
"--mysql-version",
"--mssql-version",
- "--load-example-dags",
- "--load-default-connections",
+ "--integration",
"--forward-credentials",
"--db-reset",
],
@@ -142,9 +154,13 @@ DEVELOPER_PARAMETERS = {
{
"name": "Advanced flag for running",
"options": [
+ "--use-airflow-version",
+ "--constraints-reference",
+ "--airflow-extras",
+ "--use-packages-from-dist",
+ "--package-format",
"--force-build",
"--mount-sources",
- "--integration",
],
},
],
@@ -204,6 +220,10 @@ DEVELOPER_PARAMETERS = {
@option_forward_credentials
@option_force_build
@option_use_airflow_version
+@option_airflow_extras
+@option_airflow_constraints_reference
+@option_use_packages_from_dist
+@option_installation_package_format
@option_mount_sources
@option_integration
@option_db_reset
@@ -222,7 +242,11 @@ def shell(
debian_version: str,
forward_credentials: bool,
mount_sources: str,
- use_airflow_version: str,
+ use_packages_from_dist: bool,
+ package_format: str,
+ use_airflow_version: Optional[str],
+ airflow_extras: str,
+ airflow_constraints_reference: str,
force_build: bool,
db_reset: bool,
answer: Optional[str],
@@ -245,6 +269,10 @@ def shell(
forward_credentials=str(forward_credentials),
mount_sources=mount_sources,
use_airflow_version=use_airflow_version,
+ airflow_extras=airflow_extras,
+ airflow_constraints_reference=airflow_constraints_reference,
+ use_packages_from_dist=use_packages_from_dist,
+ package_format=package_format,
force_build=force_build,
db_reset=db_reset,
extra_args=extra_args,
@@ -267,6 +295,10 @@ def shell(
@option_forward_credentials
@option_force_build
@option_use_airflow_version
+@option_airflow_extras
+@option_airflow_constraints_reference
+@option_use_packages_from_dist
+@option_installation_package_format
@option_mount_sources
@option_integration
@option_db_reset
@@ -286,7 +318,11 @@ def start_airflow(
mssql_version: str,
forward_credentials: bool,
mount_sources: str,
- use_airflow_version: str,
+ use_airflow_version: Optional[str],
+ airflow_extras: str,
+ airflow_constraints_reference: str,
+ use_packages_from_dist: bool,
+ package_format: str,
force_build: bool,
db_reset: bool,
answer: Optional[str],
@@ -308,6 +344,10 @@ def start_airflow(
forward_credentials=str(forward_credentials),
mount_sources=mount_sources,
use_airflow_version=use_airflow_version,
+ airflow_extras=airflow_extras,
+ airflow_constraints_reference=airflow_constraints_reference,
+ use_packages_from_dist=use_packages_from_dist,
+ package_format=package_format,
force_build=force_build,
db_reset=db_reset,
start_airflow=True,
@@ -353,18 +393,15 @@ def build_docs(
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
+ skip_environment_initialization=True,
)
extra_docker_flags = get_extra_docker_flags(MOUNT_SELECTED)
- env = construct_env_variables_docker_compose_command(params)
+ env = get_env_variables_for_docker_commands(params)
cmd = [
"docker",
"run",
"-t",
*extra_docker_flags,
- "-e",
- "GITHUB_ACTIONS=",
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
"--pull",
"never",
ci_image_name,
@@ -475,7 +512,7 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool):
if not preserve_volumes:
command_to_execute.append("--volumes")
shell_params = ShellParams(verbose=verbose)
- env_variables = construct_env_variables_docker_compose_command(shell_params)
+ env_variables = get_env_variables_for_docker_commands(shell_params)
run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables)
@@ -485,6 +522,8 @@ class DocBuildParams:
docs_only: bool
spellcheck_only: bool
for_production: bool
+ skip_environment_initialization: bool = False
+ github_actions = os.environ.get('GITHUB_ACTIONS', "false")
@property
def args_doc_builder(self) -> List[str]:
diff --git a/dev/breeze/src/airflow_breeze/commands/main.py b/dev/breeze/src/airflow_breeze/commands/main.py
index 22baa02edd..10a12359be 100644
--- a/dev/breeze/src/airflow_breeze/commands/main.py
+++ b/dev/breeze/src/airflow_breeze/commands/main.py
@@ -16,6 +16,7 @@
# under the License.
from airflow_breeze.commands.common_options import (
+ option_airflow_extras,
option_answer,
option_backend,
option_db_reset,
@@ -24,6 +25,7 @@ from airflow_breeze.commands.common_options import (
option_force_build,
option_forward_credentials,
option_github_repository,
+ option_installation_package_format,
option_integration,
option_mount_sources,
option_mssql_version,
@@ -31,6 +33,7 @@ from airflow_breeze.commands.common_options import (
option_postgres_version,
option_python,
option_use_airflow_version,
+ option_use_packages_from_dist,
option_verbose,
)
from airflow_breeze.commands.configure_rich_click import click
@@ -50,6 +53,9 @@ from airflow_breeze.utils.path_utils import create_directories
@option_forward_credentials
@option_force_build
@option_use_airflow_version
+@option_airflow_extras
+@option_use_packages_from_dist
+@option_installation_package_format
@option_mount_sources
@option_integration
@option_db_reset
diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py b/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
index fcd09549cd..ac5bc7fe94 100644
--- a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
@@ -60,12 +60,10 @@ from airflow_breeze.commands.common_options import (
option_verbose,
option_verify_image,
option_wait_for_image,
- option_with_ci_group,
)
from airflow_breeze.commands.custom_param_types import BetterChoice
from airflow_breeze.commands.main import main
from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS
-from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
from airflow_breeze.utils.python_versions import get_python_version_list
@@ -182,7 +180,6 @@ PRODUCTION_IMAGE_TOOLS_PARAMETERS = {
@option_verbose
@option_dry_run
@option_answer
-@option_with_ci_group
@main.command(name='build-prod-image')
@option_python
@option_build_multiple_images
@@ -252,7 +249,6 @@ PRODUCTION_IMAGE_TOOLS_PARAMETERS = {
def build_prod_image(
verbose: bool,
dry_run: bool,
- with_ci_group: bool,
build_multiple_images: bool,
python_versions: str,
answer: Optional[str],
@@ -264,7 +260,7 @@ def build_prod_image(
def run_build(prod_image_params: BuildProdParams) -> None:
return_code, info = build_production_image(
- verbose=verbose, dry_run=dry_run, with_ci_group=with_ci_group, prod_image_params=prod_image_params
+ verbose=verbose, dry_run=dry_run, prod_image_params=prod_image_params
)
if return_code != 0:
get_console().print(f"[error]Error when building image! {info}")
@@ -272,13 +268,12 @@ def build_prod_image(
parameters_passed = filter_out_none(**kwargs)
if build_multiple_images:
- with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
- python_version_list = get_python_version_list(python_versions)
- for python in python_version_list:
- params = BuildProdParams(**parameters_passed)
- params.python = python
- params.answer = answer
- run_build(prod_image_params=params)
+ python_version_list = get_python_version_list(python_versions)
+ for python in python_version_list:
+ params = BuildProdParams(**parameters_passed)
+ params.python = python
+ params.answer = answer
+ run_build(prod_image_params=params)
else:
params = BuildProdParams(**parameters_passed)
run_build(prod_image_params=params)
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management.py b/dev/breeze/src/airflow_breeze/commands/release_management.py
index 4655b6197c..842759693f 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management.py
@@ -14,19 +14,23 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+import multiprocessing as mp
+import shlex
import sys
-from typing import IO, List, Optional, Tuple
+from subprocess import CalledProcessError, CompletedProcess
+from typing import IO, List, Optional, Tuple, Union
import click
from airflow_breeze.commands.common_options import (
argument_packages,
+ option_airflow_constraints_reference,
+ option_airflow_extras,
option_answer,
- option_branch,
option_dry_run,
option_github_repository,
option_image_tag,
+ option_installation_package_format,
option_max_age,
option_package_format,
option_parallelism,
@@ -35,9 +39,10 @@ from airflow_breeze.commands.common_options import (
option_run_in_parallel,
option_timezone,
option_updated_on_or_after,
+ option_use_airflow_version,
+ option_use_packages_from_dist,
option_verbose,
option_version_suffix_for_pypi,
- option_with_ci_group,
)
from airflow_breeze.commands.custom_param_types import BetterChoice
from airflow_breeze.commands.main import main
@@ -45,17 +50,18 @@ from airflow_breeze.global_constants import (
ALLOWED_GENERATE_CONSTRAINTS_MODES,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
MOUNT_ALL,
+ MOUNT_NONE,
+ MOUNT_SELECTED,
)
from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.confirm import Answer, user_confirm
from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.constraints import run_generate_constraints, run_generate_constraints_in_parallel
from airflow_breeze.utils.docker_command_utils import (
- construct_env_variables_docker_compose_command,
+ get_env_variables_for_docker_commands,
get_extra_docker_flags,
)
from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies
+from airflow_breeze.utils.parallel import check_async_run_results
from airflow_breeze.utils.python_versions import get_python_version_list
from airflow_breeze.utils.run_utils import run_command
@@ -63,6 +69,19 @@ RELEASE_MANAGEMENT_PARAMETERS = {
"breeze prepare-airflow-package": [
{"name": "Package flags", "options": ["--package-format", "--version-suffix-for-pypi"]}
],
+ "breeze verify-provider-packages": [
+ {
+ "name": "Provider verification flags",
+ "options": [
+ "--use-airflow-version",
+ "--constraints-reference",
+ "--airflow-extras",
+ "--use-packages-from-dist",
+ "--package-format",
+ "--debug",
+ ],
+ }
+ ],
"breeze prepare-provider-packages": [
{
"name": "Package flags",
@@ -70,21 +89,16 @@ RELEASE_MANAGEMENT_PARAMETERS = {
"--package-format",
"--version-suffix-for-pypi",
"--package-list-file",
+ "--debug",
],
}
],
"breeze prepare-provider-documentation": [
- {"name": "Provider documentation preparation flags", "options": ["--skip-package-verification"]}
- ],
- "breeze find-newer-dependencies": [
{
- "name": "Find newer dependencies flags",
+ "name": "Provider documentation preparation flags",
"options": [
- "--python",
- "--timezone",
- "--constraints-branch",
- "--updated-on-or-after",
- "--max-age",
+ "--skip-package-verification",
+ "--debug",
],
}
],
@@ -95,6 +109,7 @@ RELEASE_MANAGEMENT_PARAMETERS = {
"--image-tag",
"--python",
"--generate-constraints-mode",
+ "--debug",
],
},
{
@@ -106,11 +121,24 @@ RELEASE_MANAGEMENT_PARAMETERS = {
],
},
],
+ "breeze find-newer-dependencies": [
+ {
+ "name": "Find newer dependencies flags",
+ "options": [
+ "--python",
+ "--timezone",
+ "--constraints-reference",
+ "--updated-on-or-after",
+ "--max-age",
+ ],
+ }
+ ],
}
RELEASE_MANAGEMENT_COMMANDS = {
"name": "Release management",
"commands": [
+ "verify-provider-packages",
"prepare-provider-documentation",
"prepare-provider-packages",
"prepare-airflow-package",
@@ -119,6 +147,61 @@ RELEASE_MANAGEMENT_COMMANDS = {
],
}
+option_debug_release_management = click.option(
+ "--debug",
+ is_flag=True,
+ help="Drop user in shell instead of running the command. Useful for debugging.",
+ envvar='DEBUG',
+)
+
+
+def run_with_debug(
+ params: ShellParams,
+ command: List[str],
+ verbose: bool,
+ dry_run: bool,
+ debug: bool,
+ enable_input: bool = False,
+) -> Union[CompletedProcess, CalledProcessError]:
+ env_variables = get_env_variables_for_docker_commands(params)
+ extra_docker_flags = get_extra_docker_flags(mount_sources=params.mount_sources)
+ if enable_input or debug:
+ term_flag = "-it"
+ else:
+ term_flag = "-t"
+ base_command = [
+ "docker",
+ "run",
+ term_flag,
+ *extra_docker_flags,
+ "--pull",
+ "never",
+ params.airflow_image_name_with_tag,
+ ]
+ if debug:
+ cmd_string = ' '.join([shlex.quote(s) for s in command if s != "-c"])
+ base_command.extend(
+ [
+ "-c",
+ f"""
+echo -e '\\e[34mRun this command to debug:
+
+ {cmd_string}
+
+\\e[0m\n'; exec bash
+""",
+ ]
+ )
+ return run_command(
+ base_command,
+ verbose=verbose,
+ dry_run=dry_run,
+ env=env_variables,
+ )
+ else:
+ base_command.extend(command)
+ return run_command(base_command, verbose=verbose, dry_run=dry_run, env=env_variables, check=False)
+
@main.command(
name='prepare-airflow-package',
@@ -127,45 +210,35 @@ RELEASE_MANAGEMENT_COMMANDS = {
@option_verbose
@option_dry_run
@option_github_repository
-@option_with_ci_group
@option_package_format
@option_version_suffix_for_pypi
+@option_debug_release_management
def prepare_airflow_packages(
verbose: bool,
dry_run: bool,
github_repository: str,
- with_ci_group: bool,
package_format: str,
version_suffix_for_pypi: str,
+ debug: bool,
):
shell_params = ShellParams(
- verbose=verbose, github_repository=github_repository, python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION
+ verbose=verbose,
+ github_repository=github_repository,
+ python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ package_format=package_format,
+ version_suffix_for_pypi=version_suffix_for_pypi,
+ skip_environment_initialization=True,
+ install_providers_from_sources=False,
+ mount_sources=MOUNT_ALL,
)
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- env_variables['INSTALL_PROVIDERS_FROM_SOURCES'] = "false"
- extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
- with ci_group("Prepare Airflow package", enabled=with_ci_group):
- run_command(
- [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "-e",
- f"PACKAGE_FORMAT={package_format}",
- "-e",
- f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh",
- ],
- verbose=verbose,
- dry_run=dry_run,
- env=env_variables,
- )
+ result_command = run_with_debug(
+ params=shell_params,
+ command=["/opt/airflow/scripts/in_container/run_prepare_airflow_packages.sh"],
+ verbose=verbose,
+ dry_run=dry_run,
+ debug=debug,
+ )
+ sys.exit(result_command.returncode)
@main.command(
@@ -175,53 +248,35 @@ def prepare_airflow_packages(
@option_verbose
@option_dry_run
@option_github_repository
-@option_with_ci_group
@option_answer
-@click.option(
- '--skip-package-verification',
- help="Skip Provider package verification.",
- is_flag=True,
-)
+@option_debug_release_management
@argument_packages
def prepare_provider_documentation(
verbose: bool,
dry_run: bool,
github_repository: str,
- with_ci_group: bool,
answer: Optional[str],
- skip_package_verification: bool,
+ debug: bool,
packages: List[str],
):
shell_params = ShellParams(
verbose=verbose,
mount_sources=MOUNT_ALL,
- skip_package_verification=skip_package_verification,
github_repository=github_repository,
python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
answer=answer,
+ skip_environment_initialization=True,
)
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
- if answer and answer.lower() in ['y', 'yes']:
- term_flag = "-t"
- else:
- term_flag = "-it"
- cmd_to_run = [
- "docker",
- "run",
- term_flag,
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh",
- ]
- if packages:
- cmd_to_run.extend(packages)
- with ci_group("Prepare provider documentation", enabled=with_ci_group):
- run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
+ cmd_to_run = ["/opt/airflow/scripts/in_container/run_prepare_provider_documentation.sh", *packages]
+ result_command = run_with_debug(
+ params=shell_params,
+ command=cmd_to_run,
+ enable_input=not answer or answer.lower() not in ['y', 'yes'],
+ verbose=verbose,
+ dry_run=dry_run,
+ debug=debug,
+ )
+ sys.exit(result_command.returncode)
@main.command(
@@ -231,7 +286,6 @@ def prepare_provider_documentation(
@option_verbose
@option_dry_run
@option_github_repository
-@option_with_ci_group
@option_package_format
@option_version_suffix_for_pypi
@click.option(
@@ -239,15 +293,16 @@ def prepare_provider_documentation(
type=click.File('rt'),
help='Read list of packages from text file (one package per line)',
)
+@option_debug_release_management
@argument_packages
def prepare_provider_packages(
verbose: bool,
dry_run: bool,
github_repository: str,
- with_ci_group: bool,
package_format: str,
version_suffix_for_pypi: str,
package_list_file: IO,
+ debug: bool,
packages: Tuple[str, ...],
):
packages_list = list(packages)
@@ -258,28 +313,63 @@ def prepare_provider_packages(
mount_sources=MOUNT_ALL,
github_repository=github_repository,
python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ package_format=package_format,
+ skip_environment_initialization=True,
+ version_suffix_for_pypi=version_suffix_for_pypi,
+ )
+ cmd_to_run = ["/opt/airflow/scripts/in_container/run_prepare_provider_packages.sh", *packages_list]
+ result_command = run_with_debug(
+ params=shell_params,
+ command=cmd_to_run,
+ verbose=verbose,
+ dry_run=dry_run,
+ debug=debug,
)
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
+ sys.exit(result_command.returncode)
+
+
+def run_generate_constraints(
+ shell_params: ShellParams, dry_run: bool, verbose: bool, generate_constraints_mode: str, debug: bool
+) -> Tuple[int, str]:
cmd_to_run = [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "-e",
- f"PACKAGE_FORMAT={package_format}",
- "-e",
- f"VERSION_SUFFIX_FOR_PYPI={version_suffix_for_pypi}",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_prepare_provider_packages.sh",
+ "/opt/airflow/scripts/in_container/run_generate_constraints.sh",
]
- cmd_to_run.extend(packages_list)
- with ci_group("Prepare provider packages", enabled=with_ci_group):
- run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
+ generate_constraints_result = run_with_debug(
+ params=shell_params,
+ command=cmd_to_run,
+ verbose=verbose,
+ dry_run=dry_run,
+ debug=debug,
+ )
+ return (
+ generate_constraints_result.returncode,
+ f"Generate constraints Python {shell_params.python}:{generate_constraints_mode}",
+ )
+
+
+def run_generate_constraints_in_parallel(
+ shell_params_list: List[ShellParams],
+ python_version_list: List[str],
+ generate_constraints_mode: str,
+ parallelism: int,
+ dry_run: bool,
+ verbose: bool,
+):
+ """Run generate constraints in parallel"""
+ get_console().print(
+ f"\n[info]Generating constraints with parallelism = {parallelism} "
+ f"for the constraints: {python_version_list}[/]"
+ )
+ pool = mp.Pool(parallelism)
+ results = [
+ pool.apply_async(
+ run_generate_constraints,
+ args=(shell_param, dry_run, verbose, generate_constraints_mode, False),
+ )
+ for shell_param in shell_params_list
+ ]
+ check_async_run_results(results)
+ pool.close()
@main.command(
@@ -302,7 +392,7 @@ def prepare_provider_packages(
@option_python_versions
@option_image_tag
@option_answer
-@option_with_ci_group
+@option_debug_release_management
def generate_constraints(
verbose: bool,
dry_run: bool,
@@ -313,9 +403,12 @@ def generate_constraints(
python_versions: str,
image_tag: str,
answer: Optional[str],
+ debug: bool,
generate_constraints_mode: str,
- with_ci_group: bool,
):
+ if debug and run_in_parallel:
+ get_console().print("\n[error]Cannot run --debug and --run-in-parallel at the same time[/]\n")
+ sys.exit(1)
if run_in_parallel:
given_answer = user_confirm(
f"Did you build all CI images {python_versions} with --upgrade-to-newer-dependencies flag set?",
@@ -349,42 +442,94 @@ def generate_constraints(
)
for python in python_version_list
]
- with ci_group(f"Generating constraints with {generate_constraints_mode}", enabled=with_ci_group):
- run_generate_constraints_in_parallel(
- shell_params_list=shell_params_list,
- parallelism=parallelism,
- dry_run=dry_run,
- verbose=verbose,
- python_version_list=python_version_list,
- generate_constraints_mode=generate_constraints_mode,
- )
+ run_generate_constraints_in_parallel(
+ shell_params_list=shell_params_list,
+ parallelism=parallelism,
+ dry_run=dry_run,
+ verbose=verbose,
+ python_version_list=python_version_list,
+ generate_constraints_mode=generate_constraints_mode,
+ )
else:
- with ci_group(f"Generating constraints with {generate_constraints_mode}", enabled=with_ci_group):
- shell_params = ShellParams(
- image_tag=image_tag, python=python, github_repository=github_repository, answer=answer
- )
- return_code, info = run_generate_constraints(
- shell_params=shell_params,
- dry_run=dry_run,
- verbose=verbose,
- generate_constraints_mode=generate_constraints_mode,
- )
+ shell_params = ShellParams(
+ image_tag=image_tag,
+ python=python,
+ github_repository=github_repository,
+ answer=answer,
+ skip_environment_initialization=True,
+ )
+ return_code, info = run_generate_constraints(
+ shell_params=shell_params,
+ dry_run=dry_run,
+ verbose=verbose,
+ generate_constraints_mode=generate_constraints_mode,
+ debug=debug,
+ )
if return_code != 0:
get_console().print(f"[error]There was an error when generating constraints: {info}[/]")
sys.exit(return_code)
+@main.command(
+ name='verify-provider-packages',
+ help="Verifies if all provider code is following expectations for providers.",
+)
+@option_use_airflow_version
+@option_airflow_extras
+@option_airflow_constraints_reference
+@option_use_packages_from_dist
+@option_installation_package_format
+@option_verbose
+@option_dry_run
+@option_github_repository
+@option_debug_release_management
+def verify_provider_packages(
+ verbose: bool,
+ dry_run: bool,
+ use_airflow_version: Optional[str],
+ airflow_constraints_reference: str,
+ airflow_extras: str,
+ use_packages_from_dist: bool,
+ debug: bool,
+ package_format: str,
+ github_repository: str,
+):
+ shell_params = ShellParams(
+ verbose=verbose,
+ mount_sources=MOUNT_NONE if use_airflow_version is not None else MOUNT_SELECTED,
+ github_repository=github_repository,
+ python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ use_airflow_version=use_airflow_version,
+ airflow_extras=airflow_extras,
+ airflow_constraints_reference=airflow_constraints_reference,
+ use_packages_from_dist=use_packages_from_dist,
+ package_format=package_format,
+ )
+ cmd_to_run = [
+ "-c",
+ "python /opt/airflow/scripts/in_container/verify_providers.py",
+ ]
+ result_command = run_with_debug(
+ params=shell_params,
+ command=cmd_to_run,
+ verbose=verbose,
+ dry_run=dry_run,
+ debug=debug,
+ )
+ sys.exit(result_command.returncode)
+
+
@main.command(name="find-newer-dependencies", help="Finds which dependencies are being upgraded.")
@option_timezone
-@option_branch
+@option_airflow_constraints_reference
@option_python
@option_updated_on_or_after
@option_max_age
def breeze_find_newer_dependencies(
- constraints_branch: str, python: str, timezone: str, updated_on_or_after: str, max_age: int
+ airflow_constraints_reference: str, python: str, timezone: str, updated_on_or_after: str, max_age: int
):
return find_newer_dependencies(
- constraints_branch=constraints_branch,
+ constraints_branch=airflow_constraints_reference,
python=python,
timezone=timezone,
updated_on_or_after=updated_on_or_after,
diff --git a/dev/breeze/src/airflow_breeze/commands/testing.py b/dev/breeze/src/airflow_breeze/commands/testing.py
index e4a1ec1eb5..d51093ef72 100644
--- a/dev/breeze/src/airflow_breeze/commands/testing.py
+++ b/dev/breeze/src/airflow_breeze/commands/testing.py
@@ -38,7 +38,7 @@ from airflow_breeze.global_constants import ALLOWED_TEST_TYPES
from airflow_breeze.shell.enter_shell import check_docker_is_running, check_docker_resources
from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.docker_command_utils import construct_env_variables_docker_compose_command
+from airflow_breeze.utils.docker_command_utils import get_env_variables_for_docker_commands
from airflow_breeze.utils.run_tests import run_docker_compose_tests
from airflow_breeze.utils.run_utils import run_command
@@ -149,7 +149,7 @@ def tests(
os.environ["DB_RESET"] = "true"
exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
- env_variables = construct_env_variables_docker_compose_command(exec_shell_params)
+ env_variables = get_env_variables_for_docker_commands(exec_shell_params)
check_docker_is_running(verbose)
check_docker_resources(exec_shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
index 94a4360af4..3c9b95042d 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -80,10 +80,12 @@ ALLOWED_TEST_TYPES = [
'Quarantined',
]
ALLOWED_PACKAGE_FORMATS = ['wheel', 'sdist', 'both']
+ALLOWED_INSTALLATION_PACKAGE_FORMATS = ['wheel', 'sdist']
ALLOWED_INSTALLATION_METHODS = ['.', 'apache-airflow']
ALLOWED_DEBIAN_VERSIONS = ['bullseye', 'buster']
ALLOWED_BUILD_CACHE = ["pulled", "local", "disabled"]
ALLOWED_PLATFORMS = ["linux/amd64", "linux/arm64", "linux/amd64,linux/arm64"]
+ALLOWED_USE_AIRFLOW_VERSIONS = ['none', 'wheel', 'sdist']
PARAM_NAME_DESCRIPTION = {
"BACKEND": "backend",
@@ -233,7 +235,7 @@ DEFAULT_HELM_VERSIONS = CURRENT_HELM_VERSIONS[0]
DEFAULT_EXECUTOR = CURRENT_EXECUTORS[0]
# Initialize image build variables - Have to check if this has to go to ci dataclass
-USE_AIRFLOW_VERSION = ""
+USE_AIRFLOW_VERSION = None
GITHUB_ACTIONS = ""
ISSUE_ID = ""
diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
index e73153b60e..2249356e8f 100644
--- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
+++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
@@ -27,7 +27,7 @@ from airflow_breeze.utils.docker_command_utils import (
check_docker_is_running,
check_docker_resources,
check_docker_version,
- construct_env_variables_docker_compose_command,
+ get_env_variables_for_docker_commands,
)
from airflow_breeze.utils.rebuild_image_if_needed import rebuild_ci_image_if_needed
from airflow_breeze.utils.run_utils import filter_out_none, run_command
@@ -80,7 +80,7 @@ def run_shell_with_build_image_checks(
shell_params.print_badge_info()
cmd = ['docker-compose', 'run', '--service-ports', "-e", "BREEZE", '--rm', 'airflow']
cmd_added = shell_params.command_passed
- env_variables = construct_env_variables_docker_compose_command(shell_params)
+ env_variables = get_env_variables_for_docker_commands(shell_params)
if cmd_added is not None:
cmd.extend(['-c', cmd_added])
@@ -105,7 +105,7 @@ def find_airflow_container(verbose, dry_run) -> Optional[str]:
exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
check_docker_resources(exec_shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run)
exec_shell_params.print_badge_info()
- env_variables = construct_env_variables_docker_compose_command(exec_shell_params)
+ env_variables = get_env_variables_for_docker_commands(exec_shell_params)
cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running', 'airflow']
docker_compose_ps_command = run_command(
cmd, verbose=verbose, dry_run=dry_run, text=True, capture_output=True, env=env_variables, check=False
diff --git a/dev/breeze/src/airflow_breeze/shell/shell_params.py b/dev/breeze/src/airflow_breeze/shell/shell_params.py
index 129ef46097..e5a20fb232 100644
--- a/dev/breeze/src/airflow_breeze/shell/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/shell/shell_params.py
@@ -15,14 +15,17 @@
# specific language governing permissions and limitations
# under the License.
"""Breeze shell paameters."""
+import os
from dataclasses import dataclass
from pathlib import Path
from typing import Optional, Tuple
-from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
from airflow_breeze.global_constants import (
ALLOWED_BACKENDS,
ALLOWED_DEBIAN_VERSIONS,
+ ALLOWED_GENERATE_CONSTRAINTS_MODES,
+ ALLOWED_INSTALLATION_PACKAGE_FORMATS,
ALLOWED_MSSQL_VERSIONS,
ALLOWED_MYSQL_VERSIONS,
ALLOWED_POSTGRES_VERSIONS,
@@ -56,7 +59,15 @@ class ShellParams:
dry_run: bool = False
load_example_dags: bool = False
load_default_connections: bool = False
- use_airflow_version: str = ""
+ use_airflow_version: Optional[str] = None
+ airflow_extras: str = ""
+ airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
+ use_packages_from_dist: bool = False
+ package_format: str = ALLOWED_INSTALLATION_PACKAGE_FORMATS[0]
+ generate_constraints_mode: str = ALLOWED_GENERATE_CONSTRAINTS_MODES[0]
+ install_providers_from_sources: bool = True
+ skip_environment_initialization: bool = False
+ version_suffix_for_pypi: str = ""
install_airflow_version: str = ""
image_tag: str = "latest"
github_repository: str = "apache/airflow"
@@ -64,8 +75,8 @@ class ShellParams:
forward_credentials: str = "false"
airflow_branch: str = AIRFLOW_BRANCH
start_airflow: str = "false"
- skip_package_verification: bool = False
- github_actions: str = ""
+ github_token: str = os.environ.get('GITHUB_TOKEN', "")
+ github_actions: str = os.environ.get('GITHUB_ACTIONS', "false")
issue_id: str = ""
num_runs: str = ""
answer: Optional[str] = None
@@ -196,7 +207,7 @@ class ShellParams:
compose_ci_file.extend([backend_port_docker_compose_file])
if self.forward_credentials:
compose_ci_file.append(forward_credentials_docker_compose_file)
- if len(self.use_airflow_version) > 0:
+ if self.use_airflow_version is not None:
compose_ci_file.append(remove_sources_docker_compose_file)
if "all" in self.integration:
integrations = AVAILABLE_INTEGRATIONS
diff --git a/dev/breeze/src/airflow_breeze/utils/ci_group.py b/dev/breeze/src/airflow_breeze/utils/ci_group.py
index 4a0508334b..e0cb7b41ee 100644
--- a/dev/breeze/src/airflow_breeze/utils/ci_group.py
+++ b/dev/breeze/src/airflow_breeze/utils/ci_group.py
@@ -18,9 +18,11 @@
import os
from contextlib import contextmanager
+from airflow_breeze.utils.console import get_console
+
@contextmanager
-def ci_group(title, enabled: bool = False):
+def ci_group(title: str, enabled: bool = True):
"""
If used in GitHub Action, creates an expandable group in the GitHub Action log.
Otherwise, display simple text groups.
@@ -32,11 +34,9 @@ def ci_group(title, enabled: bool = False):
yield
return
if os.environ.get('GITHUB_ACTIONS', 'false') != "true":
- print("#" * 20, title, "#" * 20)
+ get_console().print(f"[info]{title}[/]")
yield
return
- print(f"::group::{title}")
- print()
+ get_console().print(f"::group::<EXPAND>: [info]{title}[/]")
yield
- print("\033[0m")
- print("::endgroup::")
+ get_console().print("::endgroup::")
diff --git a/dev/breeze/src/airflow_breeze/utils/constraints.py b/dev/breeze/src/airflow_breeze/utils/constraints.py
deleted file mode 100644
index 779b0d4924..0000000000
--- a/dev/breeze/src/airflow_breeze/utils/constraints.py
+++ /dev/null
@@ -1,79 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import multiprocessing as mp
-from typing import List, Tuple
-
-from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.console import get_console
-from airflow_breeze.utils.docker_command_utils import (
- construct_env_variables_docker_compose_command,
- get_extra_docker_flags,
-)
-from airflow_breeze.utils.parallel import check_async_run_results
-from airflow_breeze.utils.run_utils import run_command
-
-
-def run_generate_constraints(
- shell_params: ShellParams, dry_run: bool, verbose: bool, generate_constraints_mode: str
-) -> Tuple[int, str]:
- env_variables = construct_env_variables_docker_compose_command(shell_params)
- extra_docker_flags = get_extra_docker_flags(shell_params.mount_sources)
- cmd_to_run = [
- "docker",
- "run",
- "-t",
- *extra_docker_flags,
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "-e",
- f"GENERATE_CONSTRAINTS_MODE={generate_constraints_mode}",
- "--pull",
- "never",
- shell_params.airflow_image_name_with_tag,
- "/opt/airflow/scripts/in_container/run_generate_constraints.sh",
- ]
- generate_constraints_result = run_command(cmd_to_run, verbose=verbose, dry_run=dry_run, env=env_variables)
- return (
- generate_constraints_result.returncode,
- f"Generate constraints Python {shell_params.python}:{generate_constraints_mode}",
- )
-
-
-def run_generate_constraints_in_parallel(
- shell_params_list: List[ShellParams],
- python_version_list: List[str],
- generate_constraints_mode: str,
- parallelism: int,
- dry_run: bool,
- verbose: bool,
-):
- """Run generate constraints in parallel"""
- get_console().print(
- f"\n[info]Generating constraints with parallelism = {parallelism} "
- f"for the constraints: {python_version_list}[/]"
- )
- pool = mp.Pool(parallelism)
- results = [
- pool.apply_async(
- run_generate_constraints,
- args=(shell_param, dry_run, verbose, generate_constraints_mode),
- )
- for shell_param in shell_params_list
- ]
- check_async_run_results(results)
- pool.close()
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 9d30a86345..52539cefd3 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -103,7 +103,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]:
for flag in NECESSARY_HOST_VOLUMES:
extra_docker_flags.extend(["-v", str(AIRFLOW_SOURCES_ROOT) + flag])
else: # none
- get_console().print('[info]Skip mounting host volumes to Docker[/]')
+ extra_docker_flags.extend(["-v", f"{AIRFLOW_SOURCES_ROOT / 'empty'}:/opt/airflow/airflow"])
extra_docker_flags.extend(["-v", f"{AIRFLOW_SOURCES_ROOT}/files:/files"])
extra_docker_flags.extend(["-v", f"{AIRFLOW_SOURCES_ROOT}/dist:/dist"])
extra_docker_flags.extend(["--rm"])
@@ -286,6 +286,16 @@ Make sure docker-compose you install is first on the PATH variable of yours.
)
+def get_env_variable_value(arg_name: str, params: Union[BuildCiParams, BuildProdParams, ShellParams]):
+ raw_value = getattr(params, arg_name, None)
+ value = str(raw_value) if raw_value is not None else ''
+ value = "true" if raw_value is True else value
+ value = "false" if raw_value is False else value
+ if arg_name == "upgrade_to_newer_dependencies" and value == "true":
+ value = f"{randint(0, 2**32):x}"
+ return value
+
+
def construct_arguments_for_docker_build_command(
image_params: Union[BuildCiParams, BuildProdParams], required_args: List[str], optional_args: List[str]
) -> List[str]:
@@ -304,20 +314,14 @@ def construct_arguments_for_docker_build_command(
:return: list of `--build-arg` commands to use for the parameters passed
"""
- def get_env_variable_value(arg_name: str):
- value = str(getattr(image_params, arg_name))
- value = "true" if value.lower() in ["true", "t", "yes", "y"] else value
- value = "false" if value.lower() in ["false", "f", "no", "n"] else value
- if arg_name == "upgrade_to_newer_dependencies" and value == "true":
- value = f"{randint(0, 2**32):x}"
- return value
-
args_command = []
for required_arg in required_args:
args_command.append("--build-arg")
- args_command.append(required_arg.upper() + "=" + get_env_variable_value(arg_name=required_arg))
+ args_command.append(
+ required_arg.upper() + "=" + get_env_variable_value(arg_name=required_arg, params=image_params)
+ )
for optional_arg in optional_args:
- param_value = get_env_variable_value(optional_arg)
+ param_value = get_env_variable_value(optional_arg, params=image_params)
if len(param_value) > 0:
args_command.append("--build-arg")
args_command.append(optional_arg.upper() + "=" + param_value)
@@ -444,6 +448,7 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None:
"""
set_value_to_default_if_not_set(env, 'ANSWER', "")
set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "")
+ set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH)
set_value_to_default_if_not_set(env, 'BREEZE', "true")
set_value_to_default_if_not_set(env, 'CI', "false")
set_value_to_default_if_not_set(env, 'CI_BUILD_ID', "0")
@@ -485,9 +490,11 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None:
set_value_to_default_if_not_set(env, 'WHEEL_VERSION', "0.36.2")
-MAP_ENV_VARIABLES_TO_PARAMS_FIELDS = {
+DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES = {
"AIRFLOW_CI_IMAGE": "airflow_image_name",
"AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag",
+ "AIRFLOW_EXTRAS": "airflow_extras",
+ "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference",
"AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
"AIRFLOW_PROD_IMAGE": "airflow_image_name",
"AIRFLOW_SOURCES": "airflow_sources",
@@ -498,21 +505,21 @@ MAP_ENV_VARIABLES_TO_PARAMS_FIELDS = {
"DB_RESET": 'db_reset',
"ENABLED_INTEGRATIONS": "enabled_integrations",
"GITHUB_ACTIONS": "github_actions",
- "HOST_GROUP_ID": "host_group_id",
- "HOST_USER_ID": "host_user_id",
"INSTALL_AIRFLOW_VERSION": "install_airflow_version",
"ISSUE_ID": "issue_id",
"LOAD_EXAMPLES": "load_example_dags",
"LOAD_DEFAULT_CONNECTIONS": "load_default_connections",
"NUM_RUNS": "num_runs",
+ "PACKAGE_FORMAT": "package_format",
"PYTHON_MAJOR_MINOR_VERSION": "python",
- "SKIP_PACKAGE_VERIFICATION": "skip_package_verification",
"SQLITE_URL": "sqlite_url",
"START_AIRFLOW": "start_airflow",
"USE_AIRFLOW_VERSION": "use_airflow_version",
+ "USE_PACKAGES_FROM_DIST": "use_packages_from_dist",
+ "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi",
}
-VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS = {
+DOCKER_VARIABLE_CONSTANTS = {
"FLOWER_HOST_PORT": FLOWER_HOST_PORT,
"MSSQL_HOST_PORT": MSSQL_HOST_PORT,
"MSSQL_VERSION": MSSQL_VERSION,
@@ -525,26 +532,8 @@ VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS = {
"WEBSERVER_HOST_PORT": WEBSERVER_HOST_PORT,
}
-VARIABLES_IN_CACHE = {
- 'backend': 'BACKEND',
- 'mssql_version': 'MSSQL_VERSION',
- 'mysql_version': 'MYSQL_VERSION',
- 'postgres_version': 'POSTGRES_VERSION',
- 'python': 'PYTHON_MAJOR_MINOR_VERSION',
-}
-
-SOURCE_OF_DEFAULT_VALUES_FOR_VARIABLES = {
- 'backend': 'DEFAULT_BACKEND',
- 'mssql_version': 'MSSQL_VERSION',
- 'mysql_version': 'MYSQL_VERSION',
- 'postgres_version': 'POSTGRES_VERSION',
- 'python': 'DEFAULT_PYTHON_MAJOR_MINOR_VERSION',
-}
-
-def construct_env_variables_docker_compose_command(
- params: Union[ShellParams, BuildCiParams]
-) -> Dict[str, str]:
+def get_env_variables_for_docker_commands(params: Union[ShellParams, BuildCiParams]) -> Dict[str, str]:
"""
Constructs environment variables needed by the docker-compose command, based on Shell parameters
passed to it.
@@ -559,13 +548,13 @@ def construct_env_variables_docker_compose_command(
:return: dictionary of env variables to set
"""
env_variables: Dict[str, str] = os.environ.copy()
- for variable in MAP_ENV_VARIABLES_TO_PARAMS_FIELDS:
- param_name = MAP_ENV_VARIABLES_TO_PARAMS_FIELDS[variable]
- param_value = getattr(params, param_name, None)
+ for variable in DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES:
+ param_name = DERIVE_ENV_VARIABLES_FROM_ATTRIBUTES[variable]
+ param_value = get_env_variable_value(param_name, params=params)
env_variables[variable] = str(param_value) if param_value is not None else ""
# Set constant defaults if not defined
- for variable in VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS:
- constant_param_value = VARIABLES_FOR_DOCKER_COMPOSE_CONSTANTS[variable]
+ for variable in DOCKER_VARIABLE_CONSTANTS:
+ constant_param_value = DOCKER_VARIABLE_CONSTANTS[variable]
if not env_variables.get(constant_param_value):
env_variables[variable] = str(constant_param_value)
update_expected_environment_variables(env_variables)
diff --git a/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
index f4c37d9f22..2c37a065ae 100644
--- a/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
+++ b/dev/breeze/src/airflow_breeze/utils/find_newer_dependencies.py
@@ -45,7 +45,7 @@ def find_newer_dependencies(
from packaging import version
constraints = requests.get(
- f"https://raw.githubusercontent.com/" f"apache/airflow/{constraints_branch}/constraints-{python}.txt"
+ f"https://raw.githubusercontent.com/apache/airflow/{constraints_branch}/constraints-{python}.txt"
).text
package_lines = list(filter(lambda x: not x.startswith("#"), constraints.splitlines()))
constrained_packages: Dict[str, Any] = {}
diff --git a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py
index eedec9296d..0900793a48 100644
--- a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py
+++ b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py
@@ -106,10 +106,10 @@ def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path, verbose: bool) -> bo
get_console().print('\n[warning]Likely CI image needs rebuild[/]\n')
build_needed = True
else:
- if verbose:
- get_console().print(
- '[info]Docker image build is not needed for CI build as no important files are changed![/]'
- )
+ get_console().print(
+ '[info]Docker image build is not needed for CI build as no important files are changed! '
+ 'You can add --force-build to force it[/]'
+ )
return build_needed
diff --git a/dev/breeze/src/airflow_breeze/utils/pulll_image.py b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
index 28179a2329..adc1ecfb38 100644
--- a/dev/breeze/src/airflow_breeze/utils/pulll_image.py
+++ b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
@@ -144,7 +144,10 @@ def run_pull_image(
return command_result.returncode, f"Image Python {image_params.python}"
if wait_for_image:
if verbose or dry_run:
- get_console().print(f"\n[info]Waiting for {poll_time} seconds.[/]\n")
+ get_console().print(
+ f"\n[info]Waiting for {poll_time} seconds for "
+ f"{image_params.airflow_image_name_with_tag}.[/]\n"
+ )
time.sleep(poll_time)
continue
else:
diff --git a/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py b/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py
index a3981fa542..158361997a 100644
--- a/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py
+++ b/dev/breeze/src/airflow_breeze/utils/rebuild_image_if_needed.py
@@ -55,4 +55,4 @@ def rebuild_ci_image_if_needed(
f'[warning]{build_params.the_image_type} image not built locally. Forcing build.[/]'
)
ci_image_params.force_build = True
- build_ci_image(verbose, dry_run=dry_run, with_ci_group=False, ci_image_params=ci_image_params)
+ build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params)
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index 1412655f2b..e4d4101939 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -26,12 +26,14 @@ from functools import lru_cache
from pathlib import Path
from typing import Dict, List, Mapping, Optional, Union
+from airflow_breeze.utils.ci_group import ci_group
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
def run_command(
cmd: List[str],
+ title: Optional[str] = None,
*,
check: bool = True,
verbose: bool = False,
@@ -54,6 +56,7 @@ def run_command(
needed to run the command.
:param cmd: command to run
+ :param title: optional title for the command (otherwise likely title is automatically determined)
:param check: whether to check status value and run exception (same as POpem)
:param verbose: print commands when running
:param dry_run: do not execute "the" command - just print what would happen
@@ -66,10 +69,21 @@ def run_command(
workdir: str = str(cwd) if cwd else os.getcwd()
if verbose or dry_run:
command_to_print = ' '.join(shlex.quote(c) for c in cmd)
+ if not title:
+ # Heuristics to get a short but explanatory title showing what the command does
+ # If title is not provided explicitly
+ title = ' '.join(
+ shlex.quote(c)
+ for c in cmd
+ if not c.startswith('-') # exclude options
+ and (c[0] != "/" or c.endswith(".sh")) # exclude volumes
+ and not c == "never" # exclude --pull never
+ )
env_to_print = get_environments_to_print(env)
- get_console().print(f"\n[info]Working directory {workdir} [/]\n")
- # Soft wrap allows to copy&paste and run resulting output as it has no hard EOL
- get_console().print(f"\n[info]{env_to_print}{command_to_print}[/]\n", soft_wrap=True)
+ with ci_group(title=f"Running {title}"):
+ get_console().print(f"\n[info]Working directory {workdir} [/]\n")
+ # Soft wrap allows to copy&paste and run resulting output as it has no hard EOL
+ get_console().print(f"\n[info]{env_to_print}{command_to_print}[/]\n", soft_wrap=True)
if dry_run:
return subprocess.CompletedProcess(cmd, returncode=0)
try:
@@ -102,7 +116,7 @@ def run_command(
def get_environments_to_print(env: Optional[Mapping[str, str]]):
if not env:
- return "# No environment variables \\\n"
+ return ""
system_env: Dict[str, str] = {}
my_env: Dict[str, str] = {}
for key, val in env.items():
diff --git a/dev/breeze/tests/test_commands.py b/dev/breeze/tests/test_commands.py
index 6116752699..796dfa4152 100644
--- a/dev/breeze/tests/test_commands.py
+++ b/dev/breeze/tests/test_commands.py
@@ -23,7 +23,19 @@ def test_visuals():
assert 2051 == len(ASCIIART)
-def test_get_extra_docker_flags():
- assert len(get_extra_docker_flags(MOUNT_ALL)) < 10
- assert len(get_extra_docker_flags(MOUNT_SELECTED)) > 60
- assert len(get_extra_docker_flags(MOUNT_NONE)) < 8
+def test_get_extra_docker_flags_all():
+ flags = get_extra_docker_flags(MOUNT_ALL)
+ assert "empty" not in "".join(flags)
+ assert len(flags) < 10
+
+
+def test_get_extra_docker_flags_selected():
+ flags = get_extra_docker_flags(MOUNT_SELECTED)
+ assert "empty" not in "".join(flags)
+ assert len(flags) > 60
+
+
+def test_get_extra_docker_flags_none():
+ flags = get_extra_docker_flags(MOUNT_NONE)
+ assert "empty" in "".join(flags)
+ assert len(flags) < 10
diff --git a/dev/import_all_classes.py b/dev/import_all_classes.py
deleted file mode 100755
index 398a65b866..0000000000
--- a/dev/import_all_classes.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/env python3
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import argparse
-import importlib
-import pkgutil
-import sys
-import traceback
-import warnings
-from inspect import isclass
-from typing import List, Optional, Set, Tuple
-from warnings import WarningMessage
-
-from rich.console import Console
-
-console = Console(width=400, color_system="standard")
-error_console = Console(width=400, color_system="standard", stderr=True)
-
-
-def import_all_classes(
- paths: List[str],
- prefix: str,
- provider_ids: Optional[List[str]] = None,
- print_imports: bool = False,
- print_skips: bool = False,
-) -> Tuple[List[str], List[WarningMessage]]:
- """
- Imports all classes in providers packages. This method loads and imports
- all the classes found in providers, so that we can find all the subclasses
- of operators/sensors etc.
-
- :param paths: list of paths to look the provider packages in
- :param prefix: prefix to add
- :param provider_ids - provider ids that should be loaded.
- :param print_imports - if imported class should also be printed in output
- :param print_skips - if skipped classes should also be printed in output
- :return: tuple of list of all imported classes and all warnings generated
- """
- imported_classes = []
- tracebacks: List[Tuple[str, str]] = []
- printed_packages: Set[str] = set()
-
- def mk_prefix(provider_id):
- return f'{prefix}{provider_id}'
-
- if provider_ids:
- provider_prefixes = [mk_prefix(provider_id) for provider_id in provider_ids]
- else:
- provider_prefixes = [prefix]
-
- def onerror(_):
- nonlocal tracebacks
- exception_string = traceback.format_exc()
- for provider_prefix in provider_prefixes:
- if provider_prefix in exception_string:
- start_index = exception_string.find(provider_prefix)
- end_index = exception_string.find("\n", start_index + len(provider_prefix))
- package = exception_string[start_index:end_index]
- tracebacks.append((package, exception_string))
- break
-
- all_warnings: List[WarningMessage] = []
- for modinfo in pkgutil.walk_packages(path=paths, prefix=prefix, onerror=onerror):
- if not any(modinfo.name.startswith(provider_prefix) for provider_prefix in provider_prefixes):
- if print_skips:
- console.print(f"Skipping module: {modinfo.name}")
- continue
- if print_imports:
- package_to_print = ".".join(modinfo.name.split(".")[:-1])
- if package_to_print not in printed_packages:
- printed_packages.add(package_to_print)
- console.print(f"Importing package: {package_to_print}")
- try:
- with warnings.catch_warnings(record=True) as w:
- warnings.filterwarnings("always", category=DeprecationWarning)
- _module = importlib.import_module(modinfo.name)
- for attribute_name in dir(_module):
- class_name = modinfo.name + "." + attribute_name
- attribute = getattr(_module, attribute_name)
- if isclass(attribute):
- imported_classes.append(class_name)
- if w:
- all_warnings.extend(w)
- except Exception:
- exception_str = traceback.format_exc()
- tracebacks.append((modinfo.name, exception_str))
- if tracebacks:
- console.print(
- """
-[red]ERROR: There were some import errors[/]
-""",
- )
- error_console.print("[red]----------------------------------------[/]")
- for package, trace in tracebacks:
- error_console.print(f"Exception when importing: {package}\n\n")
- error_console.print(trace)
- error_console.print("[red]----------------------------------------[/]")
- sys.exit(1)
- else:
- return imported_classes, all_warnings
-
-
-if __name__ == '__main__':
-
- parser = argparse.ArgumentParser(description='Perform import of all provider classes.')
- parser.add_argument('--path', action='append', help='paths to search providers in')
- parser.add_argument('--prefix', help='prefix to add in front of the class', default='airflow.providers.')
-
- args = parser.parse_args()
- print()
- print(f"Walking all packages in {args.path} with prefix {args.prefix}")
- print()
- classes, warns = import_all_classes(
- print_imports=True, print_skips=True, paths=args.path, prefix=args.prefix
- )
- if len(classes) == 0:
- print("[red]Something is seriously wrong - no classes imported[/]")
- sys.exit(1)
- if warns:
- print("[yellow]There were warnings generated during the import[/]")
- for w in warns:
- one_line_message = str(w.message).replace('\n', ' ')
- print(f"[yellow]{w.filename}:{w.lineno}: {one_line_message}[/]")
-
- print()
- print(f"[green]SUCCESS: All provider packages are importable! Imported {len(classes)} classes.[/]")
- print()
diff --git a/dev/provider_packages/README.md b/dev/provider_packages/README.md
deleted file mode 100644
index cd6f4ce8b1..0000000000
--- a/dev/provider_packages/README.md
+++ /dev/null
@@ -1,356 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-<!-- START doctoc generated TOC please keep comment here to allow auto update -->
-<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
-**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
-
-- [Provider packages](#provider-packages)
-- [What the provider packages are](#what-the-provider-packages-are)
- - [Increasing version number](#increasing-version-number)
-- [Generated release notes](#generated-release-notes)
- - [Generating release notes](#generating-release-notes)
- - [Preparing provider packages](#preparing-provider-packages)
-- [Testing and debugging provider preparation](#testing-and-debugging-provider-preparation)
- - [Debugging import check](#debugging-import-check)
- - [Debugging verifying provider classes](#debugging-verifying-provider-classes)
- - [Debugging preparing package documentation](#debugging-preparing-package-documentation)
- - [Debugging preparing release-notes](#debugging-preparing-release-notes)
- - [Debugging preparing setup files](#debugging-preparing-setup-files)
- - [Debugging preparing the packages](#debugging-preparing-the-packages)
-
-<!-- END doctoc generated TOC please keep comment here to allow auto update -->
-
-# Provider packages
-
-# What the provider packages are
-
-The Provider packages are separate packages (one package per provider) that implement
-integrations with external services for Airflow in the form of installable Python packages.
-
-The Release Manager prepares packages separately from the main Airflow Release, using
-`breeze` commands and accompanying scripts. This document provides an overview of the command line tools
-needed to prepare the packages.
-
-## Increasing version number
-
-First thing that release manager has to do is to change version of the provider to a target
-version. Each provider has a `provider.yaml` file that, among others, stores information
-about provider versions. When you attempt to release a provider you should update that
-information based on the changes for the provider, and it's `CHANGELOG.rst`. It might be that
-`CHANGELOG.rst` already contains the right target version. This will be especially true if some
-changes in the provider add new features (then minor version is increased) or when the changes
-introduce backwards-incompatible, breaking change in the provider (then major version is
-incremented). Committers, when approving and merging changes to the providers, should pay attention
-that the `CHANGELOG.rst` is updated whenever anything other than bugfix is added.
-
-If there are no new features or breaking changes, the release manager should simply increase the
-patch-level version for the provider.
-
-The new version should be first on the list.
-
-
-# Generated release notes
-
-Each of the provider packages contains Release notes in the form of the `CHANGELOG.rst` file that is
-automatically generated from history of the changes and code of the provider.
-They are stored in the documentation directory. The `README.md` file generated during package
-preparation is not stored anywhere in the repository - it contains however link to the Changelog
-generated.
-
-The `README.rst` file contains the following information:
-
-* summary of requirements for each provider package
-* list of dependencies (including extras to install them) when package depends on other providers package
-* link to the detailed changelog/index.rst file: generated documentation for the packages.
-
-The `index.rst` stored in the `docs\apache-airflow-providers-<PROVIDER>` folder contains:
-
-* Contents this is manually maintained there
-* the general package information (same for all packages with the name change)
-* summary of requirements for each provider package
-* list of dependencies (including extras to install them) when package depends on other providers package
-* Content of high-level CHANGELOG.rst file that is stored in the provider folder next to
- ``provider.yaml`` file.
-* Detailed list of changes generated for all versions of the provider automatically
-
-## Generating release notes
-
-When you want to prepare release notes for a package, you need to run:
-
-```
-breeze prepare-provider-documentation <PACKAGE_ID> ...
-```
-
-* <PACKAGE_ID> is usually directory in the `airflow/providers` folder (for example `google` but in several
- cases, it might be one level deeper separated with `.` for example `apache.hive`
-
-The index.rst is updated automatically in the `docs/apache-airflow-providers-<provider>` folder
-
-You can run the script with multiple package names if you want to prepare several packages at the same time.
-By default, the command runs in interactive mode when you can decide one-by-one whether the package
-documentation should be prepared or not.
-
-As soon as you are satisfied with the release notes generated you can commit generated changes/new files
-to the repository.
-
-You should manually update generated changelog and classify the commits updated and re-run the
-`prepare-documentation-readme` after all the changes.
-
-You can repeat this several times, the changes generated will automatically include new commits that
-appeared since last run.
-
-You can also run it in non-interactive mode adding `--answer yes` flag.
-
-## Preparing provider packages
-
-You build the packages in the breeze environment, so you do not have to worry about common environment.
-
-Note that readme release notes have to be generated first, so that the package preparation script reads
-the `provider.yaml`.
-
-* The provider package ids PACKAGE_ID are subdirectories in the ``providers`` directory. Sometimes they
-are one level deeper (`apache/hive` folder for example, in which case PACKAGE_ID uses "." to separate
-the folders (for example Apache Hive's PACKAGE_ID is `apache.hive` ). You can see the list of all available
-providers by running:
-
-```bash
-breeze prepare-provider-packages --help
-```
-
-The examples below show how you can build selected packages, but you can also build all packages by
-omitting the package ids altogether.
-
-By default, you build `both` packages, but you can use `--package-format wheel` to generate
-only wheel package, or `--package-format sdist` to only generate sdist package.
-
-* To build the release candidate packages for PyPI upload run the following command:
-
-```bash
-breeze prepare-provider-packages --version-suffix-for-pypi=rc1 [PACKAGE_ID] ...
-```
-
-for example:
-
-```bash
-breeze prepare-provider-packages --version-suffix-for-pypi=rc1 http ...
-```
-
-* To build the final release packages run the following command:
-
-```bash
-breeze prepare-provider-packages [--package-format PACKAGE_FORMAT] [PACKAGE_ID] ...
-```
-
-Where PACKAGE_FORMAT might be one of : `wheel`, `sdist`, `both` (`wheel` is the default format)
-
-for example:
-
-```bash
-breeze prepare-provider-packages http ...
-```
-
-* For each package, this creates a wheel package and source distribution package in your `dist` folder with
- names following the patterns:
-
- * `apache_airflow_providers_<PROVIDER>_YYYY.[M]M.[D]D[suffix]-py3-none-any.whl`
- * `apache-airflow-providers-<PROVIDER>-YYYY.[M]M.[D]D[suffix].tar.gz`
-
-Note! Even if we always use the two-digit month and day when generating the readme files,
-the version in PyPI does not contain the leading 0s in version name - therefore the artifacts generated
-also do not container the leading 0s.
-
-* You can install the .whl packages with `pip install <PACKAGE_FILE>`
-
-You can add `--verbose` flag if you want to see detailed commands executed by the script.
-
-# Testing and debugging provider preparation
-
-The provider preparation is done using `Breeze` development environment and CI image. This way we have
-common environment for package preparation, and we can easily verify if provider packages are OK and can
-be installed for released versions of Airflow (including 2.0.0 version).
-
-The same scripts and environment is run in our [CI Workflow](../../CI.rst) - the packages are prepared,
-installed and tested using the same CI image. The tests are performed via the Production image, also
-in the CI workflow. Our production images are built using Airflow and Provider packages prepared on the
-CI so that they are as close to what users will be using when they are installing from PyPI. Our scripts
-prepare `wheel` and `sdist` packages for both - airflow and provider packages and install them during
-building of the images. This is very helpful in case of testing new providers that do not yet have PyPI
-package released, but also it allows checking if provider's authors did not make breaking changes.
-
-All classes from all providers must be imported - otherwise our CI will fail. Also, verification
-of the image is performed where expected providers should be installed (for production image) and
-providers should be discoverable, as well as `pip check` with all the dependencies has to succeed.
-
-You might want to occasionally modify the preparation scripts for providers. They are all present in
-the `dev/provider_packages` folder. There are the `Breeze` commands above - they perform the sequence
-of those steps automatically, but you can manually run the scripts as follows to debug them:
-
-The commands are best to execute in the Breeze environment as it has all the dependencies installed,
-Examples below describe that. However, for development you might run them in your local development
-environment as it makes it easier to debug. Just make sure you install your development environment
-with 'devel_all' extra (make sure to use the right python version).
-
-Note that it is best to use `INSTALL_PROVIDERS_FROM_SOURCES` set to`true`, to make sure
-that any new added providers are not added as packages (in case they are not yet available in PyPI.
-
-```shell script
-INSTALL_PROVIDERS_FROM_SOURCES="true" pip install -e ".[devel_all]" \
- --constraint https://raw.githubusercontent.com/apache/airflow/constraints-main/constraintsource-providers-3.7.txt
-```
-
-Note that you might need to add some extra dependencies to your system to install "devel_all" - many
-dependencies are needed to make a clean install - the `Breeze` environment has all the
-dependencies installed in case you have problem with setting up your local virtualenv.
-
-You can also use the script `initialize_virtualenv.py` to prepare your virtualenv (it will print extra information if some
-dependencies are missing/installation fails and it will also reset your SQLite test db in
-the `${HOME}/airflow` directory:
-
-```shell script
-./scripts/tools/initialize_virtualenv.py
-```
-
-You can find description of all the commands and more information about the "prepare"
-tool by running it with `--help`
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py --help
-```
-
-You can see for example list of all provider packages:
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py list-providers-packages
-```
-
-You can add `--verbose` flag in breeze command if you want to see commands executed.
-
-## Debugging import check
-
-The script verifies if all provider's classes can be imported.
-
-1) Run import check:
-
-```shell script
-./dev/import_all_classes.py --path airflow/providers
-```
-
-It checks if all classes from provider packages can be imported.
-
-## Debugging verifying provider classes
-
-The script verifies if all provider's classes are correctly named.
-
-1) Run import check:
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py verify-provider-classes
-```
-
-It checks if all provider Operators/Hooks etc. are correctly named.
-
-
-## Debugging preparing package documentation
-
-The script updates documentation of the provider packages. Note that it uses airflow git and pulls
-the latest version of tags available in Airflow, so you need to enter Breeze with
-`--mount-all-local-sources flag`
-
-1) Run update documentation (version suffix might be empty):
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py update-package-documentation \
- --version-suffix <SUFFIX> \
- <PACKAGE>
-```
-
-This script will fetch the latest version of airflow from Airflow's repo (it will automatically add
-`apache-https-for-providers` remote and pull airflow (read only) from there. There is no need
-to setup any credentials for it.
-
-In case version being prepared is already tagged in the repo documentation preparation returns immediately
-and prints warning.
-
-You can add `--verbose` flag if you want to see detailed commands executed by the script.
-
-## Debugging preparing release-notes
-
-1) Run update changelog:
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py update-changelog <PACKAGE>
-```
-
-You can add `--verbose` flag if you want to see detailed commands executed by the script.
-
-## Debugging preparing setup files
-
-This script prepares the actual packages.
-
-1) Cleanup the artifact directories:
-
-This is needed because setup tools does not clean those files and generating packages one by one
-without cleanup, might include artifacts from previous package to be included in the new one.
-
-```shell script
-rm -rf -- *.egg-info build/
-```
-
-2) Generate setup.py/setup.cfg/MANIFEST.in/provider_info.py/README files files for:
-
-* alpha/beta packages (specify a1,a2,.../b1,b2... suffix)
-* release candidates (specify r1,r2,... suffix) - those are release candidate
-* official package (to be released in PypI as official package)
-
-The version suffix specified here will be appended to the version retrieved from
-`provider.yaml`. Note that this command will fail if the tag denoted by the
-version + suffix already exist. This means that the version was not updated since the
-last time it was generated. In the CI we always add 'dev' suffix, and we never create
-TAG for it, so in the CI the setup.py is generated and should never fail.
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py generate-setup-files \
- --version-suffix "<SUFFIX>" \
- <PACKAGE>
-```
-
-You can add `--verbose` flag if you want to see detailed commands executed by the script.
-
-## Debugging preparing the packages
-
-The script prepares the package after sources have been copied and setup files generated.
-Note that it uses airflow git and pulls the latest version of tags available in Airflow,
-so you need to enter Breeze with
-`--mount-all-local-sources flag`
-
-1) Run update documentation (version suffix might be empty):
-
-```shell script
-./dev/provider_packages/prepare_provider_packages.py build-provider-packages \
- --version-suffix <SUFFIX> \
- <PACKAGE>
-```
-
-In case version being prepared is already tagged in the repo documentation preparation returns immediately
-and prints error. You can prepare the error regardless and build the packages even if the tag exists, by
-specifying ``--version-suffix`` (for example ``--version-suffix dev``).
-
-By default, you prepare ``both`` packages, but you can add ``--package-format`` argument and specify
-``wheel``, ``sdist`` to build only one of them.
diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 319f4447fd..f5751f82f5 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -21,7 +21,6 @@
import collections
import difflib
import glob
-import importlib
import json
import logging
import os
@@ -31,7 +30,6 @@ import subprocess
import sys
import tempfile
import textwrap
-import warnings
from contextlib import contextmanager
from copy import deepcopy
from datetime import datetime, timedelta
@@ -40,7 +38,7 @@ from functools import lru_cache
from os.path import dirname, relpath
from pathlib import Path
from shutil import copyfile
-from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Type, Union
+from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Set, Tuple, Union
import jsonschema
import rich_click as click
@@ -49,8 +47,7 @@ from packaging.version import Version
from rich.console import Console
from rich.progress import Progress
from rich.syntax import Syntax
-
-from airflow.utils.yaml import safe_load
+from yaml import safe_load
ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
@@ -88,23 +85,22 @@ HTTPS_REMOTE = "apache-https-for-providers"
HEAD_OF_HTTPS_REMOTE = f"{HTTPS_REMOTE}/main"
MY_DIR_PATH = Path(__file__).parent
-SOURCE_DIR_PATH = MY_DIR_PATH.parent.parent
-AIRFLOW_PATH = SOURCE_DIR_PATH / "airflow"
-DIST_PATH = SOURCE_DIR_PATH / "dist"
+AIRFLOW_SOURCES_ROOT_PATH = MY_DIR_PATH.parents[1]
+AIRFLOW_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow"
+DIST_PATH = AIRFLOW_SOURCES_ROOT_PATH / "dist"
PROVIDERS_PATH = AIRFLOW_PATH / "providers"
-DOCUMENTATION_PATH = SOURCE_DIR_PATH / "docs"
-TARGET_PROVIDER_PACKAGES_PATH = SOURCE_DIR_PATH / "provider_packages"
+DOCUMENTATION_PATH = AIRFLOW_SOURCES_ROOT_PATH / "docs"
+TARGET_PROVIDER_PACKAGES_PATH = AIRFLOW_SOURCES_ROOT_PATH / "provider_packages"
GENERATED_AIRFLOW_PATH = TARGET_PROVIDER_PACKAGES_PATH / "airflow"
GENERATED_PROVIDERS_PATH = GENERATED_AIRFLOW_PATH / "providers"
-PROVIDER_RUNTIME_DATA_SCHEMA_PATH = SOURCE_DIR_PATH / "airflow" / "provider_info.schema.json"
+PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow" / "provider_info.schema.json"
-sys.path.insert(0, str(SOURCE_DIR_PATH))
+sys.path.insert(0, str(AIRFLOW_SOURCES_ROOT_PATH))
# those imports need to come after the above sys.path.insert to make sure that Airflow
# sources are importable without having to add the airflow sources to the PYTHONPATH before
# running the script
-from dev.import_all_classes import import_all_classes # noqa # isort:skip
from setup import PROVIDERS_REQUIREMENTS # type: ignore[attr-defined] # isort:skip # noqa
from setup import PREINSTALLED_PROVIDERS # type: ignore[attr-defined] # isort:skip # noqa
@@ -118,6 +114,33 @@ PY3 = sys.version_info[0] == 3
console = Console(width=400, color_system="standard")
+class ProviderPackageDetails(NamedTuple):
+ provider_package_id: str
+ full_package_name: str
+ pypi_package_name: str
+ source_provider_package_path: str
+ documentation_provider_package_path: str
+ provider_description: str
+ versions: List[str]
+ excluded_python_versions: List[str]
+
+
+class EntityType(Enum):
+ Operators = "Operators"
+ Transfers = "Transfers"
+ Sensors = "Sensors"
+ Hooks = "Hooks"
+ Secrets = "Secrets"
+
+
+def get_provider_packages() -> List[str]:
+ """
+ Returns all provider packages.
+
+ """
+ return list(PROVIDERS_REQUIREMENTS.keys())
+
+
@click.group(context_settings={'help_option_names': ['-h', '--help'], 'max_content_width': 500})
def cli():
...
@@ -137,6 +160,15 @@ option_git_update = click.option(
help=f"If the git remote {HTTPS_REMOTE} already exists, don't try to update it",
)
+option_package_format = click.option(
+ '--package-format',
+ type=click.Choice(["wheel", "sdist", "both"]),
+ help='Format of packages.',
+ default="wheel",
+ show_default=True,
+ envvar='PACKAGE_FORMAT',
+)
+
option_version_suffix = click.option(
"--version-suffix",
metavar="suffix",
@@ -171,92 +203,21 @@ def with_group(title):
https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#grouping-log-lines
"""
if os.environ.get('GITHUB_ACTIONS', 'false') != "true":
- console.print("[info]" + "#" * 10 + ' ' + title + ' ' + "#" * 10 + "[/]")
+ console.print("#" * 10 + ' [bright_blue]' + title + '[/] ' + "#" * 10)
yield
return
- console.print(f"::group::{title}")
+ console.print(f"::group::[bright_blue]{title}[/]")
yield
console.print("::endgroup::")
-class EntityType(Enum):
- Operators = "Operators"
- Transfers = "Transfers"
- Sensors = "Sensors"
- Hooks = "Hooks"
- Secrets = "Secrets"
-
-
-class EntityTypeSummary(NamedTuple):
- entities: List[str]
- new_entities_table: str
- wrong_entities: List[Tuple[type, str]]
-
-
-class VerifiedEntities(NamedTuple):
- all_entities: Set[str]
- wrong_entities: List[Tuple[type, str]]
-
-
-class ProviderPackageDetails(NamedTuple):
- provider_package_id: str
- full_package_name: str
- pypi_package_name: str
- source_provider_package_path: str
- documentation_provider_package_path: str
- provider_description: str
- versions: List[str]
- excluded_python_versions: List[str]
-
-
-ENTITY_NAMES = {
- EntityType.Operators: "Operators",
- EntityType.Transfers: "Transfer Operators",
- EntityType.Sensors: "Sensors",
- EntityType.Hooks: "Hooks",
- EntityType.Secrets: "Secrets",
-}
-
-TOTALS: Dict[EntityType, int] = {
- EntityType.Operators: 0,
- EntityType.Hooks: 0,
- EntityType.Sensors: 0,
- EntityType.Transfers: 0,
- EntityType.Secrets: 0,
-}
-
-OPERATORS_PATTERN = r".*Operator$"
-SENSORS_PATTERN = r".*Sensor$"
-HOOKS_PATTERN = r".*Hook$"
-SECRETS_PATTERN = r".*Backend$"
-TRANSFERS_PATTERN = r".*To[A-Z0-9].*Operator$"
-WRONG_TRANSFERS_PATTERN = r".*Transfer$|.*TransferOperator$"
-
-ALL_PATTERNS = {
- OPERATORS_PATTERN,
- SENSORS_PATTERN,
- HOOKS_PATTERN,
- SECRETS_PATTERN,
- TRANSFERS_PATTERN,
- WRONG_TRANSFERS_PATTERN,
-}
-
-EXPECTED_SUFFIXES: Dict[EntityType, str] = {
- EntityType.Operators: "Operator",
- EntityType.Hooks: "Hook",
- EntityType.Sensors: "Sensor",
- EntityType.Secrets: "Backend",
- EntityType.Transfers: "Operator",
-}
-
-
def get_source_airflow_folder() -> str:
"""
Returns source directory for whole airflow (from the main airflow project).
:return: the folder path
"""
- return os.path.abspath(SOURCE_DIR_PATH)
+ return os.path.abspath(AIRFLOW_SOURCES_ROOT_PATH)
def get_source_providers_folder() -> str:
@@ -420,320 +381,6 @@ def get_package_extras(provider_package_id: str) -> Dict[str, List[str]]:
return extras_dict
-def get_provider_packages() -> List[str]:
- """
- Returns all provider packages.
-
- """
- return list(PROVIDERS_REQUIREMENTS.keys())
-
-
-def is_imported_from_same_module(the_class: str, imported_name: str) -> bool:
- """
- Is the class imported from another module?
-
- :param the_class: the class object itself
- :param imported_name: name of the imported class
- :return: true if the class was imported from another module
- """
- return ".".join(imported_name.split(".")[:-1]) == the_class.__module__
-
-
-def is_example_dag(imported_name: str) -> bool:
- """
- Is the class an example_dag class?
-
- :param imported_name: name where the class is imported from
- :return: true if it is an example_dags class
- """
- return ".example_dags." in imported_name
-
-
-def is_from_the_expected_base_package(the_class: Type, expected_package: str) -> bool:
- """
- Returns true if the class is from the package expected.
- :param the_class: the class object
- :param expected_package: package expected for the class
- :return:
- """
- return the_class.__module__.startswith(expected_package)
-
-
-def inherits_from(the_class: Type, expected_ancestor: Optional[Type] = None) -> bool:
- """
- Returns true if the class inherits (directly or indirectly) from the class specified.
- :param the_class: The class to check
- :param expected_ancestor: expected class to inherit from
- :return: true is the class inherits from the class expected
- """
- if expected_ancestor is None:
- return False
- import inspect
-
- mro = inspect.getmro(the_class)
- return the_class is not expected_ancestor and expected_ancestor in mro
-
-
-def is_class(the_class: Type) -> bool:
- """
- Returns true if the object passed is a class
- :param the_class: the class to pass
- :return: true if it is a class
- """
- import inspect
-
- return inspect.isclass(the_class)
-
-
-def package_name_matches(the_class: Type, expected_pattern: Optional[str] = None) -> bool:
- """
- In case expected_pattern is set, it checks if the package name matches the pattern.
- .
- :param the_class: imported class
- :param expected_pattern: the pattern that should match the package
- :return: true if the expected_pattern is None or the pattern matches the package
- """
- return expected_pattern is None or re.match(expected_pattern, the_class.__module__) is not None
-
-
-def find_all_entities(
- imported_classes: List[str],
- base_package: str,
- ancestor_match: Type,
- sub_package_pattern_match: str,
- expected_class_name_pattern: str,
- unexpected_class_name_patterns: Set[str],
- exclude_class_type: Optional[Type] = None,
- false_positive_class_names: Optional[Set[str]] = None,
-) -> VerifiedEntities:
- """
- Returns set of entities containing all subclasses in package specified.
-
- :param imported_classes: entities imported from providers
- :param base_package: base package name where to start looking for the entities
- :param sub_package_pattern_match: this string is expected to appear in the sub-package name
- :param ancestor_match: type of the object the method looks for
- :param expected_class_name_pattern: regexp of class name pattern to expect
- :param unexpected_class_name_patterns: set of regexp of class name pattern that are not expected
- :param exclude_class_type: exclude class of this type (Sensor are also Operators, so
- they should be excluded from the list)
- :param false_positive_class_names: set of class names that are wrongly recognised as badly named
- """
- found_entities: Set[str] = set()
- wrong_entities: List[Tuple[type, str]] = []
- for imported_name in imported_classes:
- module, class_name = imported_name.rsplit(".", maxsplit=1)
- the_class = getattr(importlib.import_module(module), class_name)
- if (
- is_class(the_class=the_class)
- and not is_example_dag(imported_name=imported_name)
- and is_from_the_expected_base_package(the_class=the_class, expected_package=base_package)
- and is_imported_from_same_module(the_class=the_class, imported_name=imported_name)
- and inherits_from(the_class=the_class, expected_ancestor=ancestor_match)
- and not inherits_from(the_class=the_class, expected_ancestor=exclude_class_type)
- and package_name_matches(the_class=the_class, expected_pattern=sub_package_pattern_match)
- ):
-
- if not false_positive_class_names or class_name not in false_positive_class_names:
- if not re.match(expected_class_name_pattern, class_name):
- wrong_entities.append(
- (
- the_class,
- f"The class name {class_name} is wrong. "
- f"It should match {expected_class_name_pattern}",
- )
- )
- continue
- if unexpected_class_name_patterns:
- for unexpected_class_name_pattern in unexpected_class_name_patterns:
- if re.match(unexpected_class_name_pattern, class_name):
- wrong_entities.append(
- (
- the_class,
- f"The class name {class_name} is wrong. "
- f"It should not match {unexpected_class_name_pattern}",
- )
- )
- continue
- found_entities.add(imported_name)
- return VerifiedEntities(all_entities=found_entities, wrong_entities=wrong_entities)
-
-
-def convert_classes_to_table(entity_type: EntityType, entities: List[str], full_package_name: str) -> str:
- """
- Converts new entities to a Markdown table.
-
- :param entity_type: entity type to convert to markup
- :param entities: list of entities
- :param full_package_name: name of the provider package
- :return: table of new classes
- """
- from tabulate import tabulate
-
- headers = [f"New Airflow 2.0 {entity_type.value.lower()}: `{full_package_name}` package"]
- table = [(get_class_code_link(full_package_name, class_name, "main"),) for class_name in entities]
- return tabulate(table, headers=headers, tablefmt="pipe")
-
-
-def get_details_about_classes(
- entity_type: EntityType,
- entities: Set[str],
- wrong_entities: List[Tuple[type, str]],
- full_package_name: str,
-) -> EntityTypeSummary:
- """
- Get details about entities.
-
- :param entity_type: type of entity (Operators, Hooks etc.)
- :param entities: set of entities found
- :param wrong_entities: wrong entities found for that type
- :param full_package_name: full package name
- :return:
- """
- all_entities = list(entities)
- all_entities.sort()
- TOTALS[entity_type] += len(all_entities)
- return EntityTypeSummary(
- entities=all_entities,
- new_entities_table=convert_classes_to_table(
- entity_type=entity_type,
- entities=all_entities,
- full_package_name=full_package_name,
- ),
- wrong_entities=wrong_entities,
- )
-
-
-def strip_package_from_class(base_package: str, class_name: str) -> str:
- """
- Strips base package name from the class (if it starts with the package name).
- """
- if class_name.startswith(base_package):
- return class_name[len(base_package) + 1 :]
- else:
- return class_name
-
-
-def convert_class_name_to_url(base_url: str, class_name) -> str:
- """
- Converts the class name to URL that the class can be reached
-
- :param base_url: base URL to use
- :param class_name: name of the class
- :return: URL to the class
- """
- return base_url + os.path.sep.join(class_name.split(".")[:-1]) + ".py"
-
-
-def get_class_code_link(base_package: str, class_name: str, git_tag: str) -> str:
- """
- Provides a Markdown link for the class passed as parameter.
-
- :param base_package: base package to strip from most names
- :param class_name: name of the class
- :param git_tag: tag to use for the URL link
- :return: URL to the class
- """
- url_prefix = f'https://github.com/apache/airflow/blob/{git_tag}/'
- return (
- f'[{strip_package_from_class(base_package, class_name)}]'
- f'({convert_class_name_to_url(url_prefix, class_name)})'
- )
-
-
-def print_wrong_naming(entity_type: EntityType, wrong_classes: List[Tuple[type, str]]):
- """
- Prints wrong entities of a given entity type if there are any
- :param entity_type: type of the class to print
- :param wrong_classes: list of wrong entities
- """
- if wrong_classes:
- console.print(f"\n[red]There are wrongly named entities of type {entity_type}:[/]\n")
- for wrong_entity_type, message in wrong_classes:
- console.print(f"{wrong_entity_type}: {message}")
-
-
-def get_package_class_summary(
- full_package_name: str, imported_classes: List[str]
-) -> Dict[EntityType, EntityTypeSummary]:
- """
- Gets summary of the package in the form of dictionary containing all types of entities
- :param full_package_name: full package name
- :param imported_classes: entities imported_from providers
- :return: dictionary of objects usable as context for JINJA2 templates - or None if there are some errors
- """
- from airflow.hooks.base import BaseHook
- from airflow.models.baseoperator import BaseOperator
- from airflow.secrets import BaseSecretsBackend
- from airflow.sensors.base import BaseSensorOperator
-
- all_verified_entities: Dict[EntityType, VerifiedEntities] = {
- EntityType.Operators: find_all_entities(
- imported_classes=imported_classes,
- base_package=full_package_name,
- sub_package_pattern_match=r".*\.operators\..*",
- ancestor_match=BaseOperator,
- expected_class_name_pattern=OPERATORS_PATTERN,
- unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN},
- exclude_class_type=BaseSensorOperator,
- false_positive_class_names={
- 'CloudVisionAddProductToProductSetOperator',
- 'CloudDataTransferServiceGCSToGCSOperator',
- 'CloudDataTransferServiceS3ToGCSOperator',
- 'BigQueryCreateDataTransferOperator',
- 'CloudTextToSpeechSynthesizeOperator',
- 'CloudSpeechToTextRecognizeSpeechOperator',
- },
- ),
- EntityType.Sensors: find_all_entities(
- imported_classes=imported_classes,
- base_package=full_package_name,
- sub_package_pattern_match=r".*\.sensors\..*",
- ancestor_match=BaseSensorOperator,
- expected_class_name_pattern=SENSORS_PATTERN,
- unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN, SENSORS_PATTERN},
- ),
- EntityType.Hooks: find_all_entities(
- imported_classes=imported_classes,
- base_package=full_package_name,
- sub_package_pattern_match=r".*\.hooks\..*",
- ancestor_match=BaseHook,
- expected_class_name_pattern=HOOKS_PATTERN,
- unexpected_class_name_patterns=ALL_PATTERNS - {HOOKS_PATTERN},
- ),
- EntityType.Secrets: find_all_entities(
- imported_classes=imported_classes,
- sub_package_pattern_match=r".*\.secrets\..*",
- base_package=full_package_name,
- ancestor_match=BaseSecretsBackend,
- expected_class_name_pattern=SECRETS_PATTERN,
- unexpected_class_name_patterns=ALL_PATTERNS - {SECRETS_PATTERN},
- ),
- EntityType.Transfers: find_all_entities(
- imported_classes=imported_classes,
- base_package=full_package_name,
- sub_package_pattern_match=r".*\.transfers\..*",
- ancestor_match=BaseOperator,
- expected_class_name_pattern=TRANSFERS_PATTERN,
- unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN, TRANSFERS_PATTERN},
- ),
- }
- for entity in EntityType:
- print_wrong_naming(entity, all_verified_entities[entity].wrong_entities)
-
- entities_summary: Dict[EntityType, EntityTypeSummary] = {}
-
- for entity_type in EntityType:
- entities_summary[entity_type] = get_details_about_classes(
- entity_type,
- all_verified_entities[entity_type].all_entities,
- all_verified_entities[entity_type].wrong_entities,
- full_package_name,
- )
-
- return entities_summary
-
-
def render_template(
template_name: str,
context: Dict[str, Any],
@@ -1182,49 +829,6 @@ def get_additional_package_info(provider_package_path: str) -> str:
return ""
-def is_camel_case_with_acronyms(s: str):
- """
- Checks if the string passed is Camel Case (with capitalised acronyms allowed).
- :param s: string to check
- :return: true if the name looks cool as Class name.
- """
- return s != s.lower() and s != s.upper() and "_" not in s and s[0].upper() == s[0]
-
-
-def check_if_classes_are_properly_named(
- entity_summary: Dict[EntityType, EntityTypeSummary]
-) -> Tuple[int, int]:
- """
- Check if all entities in the dictionary are named properly. It prints names at the output
- and returns the status of class names.
-
- :param entity_summary: dictionary of class names to check, grouped by types.
- :return: Tuple of 2 ints = total number of entities and number of badly named entities
- """
- total_class_number = 0
- badly_named_class_number = 0
- for entity_type, class_suffix in EXPECTED_SUFFIXES.items():
- for class_full_name in entity_summary[entity_type].entities:
- _, class_name = class_full_name.rsplit(".", maxsplit=1)
- error_encountered = False
- if not is_camel_case_with_acronyms(class_name):
- console.print(
- f"[red]The class {class_full_name} is wrongly named. The "
- f"class name should be CamelCaseWithACRONYMS ![/]"
- )
- error_encountered = True
- if not class_name.endswith(class_suffix):
- console.print(
- f"[red]The class {class_full_name} is wrongly named. It is one of the {entity_type.value}"
- f" so it should end with {class_suffix}[/]"
- )
- error_encountered = True
- total_class_number += 1
- if error_encountered:
- badly_named_class_number += 1
- return total_class_number, badly_named_class_number
-
-
def get_package_pip_name(provider_package_id: str):
return f"apache-airflow-providers-{provider_package_id.replace('.', '-')}"
@@ -1707,7 +1311,7 @@ def update_commits_rst(
def black_mode():
from black import Mode, parse_pyproject_toml, target_version_option_callback
- config = parse_pyproject_toml(os.path.join(SOURCE_DIR_PATH, "pyproject.toml"))
+ config = parse_pyproject_toml(os.path.join(AIRFLOW_SOURCES_ROOT_PATH, "pyproject.toml"))
target_versions = set(
target_version_option_callback(None, None, tuple(config.get('target_version', ()))),
@@ -1909,12 +1513,10 @@ def generate_setup_files(
current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose)
if tag_exists_for_version(provider_package_id, current_tag, verbose):
console.print(f"[yellow]The tag {current_tag} exists. Not preparing the package.[/]")
- # Returns 1 in case of skipped package
- sys.exit(1)
+ sys.exit(64)
if update_setup_files(provider_package_id, version_suffix):
console.print(f"[green]Generated regular package setup files for {provider_package_id}[/]")
else:
- # Returns 64 in case of skipped package
sys.exit(64)
@@ -1956,12 +1558,7 @@ def verify_setup_cfg_prepared(provider_package):
@cli.command()
-@click.option(
- '--package-format',
- type=click.Choice(['sdist', 'wheel', 'both']),
- default='wheel',
- help='Optional format - only used in case of building packages (default: wheel)',
-)
+@option_package_format
@option_git_update
@option_version_suffix
@argument_package_id
@@ -2026,267 +1623,6 @@ def build_provider_packages(
shutil.rmtree(tmp_dist_dir, ignore_errors=True)
-def verify_provider_classes_for_single_provider(imported_classes: List[str], provider_package_id: str):
- """Verify naming of provider classes for single provider."""
- full_package_name = f"airflow.providers.{provider_package_id}"
- entity_summaries = get_package_class_summary(full_package_name, imported_classes)
- total, bad = check_if_classes_are_properly_named(entity_summaries)
- bad += sum(len(entity_summary.wrong_entities) for entity_summary in entity_summaries.values())
- if bad != 0:
- console.print()
- console.print(f"[red]There are {bad} errors of {total} entities for {provider_package_id}[/]")
- console.print()
- return total, bad
-
-
-def summarise_total_vs_bad_and_warnings(total: int, bad: int, warns: List[warnings.WarningMessage]) -> bool:
- """Summarises Bad/Good class names for providers and warnings"""
- raise_error = False
- if bad == 0:
- console.print()
- console.print(f"[green]OK: All {total} entities are properly named[/]")
- console.print()
- console.print("Totals:")
- console.print()
- for entity in EntityType:
- console.print(f"{entity.value}: {TOTALS[entity]}")
- console.print()
- else:
- console.print()
- console.print(
- f"[red]ERROR! There are in total: {bad} entities badly named out of {total} entities[/]"
- )
- console.print()
- raise_error = True
- if warns:
- if os.environ.get('GITHUB_ACTIONS'):
- # Ends group in GitHub Actions so that the errors are immediately visible in CI log
- console.print("::endgroup::")
- console.print()
- console.print("[red]Unknown warnings generated:[/]")
- console.print()
- for w in warns:
- one_line_message = str(w.message).replace('\n', ' ')
- console.print(f"{w.filename}:{w.lineno}:[yellow]{one_line_message}[/]")
- console.print()
- console.print(f"[red]ERROR! There were {len(warns)} warnings generated during the import[/]")
- console.print()
- console.print("[yellow]Ideally, fix it, so that no warnings are generated during import.[/]")
- console.print("[yellow]There are three cases that are legitimate deprecation warnings though:[/]")
- console.print("[yellow] 1) when you deprecate whole module or class and replace it in provider[/]")
- console.print("[yellow] 2) when 3rd-party module generates Deprecation and you cannot upgrade it[/]")
- console.print(
- "[yellow] 3) when many 3rd-party module generates same Deprecation warning that "
- "comes from another common library[/]"
- )
- console.print()
- console.print(
- "[yellow]In case 1), add the deprecation message to "
- "the KNOWN_DEPRECATED_DIRECT_IMPORTS in prepare_provider_packages.py[/]"
- )
- console.print(
- "[yellow]In case 2), add the deprecation message together with module it generates to "
- "the KNOWN_DEPRECATED_MESSAGES in prepare_provider_packages.py[/]"
- )
- console.print(
- "[yellow]In case 3), add the deprecation message to "
- "the KNOWN_COMMON_DEPRECATED_MESSAGES in prepare_provider_packages.py[/]"
- )
- console.print()
- raise_error = True
- else:
- console.print()
- console.print("[green]OK: No warnings generated[/]")
- console.print()
-
- if raise_error:
- console.print("[red]Please fix the problems listed above [/]")
- return False
- return True
-
-
-# The set of known deprecation messages that we know about.
-# It contains tuples of "message" and the module that generates the warning - so when the
-# Same warning is generated by different module, it is not treated as "known" warning.
-KNOWN_DEPRECATED_MESSAGES: Set[Tuple[str, str]] = {
- (
- 'This version of Apache Beam has not been sufficiently tested on Python 3.9. '
- 'You may encounter bugs or missing features.',
- "apache_beam",
- ),
- (
- 'This version of Apache Beam has not been sufficiently tested on Python 3.10. '
- 'You may encounter bugs or missing features.',
- "apache_beam",
- ),
- (
- "Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since"
- " Python 3.3,and in 3.9 it will stop working",
- "apache_beam",
- ),
- (
- 'pyarrow.HadoopFileSystem is deprecated as of 2.0.0, please use pyarrow.fs.HadoopFileSystem instead.',
- "papermill",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that "
- "adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
- "apache_beam",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that "
- "adheres to: 'pyarrow<5.1.0,>=5.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- ("dns.hash module will be removed in future versions. Please use hashlib instead.", "dns"),
- ("PKCS#7 support in pyOpenSSL is deprecated. You should use the APIs in cryptography.", "eventlet"),
- ("PKCS#12 support in pyOpenSSL is deprecated. You should use the APIs in cryptography.", "eventlet"),
- (
- "the imp module is deprecated in favour of importlib; see the module's documentation"
- " for alternative uses",
- "hdfs",
- ),
- ("This operator is deprecated. Please use `airflow.providers.tableau.operators.tableau`.", "salesforce"),
- (
- "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that"
- " adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- (
- "You have an incompatible version of 'pyarrow' installed (6.0.1), please install a version that"
- " adheres to: 'pyarrow<5.1.0,>=5.0.0; extra == \"pandas\"'",
- "snowflake",
- ),
- ("SelectableGroups dict interface is deprecated. Use select.", "kombu"),
- ("The module cloudant is now deprecated. The replacement is ibmcloudant.", "cloudant"),
- ("This module is deprecated. Please use `airflow.operators.empty`.", "dbt"),
- ("This module is deprecated. Please use `airflow.operators.empty`.", "jdbc"),
- ("This module is deprecated. Please use `airflow.operators.empty`.", "azure"),
- ("This module is deprecated. Please use `airflow.operators.empty`.", "qubole"),
- ("This module is deprecated. Please use `airflow.operators.empty`.", "winrm"),
- ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "dbt"),
- ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "jdbc"),
- ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "azure"),
- ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "qubole"),
- ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "winrm"),
-}
-
-KNOWN_COMMON_DEPRECATED_MESSAGES: Set[str] = {
- "distutils Version classes are deprecated. Use packaging.version instead."
-}
-
-# The set of warning messages generated by direct importing of some deprecated modules. We should only
-# ignore those messages when the warnings are generated directly by importlib - which means that
-# we imported it directly during module walk by the importlib library
-KNOWN_DEPRECATED_DIRECT_IMPORTS: Set[str] = {
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.batch`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.container_instance`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.container_registry`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.container_volume`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.cosmos`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.data_factory`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.data_lake`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.fileshare`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.operators.batch`.",
- "This module is deprecated. "
- "Please use `airflow.providers.microsoft.azure.operators.container_instances`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.operators.cosmos`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.secrets.key_vault`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.sensors.cosmos`.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.dynamodb`.",
- "This module is deprecated. Please use `airflow.providers.microsoft.azure.transfers.local_to_wasb`.",
- "This module is deprecated. Please use `airflow.providers.tableau.operators.tableau_refresh_workbook`.",
- "This module is deprecated. Please use `airflow.providers.tableau.sensors.tableau_job_status`.",
- "This module is deprecated. Please use `airflow.providers.tableau.hooks.tableau`.",
- "This module is deprecated. Please use `kubernetes.client.models.V1Volume`.",
- "This module is deprecated. Please use `kubernetes.client.models.V1VolumeMount`.",
- (
- "This module is deprecated. Please use `kubernetes.client.models.V1ResourceRequirements`"
- " and `kubernetes.client.models.V1ContainerPort`."
- ),
- "This module is deprecated. Please use `kubernetes.client.models.V1EnvVar`.",
- 'numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header,'
- ' got 216 from PyObject',
- "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.step_function`.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.step_function`.",
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.ec2`.',
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.ec2`.',
- "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.s3`.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.s3`.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.dms`.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.dms`.",
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.emr`.',
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.emr`.',
- "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.redshift_cluster` "
- "or `airflow.providers.amazon.aws.hooks.redshift_sql` as appropriate.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.redshift_sql` "
- "or `airflow.providers.amazon.aws.operators.redshift_cluster` as appropriate.",
- "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.redshift_cluster`.",
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.sagemaker`.',
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.sagemaker`.',
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.emr`.',
- 'This module is deprecated. Please use `airflow.providers.opsgenie.hooks.opsgenie`.',
- 'This module is deprecated. Please use `airflow.providers.opsgenie.operators.opsgenie`.',
- 'This module is deprecated. Please use `airflow.hooks.redshift_sql` '
- 'or `airflow.hooks.redshift_cluster` as appropriate.',
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.redshift_sql` or '
- '`airflow.providers.amazon.aws.operators.redshift_cluster` as appropriate.',
- 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.redshift_cluster`.',
- "This module is deprecated. Please use airflow.providers.amazon.aws.transfers.sql_to_s3`.",
-}
-
-
-def filter_known_warnings(warn: warnings.WarningMessage) -> bool:
- msg_string = str(warn.message).replace("\n", " ")
- for m in KNOWN_DEPRECATED_MESSAGES:
- expected_package_string = "/" + m[1] + "/"
- if msg_string == m[0] and warn.filename.find(expected_package_string) != -1:
- return False
- return True
-
-
-def filter_direct_importlib_warning(warn: warnings.WarningMessage) -> bool:
- msg_string = str(warn.message).replace("\n", " ")
- for m in KNOWN_DEPRECATED_DIRECT_IMPORTS:
- if msg_string == m and warn.filename.find("/importlib/") != -1:
- return False
- return True
-
-
-def filter_known_common_deprecated_messages(warn: warnings.WarningMessage) -> bool:
- msg_string = str(warn.message).replace("\n", " ")
- for m in KNOWN_COMMON_DEPRECATED_MESSAGES:
- if msg_string == m:
- return False
- return True
-
-
-@cli.command()
-def verify_provider_classes():
- """Verifies names for all provider classes."""
- with with_group("Verifies names for all provider classes"):
- provider_ids = get_all_providers()
- imported_classes, warns = import_all_classes(
- provider_ids=provider_ids,
- print_imports=True,
- paths=[str(PROVIDERS_PATH)],
- prefix="airflow.providers.",
- )
- total = 0
- bad = 0
- for provider_package_id in provider_ids:
- inc_total, inc_bad = verify_provider_classes_for_single_provider(
- imported_classes, provider_package_id
- )
- total += inc_total
- bad += inc_bad
- warns = list(filter(filter_known_warnings, warns))
- warns = list(filter(filter_direct_importlib_warning, warns))
- warns = list(filter(filter_known_common_deprecated_messages, warns))
- if not summarise_total_vs_bad_and_warnings(total, bad, warns):
- sys.exit(1)
-
-
def find_insertion_index_for_version(content: List[str], version: str) -> Tuple[int, bool]:
"""
Finds insertion index for the specified version from the .rst changelog content.
diff --git a/docs/apache-airflow-providers/howto/create-update-providers.rst b/docs/apache-airflow-providers/howto/create-update-providers.rst
index 6cb5886e84..c81b99a73c 100644
--- a/docs/apache-airflow-providers/howto/create-update-providers.rst
+++ b/docs/apache-airflow-providers/howto/create-update-providers.rst
@@ -99,26 +99,6 @@ breeze and I'll run unit tests for my Hook.
root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers/<NEW_PROVIDER>/hook/<NEW_PROVIDER>.py
-Update Airflow validation tests
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-There are some tests that Airflow performs to ensure consistency that is related to the providers.
-
- .. code-block:: bash
-
- airflow/scripts/in_container/
- └── run_install_and_test_provider_packages.sh
- tests/core/
- └── test_providers_manager.py
-
-Change expected number of providers, hooks and connections if needed in ``run_install_and_test_provider_packages.sh`` file.
-
-Add your provider information in the following variables in ``test_providers_manager.py``:
-
-- add your provider to ``ALL_PROVIDERS`` list;
-- add your provider into ``CONNECTIONS_LIST`` if your provider create a new connection type.
-
-
Integration tests
^^^^^^^^^^^^^^^^^
diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg
index 8ff7a50baa..2271337601 100644
--- a/images/breeze/output-build-image.svg
+++ b/images/breeze/output-build-image.svg
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-662778973-terminal-wrapper span {
+ .rich-svg-1034040239-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-662778973-terminal-wrapper a {
+ .rich-svg-1034040239-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-662778973-terminal-body .blink {
- animation: rich-svg-662778973-blinker 1s infinite;
+ .rich-svg-1034040239-terminal-body .blink {
+ animation: rich-svg-1034040239-blinker 1s infinite;
}
- @keyframes rich-svg-662778973-blinker {
+ @keyframes rich-svg-1034040239-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-662778973-terminal-wrapper {
+ .rich-svg-1034040239-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-662778973-terminal {
+ .rich-svg-1034040239-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-662778973-terminal:after {
+ .rich-svg-1034040239-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-662778973-terminal-header {
+ .rich-svg-1034040239-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-662778973-terminal-title-tab {
+ .rich-svg-1034040239-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-662778973-terminal-traffic-lights {
+ .rich-svg-1034040239-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-662778973-terminal-body {
+ .rich-svg-1034040239-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-662778973-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-662778973-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-662778973-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-662778973-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-662778973-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-662778973-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-662778973-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-1034040239-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1034040239-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1034040239-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1034040239-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1034040239-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1034040239-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1034040239-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-662778973-terminal-wrapper">
- <div class="rich-svg-662778973-terminal">
- <div class="rich-svg-662778973-terminal-header">
- <svg class="rich-svg-662778973-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-1034040239-terminal-wrapper">
+ <div class="rich-svg-1034040239-terminal">
+ <div class="rich-svg-1034040239-terminal-header">
+ <svg class="rich-svg-1034040239-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-662778973-terminal-title-tab">Command: build-image</div>
+ <div class="rich-svg-1034040239-terminal-title-tab">Command: build-image</div>
</div>
- <div class="rich-svg-662778973-terminal-body">
+ <div class="rich-svg-1034040239-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze build-image [OPTIONS] </span></div>
<div><span class="r2"> </span></div>
@@ -125,11 +125,11 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--upgrade-to-newer-dependencies</span><span class="r1"> </span><span class="r6">-u</span><span class="r1"> </span><span class="r1">When set, upgrade all PIP packages to latest. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--docker-cache</span><span class="r1"> </span><span class="r6">-c</span><span class="r1"> </span><span class="r1">Cache option for image used during the build.</span><span class="r1"> </span><span class="r7">(pulled | local | disabled)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: pulled] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--build-multiple-images</span><span class="r1"> </span><span class="r1">Run the operation sequentially on all or selected subset of Python versions. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg
index 47346d92ee..794e4438c4 100644
--- a/images/breeze/output-build-prod-image.svg
+++ b/images/breeze/output-build-prod-image.svg
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-787753248-terminal-wrapper span {
+ .rich-svg-2806393034-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-787753248-terminal-wrapper a {
+ .rich-svg-2806393034-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-787753248-terminal-body .blink {
- animation: rich-svg-787753248-blinker 1s infinite;
+ .rich-svg-2806393034-terminal-body .blink {
+ animation: rich-svg-2806393034-blinker 1s infinite;
}
- @keyframes rich-svg-787753248-blinker {
+ @keyframes rich-svg-2806393034-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-787753248-terminal-wrapper {
+ .rich-svg-2806393034-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-787753248-terminal {
+ .rich-svg-2806393034-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-787753248-terminal:after {
+ .rich-svg-2806393034-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-787753248-terminal-header {
+ .rich-svg-2806393034-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-787753248-terminal-title-tab {
+ .rich-svg-2806393034-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-787753248-terminal-traffic-lights {
+ .rich-svg-2806393034-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-787753248-terminal-body {
+ .rich-svg-2806393034-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-787753248-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-787753248-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-787753248-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-787753248-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-787753248-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-787753248-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-787753248-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-2806393034-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-2806393034-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-2806393034-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2806393034-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-2806393034-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2806393034-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2806393034-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-787753248-terminal-wrapper">
- <div class="rich-svg-787753248-terminal">
- <div class="rich-svg-787753248-terminal-header">
- <svg class="rich-svg-787753248-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-2806393034-terminal-wrapper">
+ <div class="rich-svg-2806393034-terminal">
+ <div class="rich-svg-2806393034-terminal-header">
+ <svg class="rich-svg-2806393034-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-787753248-terminal-title-tab">Command: build-prod-image</div>
+ <div class="rich-svg-2806393034-terminal-title-tab">Command: build-prod-image</div>
</div>
- <div class="rich-svg-787753248-terminal-body">
+ <div class="rich-svg-2806393034-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze build-prod-image [OPTIONS] </span></div>
<div><span class="r2"> </span></div>
@@ -126,7 +126,7 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--install-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Install version of Airflow from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--upgrade-to-newer-dependencies</span><span class="r1"> </span><span class="r6">-u</span><span class="r1"> </span><span class="r1">When set, upgrade all PIP packages to latest. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--docker-cache</span><span class="r1"> </span><span class="r6">-c</span><span class="r1"> </span><span class="r1">Cache option for image used during the build.</span><span class="r1"> </span><span class="r7">(pulled | local | disabled)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: pulled] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg
index 59d7de7f66..e5e69be605 100644
--- a/images/breeze/output-commands.svg
+++ b/images/breeze/output-commands.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1880" viewBox="0 0 1720.0 1880"
+<svg width="1720.0" height="2144" viewBox="0 0 1720.0 2144"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-1735457778-terminal-wrapper span {
+ .rich-svg-1738564615-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-1735457778-terminal-wrapper a {
+ .rich-svg-1738564615-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-1735457778-terminal-body .blink {
- animation: rich-svg-1735457778-blinker 1s infinite;
+ .rich-svg-1738564615-terminal-body .blink {
+ animation: rich-svg-1738564615-blinker 1s infinite;
}
- @keyframes rich-svg-1735457778-blinker {
+ @keyframes rich-svg-1738564615-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-1735457778-terminal-wrapper {
+ .rich-svg-1738564615-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-1735457778-terminal {
+ .rich-svg-1738564615-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-1735457778-terminal:after {
+ .rich-svg-1738564615-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-1735457778-terminal-header {
+ .rich-svg-1738564615-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-1735457778-terminal-title-tab {
+ .rich-svg-1738564615-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-1735457778-terminal-traffic-lights {
+ .rich-svg-1738564615-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-1735457778-terminal-body {
+ .rich-svg-1738564615-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-1735457778-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-1735457778-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1735457778-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1735457778-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1735457778-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1735457778-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1735457778-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-1738564615-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1738564615-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1738564615-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1738564615-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1738564615-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1738564615-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1738564615-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-1735457778-terminal-wrapper">
- <div class="rich-svg-1735457778-terminal">
- <div class="rich-svg-1735457778-terminal-header">
- <svg class="rich-svg-1735457778-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-1738564615-terminal-wrapper">
+ <div class="rich-svg-1738564615-terminal">
+ <div class="rich-svg-1738564615-terminal-header">
+ <svg class="rich-svg-1738564615-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-1735457778-terminal-title-tab">Breeze commands</div>
+ <div class="rich-svg-1738564615-terminal-title-tab">Breeze commands</div>
</div>
- <div class="rich-svg-1735457778-terminal-body">
+ <div class="rich-svg-1738564615-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze [OPTIONS] COMMAND [ARGS]... </span></div>
<div><span class="r2"> </span></div>
@@ -122,20 +122,31 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r4">[default: sqlite]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Use (reinstall at entry) Airflow version from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">all) </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--forward-credentials</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Forward local credentials to container when running. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--db-reset</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Reset DB when entering the container. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Advanced flags for the default (shell) command ─────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Choose scope of local sources should be mounted (default = selected).</span><span class="r1"> </span><span class="r7">(selected | all | none)</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: selected] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `sdist` if Airflow should be removed, installed from wheel packages or sdist packages </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> available in dist folder respectively. Important! Using it Implies </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `none`. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(none | wheel | sdist | <airflow_version>) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-extras</span><span class="r1"> Airflow extras to install when </span><span class="r5">--use-airflow-version</span><span class="r1"> is used </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-packages-from-dist</span><span class="r1"> Install all found packages (</span><span class="r5">--package-format</span><span class="r1"> determines type) from 'dist' folder when </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> entering breeze. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--package-format</span><span class="r1"> </span><span class="r1">Format of packages that should be installed from dist.</span><span class="r1"> </span><span class="r7">(wheel | sdist)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: wheel] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r1">Choose scope of local sources should be mounted (default = selected). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(selected | all | none) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: selected] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
@@ -178,6 +189,7 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> verify-prod-image </span><span class="r1"> </span><span class="r1">Verify Production image. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Release management ─────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> verify-provider-packages </span><span class="r1"> </span><span class="r1">Verifies if all provider code is following expectations for providers. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> prepare-provider-documentation </span><span class="r1"> </span><span class="r1">Prepare CHANGELOG, README and COMMITS information for providers. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> prepare-provider-packages </span><span class="r1"> </span><span class="r1">Prepare sdist/whl packages of Airflow Providers. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> prepare-airflow-package </span><span class="r1"> </span><span class="r1">Prepare sdist/whl package of Airflow. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-find-newer-dependencies.svg b/images/breeze/output-find-newer-dependencies.svg
index a1e69ef7dc..1791cc795f 100644
--- a/images/breeze/output-find-newer-dependencies.svg
+++ b/images/breeze/output-find-newer-dependencies.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="714" viewBox="0 0 1720.0 714"
+<svg width="1720.0" height="758" viewBox="0 0 1720.0 758"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-3339896715-terminal-wrapper span {
+ .rich-svg-1486174241-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-3339896715-terminal-wrapper a {
+ .rich-svg-1486174241-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-3339896715-terminal-body .blink {
- animation: rich-svg-3339896715-blinker 1s infinite;
+ .rich-svg-1486174241-terminal-body .blink {
+ animation: rich-svg-1486174241-blinker 1s infinite;
}
- @keyframes rich-svg-3339896715-blinker {
+ @keyframes rich-svg-1486174241-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-3339896715-terminal-wrapper {
+ .rich-svg-1486174241-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-3339896715-terminal {
+ .rich-svg-1486174241-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-3339896715-terminal:after {
+ .rich-svg-1486174241-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-3339896715-terminal-header {
+ .rich-svg-1486174241-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-3339896715-terminal-title-tab {
+ .rich-svg-1486174241-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-3339896715-terminal-traffic-lights {
+ .rich-svg-1486174241-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-3339896715-terminal-body {
+ .rich-svg-1486174241-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-3339896715-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-3339896715-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3339896715-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3339896715-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3339896715-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3339896715-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3339896715-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-1486174241-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1486174241-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1486174241-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1486174241-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1486174241-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1486174241-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1486174241-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-3339896715-terminal-wrapper">
- <div class="rich-svg-3339896715-terminal">
- <div class="rich-svg-3339896715-terminal-header">
- <svg class="rich-svg-3339896715-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-1486174241-terminal-wrapper">
+ <div class="rich-svg-1486174241-terminal">
+ <div class="rich-svg-1486174241-terminal-header">
+ <svg class="rich-svg-1486174241-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-3339896715-terminal-title-tab">Command: find-newer-dependencies</div>
+ <div class="rich-svg-1486174241-terminal-title-tab">Command: find-newer-dependencies</div>
</div>
- <div class="rich-svg-3339896715-terminal-body">
+ <div class="rich-svg-1486174241-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze find-newer-dependencies [OPTIONS] </span></div>
<div><span class="r2"> </span></div>
@@ -124,12 +124,14 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--timezone</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Timezone to use during the check</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--constraints-branch</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Constraint branch to use to find newer dependencies</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--updated-on-or-after</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Date when the release was updated after</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--max-age</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Max age of the last release (used if no updated-on-or-after if specified)</span><span class="r1"> </span><span class="r7">(INTEGER)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-constraints-reference</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> Constraint reference to use. Useful with </span><span class="r5">--use-airflow-version</span><span class="r1"> parameter to </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> specify constraints for the installed version and to find newer dependencies </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(TEXT) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
diff --git a/images/breeze/output-generate-constraints.svg b/images/breeze/output-generate-constraints.svg
index e30b53ac33..a725cca3e7 100644
--- a/images/breeze/output-generate-constraints.svg
+++ b/images/breeze/output-generate-constraints.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
+<svg width="1720.0" height="978" viewBox="0 0 1720.0 978"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-1608918125-terminal-wrapper span {
+ .rich-svg-2279249032-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-1608918125-terminal-wrapper a {
+ .rich-svg-2279249032-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-1608918125-terminal-body .blink {
- animation: rich-svg-1608918125-blinker 1s infinite;
+ .rich-svg-2279249032-terminal-body .blink {
+ animation: rich-svg-2279249032-blinker 1s infinite;
}
- @keyframes rich-svg-1608918125-blinker {
+ @keyframes rich-svg-2279249032-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-1608918125-terminal-wrapper {
+ .rich-svg-2279249032-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-1608918125-terminal {
+ .rich-svg-2279249032-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-1608918125-terminal:after {
+ .rich-svg-2279249032-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-1608918125-terminal-header {
+ .rich-svg-2279249032-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-1608918125-terminal-title-tab {
+ .rich-svg-2279249032-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-1608918125-terminal-traffic-lights {
+ .rich-svg-2279249032-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-1608918125-terminal-body {
+ .rich-svg-2279249032-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-1608918125-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-2279249032-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-2279249032-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-2279249032-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2279249032-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-2279249032-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2279249032-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2279249032-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-1608918125-terminal-wrapper">
- <div class="rich-svg-1608918125-terminal">
- <div class="rich-svg-1608918125-terminal-header">
- <svg class="rich-svg-1608918125-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-2279249032-terminal-wrapper">
+ <div class="rich-svg-2279249032-terminal">
+ <div class="rich-svg-2279249032-terminal-header">
+ <svg class="rich-svg-2279249032-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-1608918125-terminal-title-tab">Command: generate-constraints</div>
+ <div class="rich-svg-2279249032-terminal-title-tab">Command: generate-constraints</div>
</div>
- <div class="rich-svg-1608918125-terminal-body">
+ <div class="rich-svg-2279249032-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze generate-constraints [OPTIONS] </span></div>
<div><span class="r2"> </span></div>
@@ -127,6 +127,7 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--generate-constraints-mode</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Mode of generating constraints</span><span class="r1"> </span><span class="r7">(source-providers | pypi-providers | no-providers)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: source-providers] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debug</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Drop user in shell instead of running the command. Useful for debugging. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--run-in-parallel</span><span class="r1"> </span><span class="r1">Run the operation in parallel on all or selected subset of Python versions. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-prepare-airflow-package.svg b/images/breeze/output-prepare-airflow-package.svg
index d1e487553e..8a2413fca8 100644
--- a/images/breeze/output-prepare-airflow-package.svg
+++ b/images/breeze/output-prepare-airflow-package.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="692" viewBox="0 0 1720.0 692"
+<svg width="1720.0" height="714" viewBox="0 0 1720.0 714"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-1135701573-terminal-wrapper span {
+ .rich-svg-3263880800-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-1135701573-terminal-wrapper a {
+ .rich-svg-3263880800-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-1135701573-terminal-body .blink {
- animation: rich-svg-1135701573-blinker 1s infinite;
+ .rich-svg-3263880800-terminal-body .blink {
+ animation: rich-svg-3263880800-blinker 1s infinite;
}
- @keyframes rich-svg-1135701573-blinker {
+ @keyframes rich-svg-3263880800-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-1135701573-terminal-wrapper {
+ .rich-svg-3263880800-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-1135701573-terminal {
+ .rich-svg-3263880800-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-1135701573-terminal:after {
+ .rich-svg-3263880800-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-1135701573-terminal-header {
+ .rich-svg-3263880800-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-1135701573-terminal-title-tab {
+ .rich-svg-3263880800-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-1135701573-terminal-traffic-lights {
+ .rich-svg-3263880800-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-1135701573-terminal-body {
+ .rich-svg-3263880800-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-1135701573-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-1135701573-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1135701573-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1135701573-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1135701573-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1135701573-terminal-body .r6 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
-.rich-svg-1135701573-terminal-body .r7 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+ .rich-svg-3263880800-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-3263880800-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3263880800-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3263880800-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3263880800-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3263880800-terminal-body .r6 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+.rich-svg-3263880800-terminal-body .r7 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-1135701573-terminal-wrapper">
- <div class="rich-svg-1135701573-terminal">
- <div class="rich-svg-1135701573-terminal-header">
- <svg class="rich-svg-1135701573-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-3263880800-terminal-wrapper">
+ <div class="rich-svg-3263880800-terminal">
+ <div class="rich-svg-3263880800-terminal-header">
+ <svg class="rich-svg-3263880800-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-1135701573-terminal-title-tab">Command: prepare-airflow-package</div>
+ <div class="rich-svg-3263880800-terminal-title-tab">Command: prepare-airflow-package</div>
</div>
- <div class="rich-svg-1135701573-terminal-body">
+ <div class="rich-svg-3263880800-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze prepare-airflow-package [OPTIONS] </span></div>
<div><span class="r2"> </span></div>
@@ -128,6 +128,7 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r7">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r7">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r7">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r6">(TEXT)</span><span class="r1"> </span><span class="r4">[default: apache/airflow]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debug</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Drop user in shell instead of running the command. Useful for debugging. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r7">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
diff --git a/images/breeze/output-prepare-provider-documentation.svg b/images/breeze/output-prepare-provider-documentation.svg
index ac42860fac..16d2d5b0a9 100644
--- a/images/breeze/output-prepare-provider-documentation.svg
+++ b/images/breeze/output-prepare-provider-documentation.svg
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-920130443-terminal-wrapper span {
+ .rich-svg-1358763671-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-920130443-terminal-wrapper a {
+ .rich-svg-1358763671-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-920130443-terminal-body .blink {
- animation: rich-svg-920130443-blinker 1s infinite;
+ .rich-svg-1358763671-terminal-body .blink {
+ animation: rich-svg-1358763671-blinker 1s infinite;
}
- @keyframes rich-svg-920130443-blinker {
+ @keyframes rich-svg-1358763671-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-920130443-terminal-wrapper {
+ .rich-svg-1358763671-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-920130443-terminal {
+ .rich-svg-1358763671-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-920130443-terminal:after {
+ .rich-svg-1358763671-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-920130443-terminal-header {
+ .rich-svg-1358763671-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-920130443-terminal-title-tab {
+ .rich-svg-1358763671-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-920130443-terminal-traffic-lights {
+ .rich-svg-1358763671-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-920130443-terminal-body {
+ .rich-svg-1358763671-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-920130443-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-920130443-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-920130443-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-920130443-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-920130443-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-920130443-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-920130443-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-1358763671-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1358763671-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1358763671-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1358763671-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1358763671-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1358763671-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1358763671-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-920130443-terminal-wrapper">
- <div class="rich-svg-920130443-terminal">
- <div class="rich-svg-920130443-terminal-header">
- <svg class="rich-svg-920130443-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-1358763671-terminal-wrapper">
+ <div class="rich-svg-1358763671-terminal">
+ <div class="rich-svg-1358763671-terminal-header">
+ <svg class="rich-svg-1358763671-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-920130443-terminal-title-tab">Command: prepare-provider-documentation</div>
+ <div class="rich-svg-1358763671-terminal-title-tab">Command: prepare-provider-documentation</div>
</div>
- <div class="rich-svg-920130443-terminal-body">
+ <div class="rich-svg-1358763671-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze prepare-provider-documentation [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | </span></div>
<div><span class="r2"> apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | </span></div>
@@ -132,7 +132,7 @@
<div><span class="r1"> </span><span class="r1">Prepare CHANGELOG, README and COMMITS information for providers. </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
<div><span class="r4">╭─ Provider documentation preparation flags ───────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--skip-package-verification</span><span class="r1"> </span><span class="r1">Skip Provider package verification. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debug</span><span class="r1"> </span><span class="r1">Drop user in shell instead of running the command. Useful for debugging. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-prepare-provider-packages.svg b/images/breeze/output-prepare-provider-packages.svg
index d2c8b2219f..a65ef5d483 100644
--- a/images/breeze/output-prepare-provider-packages.svg
+++ b/images/breeze/output-prepare-provider-packages.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
+<svg width="1720.0" height="978" viewBox="0 0 1720.0 978"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-3921007243-terminal-wrapper span {
+ .rich-svg-3971887859-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-3921007243-terminal-wrapper a {
+ .rich-svg-3971887859-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-3921007243-terminal-body .blink {
- animation: rich-svg-3921007243-blinker 1s infinite;
+ .rich-svg-3971887859-terminal-body .blink {
+ animation: rich-svg-3971887859-blinker 1s infinite;
}
- @keyframes rich-svg-3921007243-blinker {
+ @keyframes rich-svg-3971887859-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-3921007243-terminal-wrapper {
+ .rich-svg-3971887859-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-3921007243-terminal {
+ .rich-svg-3971887859-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-3921007243-terminal:after {
+ .rich-svg-3971887859-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-3921007243-terminal-header {
+ .rich-svg-3971887859-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-3921007243-terminal-title-tab {
+ .rich-svg-3971887859-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-3921007243-terminal-traffic-lights {
+ .rich-svg-3971887859-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-3921007243-terminal-body {
+ .rich-svg-3971887859-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-3921007243-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-3921007243-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3921007243-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3921007243-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3921007243-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3921007243-terminal-body .r6 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
-.rich-svg-3921007243-terminal-body .r7 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+ .rich-svg-3971887859-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-3971887859-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3971887859-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3971887859-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3971887859-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3971887859-terminal-body .r6 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+.rich-svg-3971887859-terminal-body .r7 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-3921007243-terminal-wrapper">
- <div class="rich-svg-3921007243-terminal">
- <div class="rich-svg-3921007243-terminal-header">
- <svg class="rich-svg-3921007243-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-3971887859-terminal-wrapper">
+ <div class="rich-svg-3971887859-terminal">
+ <div class="rich-svg-3971887859-terminal-header">
+ <svg class="rich-svg-3971887859-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-3921007243-terminal-title-tab">Command: prepare-provider-packages</div>
+ <div class="rich-svg-3971887859-terminal-title-tab">Command: prepare-provider-packages</div>
</div>
- <div class="rich-svg-3921007243-terminal-body">
+ <div class="rich-svg-3971887859-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze prepare-provider-packages [OPTIONS] [airbyte | alibaba | amazon | apache.beam | apache.cassandra | </span></div>
<div><span class="r2"> apache.drill | apache.druid | apache.hdfs | apache.hive | apache.kylin | </span></div>
@@ -135,6 +135,7 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--package-format</span><span class="r1"> </span><span class="r1">Format of packages.</span><span class="r1"> </span><span class="r6">(wheel | sdist | both)</span><span class="r1"> </span><span class="r4">[default: wheel]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--version-suffix-for-pypi</span><span class="r1"> </span><span class="r1">Version suffix used for PyPI packages (alpha, beta, rc1, etc.).</span><span class="r1"> </span><span class="r6">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--package-list-file</span><span class="r1"> </span><span class="r1">Read list of packages from text file (one package per line)</span><span class="r1"> </span><span class="r6">(FILENAME)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debug</span><span class="r1"> </span><span class="r1">Drop user in shell instead of running the command. Useful for debugging. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r7">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-shell.svg b/images/breeze/output-shell.svg
index aee6d87e98..722a62a934 100644
--- a/images/breeze/output-shell.svg
+++ b/images/breeze/output-shell.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1044" viewBox="0 0 1720.0 1044"
+<svg width="1720.0" height="1374" viewBox="0 0 1720.0 1374"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-2505769163-terminal-wrapper span {
+ .rich-svg-3737790805-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-2505769163-terminal-wrapper a {
+ .rich-svg-3737790805-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-2505769163-terminal-body .blink {
- animation: rich-svg-2505769163-blinker 1s infinite;
+ .rich-svg-3737790805-terminal-body .blink {
+ animation: rich-svg-3737790805-blinker 1s infinite;
}
- @keyframes rich-svg-2505769163-blinker {
+ @keyframes rich-svg-3737790805-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-2505769163-terminal-wrapper {
+ .rich-svg-3737790805-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-2505769163-terminal {
+ .rich-svg-3737790805-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-2505769163-terminal:after {
+ .rich-svg-3737790805-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-2505769163-terminal-header {
+ .rich-svg-3737790805-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-2505769163-terminal-title-tab {
+ .rich-svg-3737790805-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-2505769163-terminal-traffic-lights {
+ .rich-svg-3737790805-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-2505769163-terminal-body {
+ .rich-svg-3737790805-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-2505769163-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-2505769163-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2505769163-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2505769163-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2505769163-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2505769163-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2505769163-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-3737790805-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-3737790805-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3737790805-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3737790805-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3737790805-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3737790805-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3737790805-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-2505769163-terminal-wrapper">
- <div class="rich-svg-2505769163-terminal">
- <div class="rich-svg-2505769163-terminal-header">
- <svg class="rich-svg-2505769163-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-3737790805-terminal-wrapper">
+ <div class="rich-svg-3737790805-terminal">
+ <div class="rich-svg-3737790805-terminal-header">
+ <svg class="rich-svg-3737790805-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-2505769163-terminal-title-tab">Command: shell</div>
+ <div class="rich-svg-3737790805-terminal-title-tab">Command: shell</div>
</div>
- <div class="rich-svg-2505769163-terminal-body">
+ <div class="rich-svg-3737790805-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze shell [OPTIONS] [EXTRA_ARGS]... </span></div>
<div><span class="r2"> </span></div>
@@ -124,27 +124,42 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images.</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10)</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r4">[default: sqlite]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Use (reinstall at entry) Airflow version from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">all) </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--forward-credentials</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Forward local credentials to container when running. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--db-reset</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Reset DB when entering the container. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Choose scope of local sources should be mounted (default = selected).</span><span class="r1"> </span><span class="r7">(selected | all | none)</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: selected] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `sdist` if Airflow should be removed, installed from wheel packages or sdist packages </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> available in dist folder respectively. Important! Using it Implies </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `none`. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(none | wheel | sdist | <airflow_version>) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-extras</span><span class="r1"> Airflow extras to install when </span><span class="r5">--use-airflow-version</span><span class="r1"> is used </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-packages-from-dist</span><span class="r1"> Install all found packages (</span><span class="r5">--package-format</span><span class="r1"> determines type) from 'dist' folder when </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> entering breeze. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--package-format</span><span class="r1"> </span><span class="r1">Format of packages that should be installed from dist.</span><span class="r1"> </span><span class="r7">(wheel | sdist)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: wheel] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r1">Choose scope of local sources should be mounted (default = selected). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(selected | all | none) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: selected] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debian-version</span><span class="r1"> </span><span class="r1">Debian version used for the image.</span><span class="r1"> </span><span class="r7">(bullseye | buster)</span><span class="r1"> </span><span class="r4">[default: bullseye]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">[default: apache/airflow]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: apache/airflow] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-constraints-reference</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> Constraint reference to use. Useful with </span><span class="r5">--use-airflow-version</span><span class="r1"> parameter to </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> specify constraints for the installed version and to find newer dependencies </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(TEXT) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
diff --git a/images/breeze/output-start-airflow.svg b/images/breeze/output-start-airflow.svg
index 4efbcdbaa1..d1365f387b 100644
--- a/images/breeze/output-start-airflow.svg
+++ b/images/breeze/output-start-airflow.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1088" viewBox="0 0 1720.0 1088"
+<svg width="1720.0" height="1418" viewBox="0 0 1720.0 1418"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-2393603551-terminal-wrapper span {
+ .rich-svg-2513811073-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-2393603551-terminal-wrapper a {
+ .rich-svg-2513811073-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-2393603551-terminal-body .blink {
- animation: rich-svg-2393603551-blinker 1s infinite;
+ .rich-svg-2513811073-terminal-body .blink {
+ animation: rich-svg-2513811073-blinker 1s infinite;
}
- @keyframes rich-svg-2393603551-blinker {
+ @keyframes rich-svg-2513811073-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-2393603551-terminal-wrapper {
+ .rich-svg-2513811073-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-2393603551-terminal {
+ .rich-svg-2513811073-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-2393603551-terminal:after {
+ .rich-svg-2513811073-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-2393603551-terminal-header {
+ .rich-svg-2513811073-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-2393603551-terminal-title-tab {
+ .rich-svg-2513811073-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-2393603551-terminal-traffic-lights {
+ .rich-svg-2513811073-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-2393603551-terminal-body {
+ .rich-svg-2513811073-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-2393603551-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-2393603551-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2393603551-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2393603551-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2393603551-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2393603551-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2393603551-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-2513811073-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-2513811073-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-2513811073-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2513811073-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-2513811073-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2513811073-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-2513811073-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-2393603551-terminal-wrapper">
- <div class="rich-svg-2393603551-terminal">
- <div class="rich-svg-2393603551-terminal-header">
- <svg class="rich-svg-2393603551-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-2513811073-terminal-wrapper">
+ <div class="rich-svg-2513811073-terminal">
+ <div class="rich-svg-2513811073-terminal-header">
+ <svg class="rich-svg-2513811073-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-2393603551-terminal-title-tab">Command: start-airflow</div>
+ <div class="rich-svg-2513811073-terminal-title-tab">Command: start-airflow</div>
</div>
- <div class="rich-svg-2393603551-terminal-body">
+ <div class="rich-svg-2513811073-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze start-airflow [OPTIONS] [EXTRA_ARGS]... </span></div>
<div><span class="r2"> </span></div>
@@ -124,29 +124,44 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--load-example-dags</span><span class="r1"> </span><span class="r6">-e</span><span class="r1"> </span><span class="r1">Enable configuration to load example DAGs when starting Airflow. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--load-default-connections</span><span class="r1"> </span><span class="r6">-c</span><span class="r1"> </span><span class="r1">Enable configuration to load default connections when starting Airflow. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--backend</span><span class="r1"> </span><span class="r6">-b</span><span class="r1"> </span><span class="r1">Database backend to use.</span><span class="r1"> </span><span class="r7">(>sqlite< | mysql | postgres | mssql)</span><span class="r1"> </span><span class="r4">[default: sqlite]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> </span><span class="r6">-V</span><span class="r1"> </span><span class="r1">Use (reinstall at entry) Airflow version from PyPI.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--postgres-version</span><span class="r1"> </span><span class="r6">-P</span><span class="r1"> </span><span class="r1">Version of Postgres used.</span><span class="r1"> </span><span class="r7">(>10< | 11 | 12 | 13)</span><span class="r1"> </span><span class="r4">[default: 10]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mysql-version</span><span class="r1"> </span><span class="r6">-M</span><span class="r1"> </span><span class="r1">Version of MySQL used.</span><span class="r1"> </span><span class="r7">(>5.7< | 8)</span><span class="r1"> </span><span class="r4">[default: 5.7]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mssql-version</span><span class="r1"> </span><span class="r6">-S</span><span class="r1"> </span><span class="r1">Version of MsSQL used.</span><span class="r1"> </span><span class="r7">(>2017-latest< | 2019-latest)</span><span class="r1"> </span><span class="r4">[default: 2017-latest]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--load-example-dags</span><span class="r1"> </span><span class="r6">-e</span><span class="r1"> </span><span class="r1">Enable configuration to load example DAGs when starting Airflow. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--load-default-connections</span><span class="r1"> </span><span class="r6">-c</span><span class="r1"> </span><span class="r1">Enable configuration to load default connections when starting Airflow. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">trino | all) </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--forward-credentials</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Forward local credentials to container when running. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--db-reset</span><span class="r1"> </span><span class="r6">-d</span><span class="r1"> </span><span class="r1">Reset DB when entering the container. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r6">-f</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Choose scope of local sources should be mounted (default = selected).</span><span class="r1"> </span><span class="r7">(selected | all | none)</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: selected] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--integration</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Integration(s) to enable when running (can be more than one). </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `sdist` if Airflow should be removed, installed from wheel packages or sdist packages </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> available in dist folder respectively. Important! Using it Implies </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `none`. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(none | wheel | sdist | <airflow_version>) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-extras</span><span class="r1"> Airflow extras to install when </span><span class="r5">--use-airflow-version</span><span class="r1"> is used </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-packages-from-dist</span><span class="r1"> Install all found packages (</span><span class="r5">--package-format</span><span class="r1"> determines type) from 'dist' folder when </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> entering breeze. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--package-format</span><span class="r1"> </span><span class="r1">Format of packages that should be installed from dist.</span><span class="r1"> </span><span class="r7">(wheel | sdist)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: wheel] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--force-build</span><span class="r1"> </span><span class="r1">Force image build no matter if it is determined as needed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r1">Choose scope of local sources should be mounted (default = selected). </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(selected | all | none) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: selected] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">[default: apache/airflow]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: apache/airflow] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-constraints-reference</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> Constraint reference to use. Useful with </span><span class="r5">--use-airflow-version</span><span class="r1"> parameter to </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> specify constraints for the installed version and to find newer dependencies </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(TEXT) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
diff --git a/images/breeze/output-generate-constraints.svg b/images/breeze/output-verify-provider-packages.svg
similarity index 54%
copy from images/breeze/output-generate-constraints.svg
copy to images/breeze/output-verify-provider-packages.svg
index e30b53ac33..d5651ef341 100644
--- a/images/breeze/output-generate-constraints.svg
+++ b/images/breeze/output-verify-provider-packages.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
+<svg width="1720.0" height="978" viewBox="0 0 1720.0 978"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-1608918125-terminal-wrapper span {
+ .rich-svg-3603185097-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-1608918125-terminal-wrapper a {
+ .rich-svg-3603185097-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-1608918125-terminal-body .blink {
- animation: rich-svg-1608918125-blinker 1s infinite;
+ .rich-svg-3603185097-terminal-body .blink {
+ animation: rich-svg-3603185097-blinker 1s infinite;
}
- @keyframes rich-svg-1608918125-blinker {
+ @keyframes rich-svg-3603185097-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-1608918125-terminal-wrapper {
+ .rich-svg-3603185097-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-1608918125-terminal {
+ .rich-svg-3603185097-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-1608918125-terminal:after {
+ .rich-svg-3603185097-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-1608918125-terminal-header {
+ .rich-svg-3603185097-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-1608918125-terminal-title-tab {
+ .rich-svg-3603185097-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,62 +85,63 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-1608918125-terminal-traffic-lights {
+ .rich-svg-3603185097-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-1608918125-terminal-body {
+ .rich-svg-3603185097-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-1608918125-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-1608918125-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-3603185097-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-3603185097-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3603185097-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3603185097-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-3603185097-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-3603185097-terminal-body .r6 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+.rich-svg-3603185097-terminal-body .r7 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-1608918125-terminal-wrapper">
- <div class="rich-svg-1608918125-terminal">
- <div class="rich-svg-1608918125-terminal-header">
- <svg class="rich-svg-1608918125-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-3603185097-terminal-wrapper">
+ <div class="rich-svg-3603185097-terminal">
+ <div class="rich-svg-3603185097-terminal-header">
+ <svg class="rich-svg-3603185097-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-1608918125-terminal-title-tab">Command: generate-constraints</div>
+ <div class="rich-svg-3603185097-terminal-title-tab">Command: verify-provider-packages</div>
</div>
- <div class="rich-svg-1608918125-terminal-body">
+ <div class="rich-svg-3603185097-terminal-body">
<div><span class="r2"> </span></div>
-<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze generate-constraints [OPTIONS] </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze verify-provider-packages [OPTIONS] </span></div>
<div><span class="r2"> </span></div>
-<div><span class="r1"> </span><span class="r1">Generates pinned constraint files with all extras from setup.py in parallel. </span><span class="r1"> </span></div>
+<div><span class="r1"> </span><span class="r1">Verifies if all provider code is following expectations for providers. </span><span class="r1"> </span></div>
<div><span class="r1"> </span></div>
-<div><span class="r4">╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--image-tag</span><span class="r1"> </span><span class="r6">-t</span><span class="r1"> </span><span class="r1">Tag added to the default naming conventions of Airflow CI/PROD images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python</span><span class="r1"> </span><span class="r6">-p</span><span class="r1"> </span><span class="r1">Python major/minor version used in Airflow image for images. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r7">(>3.7< | 3.8 | 3.9 | 3.10) </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--generate-constraints-mode</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Mode of generating constraints</span><span class="r1"> </span><span class="r7">(source-providers | pypi-providers | no-providers)</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: source-providers] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
-<div><span class="r4">╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--run-in-parallel</span><span class="r1"> </span><span class="r1">Run the operation in parallel on all or selected subset of Python versions. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--parallelism</span><span class="r1"> </span><span class="r1">Maximum number of processes to use while running the operation in parallel.</span><span class="r1"> </span><span class="r7">(INTEGER RANGE)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 4; 1<=x<=8] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--python-versions</span><span class="r1"> </span><span class="r1">Space separated list of python versions used for build with multiple versions.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7 3.8 3.9 3.10] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╭─ Provider verification flags ────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-airflow-version</span><span class="r1"> Use (reinstall at entry) Airflow version from PyPI. It can also be `none`, `wheel`, or </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `sdist` if Airflow should be removed, installed from wheel packages or sdist packages </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> available in dist folder respectively. Important! Using it Implies </span><span class="r5">--mount-sources</span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> `none`. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r6">(none | wheel | sdist | <airflow_version>) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-extras</span><span class="r1"> Airflow extras to install when </span><span class="r5">--use-airflow-version</span><span class="r1"> is used </span><span class="r6">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--use-packages-from-dist</span><span class="r1"> Install all found packages (</span><span class="r5">--package-format</span><span class="r1"> determines type) from 'dist' folder when </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> entering breeze. </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--package-format</span><span class="r1"> </span><span class="r1">Format of packages that should be installed from dist.</span><span class="r1"> </span><span class="r6">(wheel | sdist)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: wheel] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--debug</span><span class="r1"> </span><span class="r1">Drop user in shell instead of running the command. Useful for debugging. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r6">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r7">(TEXT)</span><span class="r1"> </span><span class="r4">[default: apache/airflow]</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--answer</span><span class="r1"> </span><span class="r6">-a</span><span class="r1"> </span><span class="r1">Force answer to questions.</span><span class="r1"> </span><span class="r7">(y | n | q | yes | no | quit)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--airflow-constraints-reference</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> Constraint reference to use. Useful with </span><span class="r5">--use-airflow-version</span><span class="r1"> parameter to </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> specify constraints for the installed version and to find newer dependencies </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r6">(TEXT) </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r7">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r7">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--github-repository</span><span class="r1"> </span><span class="r7">-g</span><span class="r1"> </span><span class="r1">GitHub repository used to pull, push run images.</span><span class="r1"> </span><span class="r6">(TEXT)</span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: apache/airflow] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r7">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r1"></span><span class="r1"> </span></div>
</div>
diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env
index 325b72aee2..eb3098c29c 100644
--- a/scripts/ci/docker-compose/_docker.env
+++ b/scripts/ci/docker-compose/_docker.env
@@ -16,6 +16,7 @@
# under the License.
AIRFLOW_CI_IMAGE
AIRFLOW_EXTRAS
+AIRFLOW_CONSTRAINTS_REFERENCE
ANSWER
BACKEND
BREEZE
@@ -59,7 +60,6 @@ LIST_OF_INTEGRATION_TESTS_TO_RUN
RUN_SYSTEM_TESTS
START_AIRFLOW
SKIP_ENVIRONMENT_INITIALIZATION
-SKIP_PACKAGE_VERIFICATION
SKIP_SSH_SETUP
TEST_TYPE
UPGRADE_TO_NEWER_DEPENDENCIES
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 04c173c865..4e45f80702 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -29,6 +29,7 @@ services:
# With env files and there are many problems with it:
- AIRFLOW_CI_IMAGE=${AIRFLOW_CI_IMAGE}
- AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+ - AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE}
- ANSWER=${ANSWER}
- BACKEND=${BACKEND}
- BREEZE=${BREEZE}
@@ -72,7 +73,6 @@ services:
- RUN_SYSTEM_TESTS=${RUN_SYSTEM_TESTS}
- START_AIRFLOW=${START_AIRFLOW}
- SKIP_ENVIRONMENT_INITIALIZATION=${SKIP_ENVIRONMENT_INITIALIZATION}
- - SKIP_PACKAGE_VERIFICATION=${SKIP_PACKAGE_VERIFICATION}
- SKIP_SSH_SETUP=${SKIP_SSH_SETUP}
- TEST_TYPE=${TEST_TYPE}
- UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index 3698ac4abe..0d26338fd1 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -57,7 +57,6 @@ RUN_TESTS="false"
LIST_OF_INTEGRATION_TESTS_TO_RUN=""
RUN_SYSTEM_TESTS=""
START_AIRFLOW="false"
-SKIP_PACKAGE_VERIFICATION="false"
SKIP_SSH_SETUP="true"
SKIP_ENVIRONMENT_INITIALIZATION="false"
TEST_TYPE=
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 5e40b98c0e..bdcd3fe57d 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -410,9 +410,6 @@ function initialization::initialize_image_build_variables() {
INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES:="true"}
export INSTALL_PROVIDERS_FROM_SOURCES
- SKIP_PACKAGE_VERIFICATION=${SKIP_PACKAGE_VERIFICATION:=""}
- export SKIP_PACKAGE_VERIFICATION
-
SKIP_SSH_SETUP=${SKIP_SSH_SETUP:="false"}
export SKIP_SSH_SETUP
diff --git a/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh b/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
deleted file mode 100755
index 6cbb89b1a3..0000000000
--- a/scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-export MOUNT_SELECTED_LOCAL_SOURCES="false"
-
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-if [[ ${PACKAGE_FORMAT} != "wheel" && ${PACKAGE_FORMAT} != "sdist" ]]; then
- echo
- echo "${COLOR_RED}ERROR: Wrong package format ${PACKAGE_FORMAT}. Should be 'wheel' or 'sdist'${COLOR_RESET}"
- echo
- exit 3
-fi
-
-AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS="all"}
-export AIRFLOW_EXTRAS
-
-function run_test_package_import_all_classes() {
- # Groups are added internally
- docker_v run "${EXTRA_DOCKER_FLAGS[@]}" \
- -t \
- -v "${AIRFLOW_SOURCES}/setup.py:/airflow_sources/setup.py:cached" \
- -v "${AIRFLOW_SOURCES}/setup.cfg:/airflow_sources/setup.cfg:cached" \
- -v "${AIRFLOW_SOURCES}/airflow/__init__.py:/airflow_sources/airflow/__init__.py:cached" \
- -v "${AIRFLOW_SOURCES}/empty:/opt/airflow/airflow:cached" \
- -v "${AIRFLOW_SOURCES}/scripts/in_container:/opt/airflow/scripts/in_container:cached" \
- -v "${AIRFLOW_SOURCES}/dev/import_all_classes.py:/opt/airflow/dev/import_all_classes.py:cached" \
- -e "SKIP_ENVIRONMENT_INITIALIZATION=true" \
- "${AIRFLOW_CI_IMAGE_WITH_TAG}" \
- "/opt/airflow/scripts/in_container/run_install_and_test_provider_packages.sh"
-}
-
-run_test_package_import_all_classes
diff --git a/scripts/ci/provider_packages/ci_make_providers_2_1_compliant.sh b/scripts/ci/provider_packages/ci_make_providers_2_1_compliant.sh
deleted file mode 100755
index f890c2a9eb..0000000000
--- a/scripts/ci/provider_packages/ci_make_providers_2_1_compliant.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-# Some of our provider sources are not Airflow 2.1 compliant any more
-# We replace them with 2.1 compliant versions from PyPI to run the checks
-
-cd "${AIRFLOW_SOURCES}" || exit 1
-rm -rvf dist/apache_airflow_providers_cncf_kubernetes* dist/apache_airflow_providers_celery*
-pip download --no-deps --dest dist apache-airflow-providers-cncf-kubernetes==3.0.0 \
- apache-airflow-providers-celery==2.1.3
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index e0aed382ca..d00b08370b 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -85,32 +85,44 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
echo
echo "${COLOR_BLUE}Skip installing airflow - only install wheel/tar.gz packages that are present locally.${COLOR_RESET}"
echo
+ echo
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
+ echo
uninstall_airflow_and_providers
elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
echo
- echo "${COLOR_BLUE}Install airflow from wheel package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
echo
uninstall_airflow_and_providers
- install_airflow_from_wheel "[${AIRFLOW_EXTRAS}]"
+ echo "${COLOR_BLUE}Install airflow from wheel package with extras: '${AIRFLOW_EXTRAS}' and constraints reference ${AIRFLOW_CONSTRAINTS_REFERENCE}.${COLOR_RESET}"
+ echo
+ install_airflow_from_wheel "${AIRFLOW_EXTRAS}" "${AIRFLOW_CONSTRAINTS_REFERENCE}"
uninstall_providers
elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then
echo
- echo "${COLOR_BLUE}Install airflow from sdist package with [${AIRFLOW_EXTRAS}] extras but uninstalling providers.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
echo
uninstall_airflow_and_providers
- install_airflow_from_sdist "[${AIRFLOW_EXTRAS}]"
+ echo
+ echo "${COLOR_BLUE}Install airflow from sdist package with extras: '${AIRFLOW_EXTRAS}' and constraints reference ${AIRFLOW_CONSTRAINTS_REFERENCE}.${COLOR_RESET}"
+ echo
+ install_airflow_from_sdist "${AIRFLOW_EXTRAS}" "${AIRFLOW_CONSTRAINTS_REFERENCE}"
uninstall_providers
else
echo
- echo "${COLOR_BLUE}Install airflow from PyPI without extras"
+ echo "${COLOR_BLUE}Uninstalling airflow and providers"
+ echo
+ uninstall_airflow_and_providers
+ echo
+ echo "${COLOR_BLUE}Install released airflow from PyPI with extras: '${AIRFLOW_EXTRAS}' and constraints reference ${AIRFLOW_CONSTRAINTS_REFERENCE}.${COLOR_RESET}"
echo
- install_released_airflow_version "${USE_AIRFLOW_VERSION}"
+ install_released_airflow_version "${USE_AIRFLOW_VERSION}" "${AIRFLOW_CONSTRAINTS_REFERENCE}"
fi
if [[ ${USE_PACKAGES_FROM_DIST=} == "true" ]]; then
echo
- echo "${COLOR_BLUE}Install all packages from dist folder"
+ echo "${COLOR_BLUE}Install all packages from dist folder${COLOR_RESET}"
if [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
- echo "(except apache-airflow)"
+ echo "${COLOR_BLUE}(except apache-airflow)${COLOR_RESET}"
fi
if [[ ${PACKAGE_FORMAT} == "both" ]]; then
echo
@@ -122,7 +134,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
installable_files=()
for file in /dist/*.{whl,tar.gz}
do
- if [[ ${USE_AIRFLOW_VERSION} == "wheel" && ${file} == "apache?airflow-[0-9]"* ]]; then
+ if [[ ${USE_AIRFLOW_VERSION} == "wheel" && ${file} == "/dist/apache?airflow-[0-9]"* ]]; then
# Skip Apache Airflow package - it's just been installed above with extras
echo "Skipping ${file}"
continue
@@ -137,7 +149,7 @@ if [[ ${SKIP_ENVIRONMENT_INITIALIZATION=} != "true" ]]; then
fi
done
if (( ${#installable_files[@]} )); then
- pip install "${installable_files[@]}" --no-deps
+ pip install "${installable_files[@]}"
fi
fi
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 57f4e4d40d..10115a7009 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -113,6 +113,7 @@ function in_container_cleanup_pycache() {
function in_container_fix_ownership() {
if [[ ${HOST_OS:=} == "Linux" ]]; then
DIRECTORIES_TO_FIX=(
+ "/dist"
"/files"
"/root/.aws"
"/root/.azure"
@@ -192,10 +193,18 @@ function dump_airflow_logs() {
function install_airflow_from_wheel() {
local extras
extras="${1:-}"
+ if [[ ${extras} != "" ]]; then
+ extras="[${extras}]"
+ fi
+ local constraints_reference
+ constraints_reference="${2:-}"
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 /dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
local airflow_package
airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache_airflow-[0-9]*.whl')
echo
- echo "Found package: ${airflow_package}. Installing."
+ echo "Found package: ${airflow_package}. Installing with extras: ${extras}, constraints reference: ${constraints_reference}."
echo
if [[ -z "${airflow_package}" ]]; then
>&2 echo
@@ -203,16 +212,25 @@ function install_airflow_from_wheel() {
>&2 echo
exit 4
fi
- pip install "${airflow_package}${extras}"
+ pip install "${airflow_package}${extras}" --constraint \
+ "https://raw.githubusercontent.com/apache/airflow/${constraints_reference}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
}
function install_airflow_from_sdist() {
local extras
- extras="${1}"
+ extras="${1:-}"
+ if [[ ${extras} != "" ]]; then
+ extras="[${extras}]"
+ fi
+ local constraints_reference
+ constraints_reference="${2:-}"
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 /dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
local airflow_package
airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache-airflow-[0-9]*.tar.gz')
echo
- echo "Found package: ${airflow_package}. Installing."
+ echo "Found package: ${airflow_package}. Installing with extras: ${extras}, constraints reference: ${constraints_reference}."
echo
if [[ -z "${airflow_package}" ]]; then
>&2 echo
@@ -220,7 +238,8 @@ function install_airflow_from_sdist() {
>&2 echo
exit 4
fi
- pip install "${airflow_package}${extras}"
+ pip install "${airflow_package}${extras}" --constraint \
+ "https://raw.githubusercontent.com/apache/airflow/${constraints_reference}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
}
function uninstall_airflow() {
@@ -248,11 +267,16 @@ function uninstall_airflow_and_providers() {
function install_released_airflow_version() {
local version="${1}"
echo
- echo "Installing released ${version} version of airflow without extras"
+ echo "Installing released ${version} version of airflow with extras: ${AIRFLOW_EXTRAS} and constraints constraints-${version}"
echo
-
rm -rf "${AIRFLOW_SOURCES}"/*.egg-info
- pip install "apache-airflow==${version}"
+ if [[ ${AIRFLOW_EXTRAS} != "" ]]; then
+ BRACKETED_AIRFLOW_EXTRAS="[${AIRFLOW_EXTRAS}]"
+ else
+ BRACKETED_AIRFLOW_EXTRAS=""
+ fi
+ pip install "apache-airflow${BRACKETED_AIRFLOW_EXTRAS}==${version}" \
+ --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/constraints-${version}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
}
function install_local_airflow_with_eager_upgrade() {
@@ -302,7 +326,10 @@ function install_all_provider_packages_from_wheels() {
echo "Installing all provider packages from wheels"
echo
uninstall_providers
- pip install /dist/apache_airflow*providers_*.whl
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 /dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ pip install /dist/apache_airflow_providers_*.whl
}
function install_all_provider_packages_from_sdist() {
@@ -310,21 +337,27 @@ function install_all_provider_packages_from_sdist() {
echo "Installing all provider packages from .tar.gz"
echo
uninstall_providers
- pip install /dist/apache-airflow-*providers-*.tar.gz
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 /dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ pip install /dist/apache-airflow-providers-*.tar.gz
}
function twine_check_provider_packages_from_wheels() {
echo
echo "Twine check of all provider packages from wheels"
echo
- twine check /dist/apache_airflow*providers_*.whl
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 /dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ twine check /dist/apache_airflow_providers_*.whl
}
function twine_check_provider_packages_from_sdist() {
echo
echo "Twine check all provider packages from sdist"
echo
- twine check /dist/apache-airflow-*providers-*.tar.gz
+ twine check /dist/apache-airflow-providers-*.tar.gz
}
function setup_provider_packages() {
@@ -346,9 +379,7 @@ function setup_provider_packages() {
function install_supported_pip_version() {
- group_start "Install supported PIP version ${AIRFLOW_PIP_VERSION}"
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
- group_end
}
function filename_to_python_module() {
diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh
deleted file mode 100755
index cc3175c73f..0000000000
--- a/scripts/in_container/run_install_and_test_provider_packages.sh
+++ /dev/null
@@ -1,344 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-
-function verify_parameters(){
- group_start "Verify parameters passed"
- echo
- echo "Testing if all classes in import packages can be imported"
- echo
-
- if [[ -z "${USE_AIRFLOW_VERSION=""}" ]]; then
- echo
- echo "${COLOR_RED}ERROR: You have to specify airflow version to install.${COLOR_RESET}"
- echo
- echo "It might be version from PyPI, wheel with extras or none to uninstall airflow"
- echo
- exit 1
- fi
-
- PACKAGE_FORMAT=${PACKAGE_FORMAT=}
-
- if [[ ${PACKAGE_FORMAT} != "wheel" && ${PACKAGE_FORMAT} != "sdist" ]]; then
- echo
- echo "${COLOR_RED}ERROR: Wrong install type ${PACKAGE_FORMAT}. Should be 'wheel' or 'sdist' ${COLOR_RESET}"
- echo
- exit 3
- fi
- group_end
-}
-
-function install_airflow_no_extras() {
- group_start "Install Airflow as specified"
- if [[ ${USE_AIRFLOW_VERSION} == "none" ]]; then
- echo
- echo "Skip installing airflow - only install wheel packages that are present locally"
- echo
- uninstall_airflow_and_providers
- elif [[ ${USE_AIRFLOW_VERSION} == "wheel" ]]; then
- echo
- echo "Install airflow from wheel (no extras)"
- echo
- uninstall_airflow_and_providers
- install_airflow_from_wheel ""
- uninstall_providers
- elif [[ ${USE_AIRFLOW_VERSION} == "sdist" ]]; then
- echo
- echo "Install airflow from sdist (no extras)"
- echo
- uninstall_airflow_and_providers
- install_airflow_from_sdist ""
- uninstall_providers
- else
- echo
- echo "Install airflow from PyPI"
- echo
- install_released_airflow_version "${USE_AIRFLOW_VERSION}"
- uninstall_providers
- fi
- group_end
-}
-
-function install_provider_packages() {
- group_start "Install provider packages"
- if [[ ${PACKAGE_FORMAT} == "wheel" ]]; then
- install_all_provider_packages_from_wheels
- elif [[ ${PACKAGE_FORMAT} == "sdist" ]]; then
- install_all_provider_packages_from_sdist
- else
- echo
- echo "${COLOR_RED}ERROR: Wrong package format ${PACKAGE_FORMAT}. Should be wheel or sdist${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function twine_check_provider_packages() {
- group_start "Twine check provider packages"
- if [[ ${PACKAGE_FORMAT} == "wheel" ]]; then
- twine_check_provider_packages_from_wheels
- elif [[ ${PACKAGE_FORMAT} == "sdist" ]]; then
- twine_check_provider_packages_from_sdist
- else
- echo
- echo "${COLOR_RED}ERROR: Wrong package format ${PACKAGE_FORMAT}. Should be wheel or sdist${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-
-function discover_all_provider_packages() {
- group_start "Listing available providers via 'airflow providers list'"
- # Columns is to force it wider, so it doesn't wrap at 80 characters
- COLUMNS=180 airflow providers list
-
- local actual_number_of_providers
- actual_providers=$(airflow providers list --output yaml | grep package_name)
- actual_number_of_providers=$(wc -l <<<"$actual_providers")
- if (( actual_number_of_providers < 67 )); then
- echo
- echo "${COLOR_RED}ERROR:Number of providers installed is wrong: ${actual_number_of_providers}${COLOR_RESET}"
- echo
- echo "Providers were:"
- echo
- echo "$actual_providers"
- exit 1
- fi
- group_end
-}
-
-function discover_all_hooks() {
- group_start "Listing available hooks via 'airflow providers hooks'"
- COLUMNS=180 airflow providers hooks
- local actual_number_of_hooks
- actual_number_of_hooks=$(airflow providers hooks --output table | grep -c "| apache" | xargs)
- if (( actual_number_of_hooks < 70 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of hooks registered is wrong: ${actual_number_of_hooks} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function discover_all_extra_links() {
- group_start "Listing available extra links via 'airflow providers links'"
- COLUMNS=180 airflow providers links
-
- local actual_number_of_extra_links
- actual_number_of_extra_links=$(airflow providers links --output table | grep -c ^airflow.providers | xargs)
- if (( actual_number_of_extra_links < 7 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of links registered is wrong: ${actual_number_of_extra_links} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function discover_all_connection_form_widgets() {
- group_start "Listing available widgets via 'airflow providers widgets'"
-
- COLUMNS=180 airflow providers widgets
-
- local actual_number_of_widgets
- actual_number_of_widgets=$(airflow providers widgets --output table | grep -c ^extra)
- if (( actual_number_of_widgets < 44 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of connections with widgets registered is wrong: ${actual_number_of_widgets} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function discover_all_field_behaviours() {
- group_start "Listing connections with custom behaviours via 'airflow providers behaviours'"
- COLUMNS=180 airflow providers behaviours
-
- local actual_number_of_connections_with_behaviours
- actual_number_of_connections_with_behaviours=$(airflow providers behaviours --output table | grep -v "===" | \
- grep -v field_behaviours | grep -cv "^ " | xargs)
- if (( actual_number_of_connections_with_behaviours < 21 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of connections with customized behaviours is wrong: ${actual_number_of_connections_with_behaviours} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function discover_all_logging_handlers() {
- group_start "Listing available logging handlers via 'airflow providers logging'"
- COLUMNS=180 airflow providers logging
-
- local actual_number_of_logging
- actual_number_of_logging=$(airflow providers logging --output table | grep -c ^airflow.providers | xargs)
- if (( actual_number_of_logging < 6 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of logging handlers registered is wrong: ${actual_number_of_logging} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function discover_all_secrets_backends() {
- group_start "Listing available secrets backends via 'airflow providers secrets'"
- COLUMNS=180 airflow providers secrets
-
- local actual_number_of_secrets
- actual_number_of_secrets=$(airflow providers logging --output table | grep -c ^airflow.providers | xargs)
- if (( actual_number_of_secrets < 5 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of secrets backends registered is wrong: ${actual_number_of_secrets} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function discover_all_auth_backends() {
- group_start "Listing available API auth backends via 'airflow providers auth'"
- COLUMNS=180 airflow providers auth
-
- local actual_number_of_auth
- actual_number_of_auth=$(airflow providers logging --output table | grep -c ^airflow.providers | xargs)
- if (( actual_number_of_auth < 1 )); then
- echo
- echo "${COLOR_RED}ERROR: Number of auth backends registered is wrong: ${actual_number_of_auth} ${COLOR_RESET}"
- echo
- exit 1
- fi
- group_end
-}
-
-function ver() {
- # convert SemVer number to comparable string (strips pre-release version)
- # shellcheck disable=SC2086,SC2183
- printf "%04d%04d%04d%.0s" ${1//[.-]/ }
-}
-
-function import_all_provider_classes() {
- group_start "Import all Airflow classes"
- # We have to move to a directory where "airflow" is
- unset PYTHONPATH
- # We need to make sure we are not in the airflow checkout, otherwise it will automatically be added to the
- # import path
- pushd /
-
- declare -a IMPORT_CLASS_PARAMETERS
-
- PROVIDER_PATHS=$(
- python3 <<EOF 2>/dev/null
-import airflow.providers;
-path=airflow.providers.__path__
-for p in path._path:
- print(p)
-EOF
- )
- export PROVIDER_PATHS
-
- echo "Searching for providers packages in:"
- echo "${PROVIDER_PATHS}"
-
- while read -r provider_path; do
- IMPORT_CLASS_PARAMETERS+=("--path" "${provider_path}")
- done < <(echo "${PROVIDER_PATHS}")
-
- if python3 /opt/airflow/dev/import_all_classes.py "${IMPORT_CLASS_PARAMETERS[@]}"; then
- popd
- group_end
- else
- popd
- group_end
- return 1
- fi
-}
-
-AIRFLOW_EXTRAS="all"
-export AIRFLOW_EXTRAS
-
-setup_provider_packages
-verify_parameters
-install_airflow_no_extras
-
-if [[ ${USE_AIRFLOW_VERSION} != "2.1.0" ]]; then
- # Airflow 2.1.0 installs importlib_metadata version that does not work well with twine
- # So we should skip twine check in this case
- twine_check_provider_packages
-fi
-install_provider_packages
-set +e
-
-import_error_file="/tmp/import_errors.txt"
-
-if import_all_provider_classes 2>"${import_error_file}"; then
- echo "${COLOR_GREEN}All classes imported properly${COLOR_RESET}"
-else
- echo "${COLOR_RED}There were errors when importing Providers!${COLOR_RESET}"
- if [[ ${USE_AIRFLOW_VERSION} =~ [0-9\.]* ]]; then
- echo "${COLOR_RED}This test is run to check Provider's compatibility with Airflow ${USE_AIRFLOW_VERSION}${COLOR_RESET}"
- echo "${COLOR_RED}So this error might mean that your providers are not Airflow ${USE_AIRFLOW_VERSION} compatible${COLOR_RESET}"
-
- if grep "airflow.utils.context" <"${import_error_file}" >/dev/null 2>&1; then
- echo
- echo "${COLOR_YELLOW}You are using Context class which is only available in Airflow 2.3${COLOR_RESET}"
- echo
- echo """
-You should use this construct to use the Context class in your operator:
-
-${COLOR_BLUE}from typing import TYPE_CHECKING${COLOR_RESET}
-
-${COLOR_BLUE}if TYPE_CHECKING:${COLOR_RESET}
-${COLOR_BLUE} from airflow.utils.context import Context${COLOR_RESET}
-
-${COLOR_BLUE}def execute(self, context: 'Context'):${COLOR_RESET}
-
-${COLOR_YELLOW}You can also have other imports that are missing in Airflow ${USE_AIRFLOW_VERSION} so look at the details below!${COLOR_RESET}
-
-"""
- fi
- fi
- echo
- echo "${COLOR_YELLOW}Detailed errors:${COLOR_RESET}"
- echo
- cat /"${import_error_file}"
- echo
- exit 1
-fi
-
-discover_all_provider_packages
-discover_all_hooks
-discover_all_connection_form_widgets
-discover_all_field_behaviours
-discover_all_extra_links
-
-# The commands below are available only for airflow version 2.2.0+
-airflow_version=$(airflow version)
-airflow_version_comparable=$(ver "${airflow_version}")
-min_airflow_version_comparable=$(ver "2.2.0")
-if (( airflow_version_comparable >= min_airflow_version_comparable )); then
- discover_all_logging_handlers
- discover_all_secrets_backends
- discover_all_auth_backends
-fi
diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh
index dd3f663513..791724d2bb 100755
--- a/scripts/in_container/run_prepare_airflow_packages.sh
+++ b/scripts/in_container/run_prepare_airflow_packages.sh
@@ -69,9 +69,13 @@ function prepare_airflow_packages() {
rm -rf -- *egg-info*
rm -rf -- build
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
echo
echo "${COLOR_GREEN}Airflow package prepared in format: ${PACKAGE_FORMAT}${COLOR_RESET}"
echo
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 ./dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
}
install_supported_pip_version
diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh
index 959f406a73..2730cc4aea 100755
--- a/scripts/in_container/run_prepare_provider_documentation.sh
+++ b/scripts/in_container/run_prepare_provider_documentation.sh
@@ -18,11 +18,6 @@
# shellcheck source=scripts/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-function verify_provider_packages_named_properly() {
- python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \
- verify-provider-classes
-}
-
function run_prepare_documentation() {
local prepared_documentation=()
local skipped_documentation=()
@@ -120,19 +115,6 @@ function run_prepare_documentation() {
python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content "${prepared_documentation[@]}"
echo
fi
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- echo
- echo "${COLOR_YELLOW}You can separately generate content of the issue to create to track testing status by running this command${COLOR_RESET}"
- echo "${COLOR_YELLOW}You can optionally exclude some PRs via --excluded-pr-list option containing coma-separated list of pr numbers${COLOR_RESET}"
- echo
- echo "python3 dev/provider_packages/prepare_provider_packages.py generate-issue-content \\"
- echo "${prepared_documentation[@]}" | fold -sw 100 | sed "s/^/ /" | sed "s/$/ \\\\/"
- echo " [--excluded-pr-list=\"\" ]"
- echo
- echo "${COLOR_YELLOW}You can also run it here by rerunning the prepare-providers-documentation${COLOR_RESET}"
- echo "${COLOR_YELLOW}with --generate-providers-issue and --forward-credentials flag (you have to have GITHUB_TOKEN variable set)${COLOR_RESET}"
- echo
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
echo
echo "${COLOR_YELLOW}Please review the updated files, classify the changelog entries and commit the changes!${COLOR_RESET}"
echo
@@ -148,17 +130,6 @@ export PYTHONPATH="${AIRFLOW_SOURCES}"
install_supported_pip_version
-if [[ ${SKIP_PACKAGE_VERIFICATION=} == "true" || ${SKIP_PACKAGE_VERIFICATION} == "True" ]]; then
- echo
- echo "${COLOR_YELLOW}Skipping package verification!${COLOR_RESET}"
- echo
-else
- group_start "Importing all classes"
- python3 "${AIRFLOW_SOURCES}/dev/import_all_classes.py" --path "airflow/providers"
- group_end
- verify_provider_packages_named_properly
-fi
-
OPTIONAL_RELEASE_VERSION_ARGUMENT=()
if [[ $# != "0" && ${1} =~ ^[0-9][0-9][0-9][0-9]\.[0-9][0-9]\.[0-9][0-9]$ ]]; then
OPTIONAL_RELEASE_VERSION_ARGUMENT+=("--release-version" "${1}")
diff --git a/scripts/in_container/run_prepare_provider_packages.sh b/scripts/in_container/run_prepare_provider_packages.sh
index c4fa53ca7c..96d80571f1 100755
--- a/scripts/in_container/run_prepare_provider_packages.sh
+++ b/scripts/in_container/run_prepare_provider_packages.sh
@@ -124,6 +124,8 @@ function build_provider_packages() {
fi
echo
echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
+ ls -w 1 /dist
+ echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
if [[ ${#error_packages[@]} != "0" ]]; then
echo
echo "${COLOR_RED}There were errors when preparing packages. Exiting! ${COLOR_RESET}"
diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py
new file mode 100755
index 0000000000..3a1765473a
--- /dev/null
+++ b/scripts/in_container/verify_providers.py
@@ -0,0 +1,916 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import importlib
+import os
+import pkgutil
+import re
+import subprocess
+import sys
+import traceback
+import warnings
+from enum import Enum
+from inspect import isclass
+from pathlib import Path
+from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Type
+from warnings import WarningMessage
+
+from rich.console import Console
+
+console = Console(width=400, color_system="standard")
+
+AIRFLOW_SOURCES_ROOT = Path(__file__).parents[2].absolute()
+PROVIDERS_PATH = AIRFLOW_SOURCES_ROOT / "airflow" / "providers"
+
+
+class EntityType(Enum):
+ Operators = "Operators"
+ Transfers = "Transfers"
+ Sensors = "Sensors"
+ Hooks = "Hooks"
+ Secrets = "Secrets"
+
+
+class EntityTypeSummary(NamedTuple):
+ entities: List[str]
+ new_entities_table: str
+ wrong_entities: List[Tuple[type, str]]
+
+
+class VerifiedEntities(NamedTuple):
+ all_entities: Set[str]
+ wrong_entities: List[Tuple[type, str]]
+
+
+class ProviderPackageDetails(NamedTuple):
+ provider_package_id: str
+ full_package_name: str
+ pypi_package_name: str
+ source_provider_package_path: str
+ documentation_provider_package_path: str
+ provider_description: str
+ versions: List[str]
+ excluded_python_versions: List[str]
+
+
+ENTITY_NAMES = {
+ EntityType.Operators: "Operators",
+ EntityType.Transfers: "Transfer Operators",
+ EntityType.Sensors: "Sensors",
+ EntityType.Hooks: "Hooks",
+ EntityType.Secrets: "Secrets",
+}
+
+TOTALS: Dict[EntityType, int] = {
+ EntityType.Operators: 0,
+ EntityType.Hooks: 0,
+ EntityType.Sensors: 0,
+ EntityType.Transfers: 0,
+ EntityType.Secrets: 0,
+}
+
+OPERATORS_PATTERN = r".*Operator$"
+SENSORS_PATTERN = r".*Sensor$"
+HOOKS_PATTERN = r".*Hook$"
+SECRETS_PATTERN = r".*Backend$"
+TRANSFERS_PATTERN = r".*To[A-Z0-9].*Operator$"
+WRONG_TRANSFERS_PATTERN = r".*Transfer$|.*TransferOperator$"
+
+ALL_PATTERNS = {
+ OPERATORS_PATTERN,
+ SENSORS_PATTERN,
+ HOOKS_PATTERN,
+ SECRETS_PATTERN,
+ TRANSFERS_PATTERN,
+ WRONG_TRANSFERS_PATTERN,
+}
+
+EXPECTED_SUFFIXES: Dict[EntityType, str] = {
+ EntityType.Operators: "Operator",
+ EntityType.Hooks: "Hook",
+ EntityType.Sensors: "Sensor",
+ EntityType.Secrets: "Backend",
+ EntityType.Transfers: "Operator",
+}
+
+
+# The set of known deprecation messages that we know about.
+# It contains tuples of "message" and the module that generates the warning - so when the
+# Same warning is generated by different module, it is not treated as "known" warning.
+KNOWN_DEPRECATED_MESSAGES: Set[Tuple[str, str]] = {
+ (
+ 'This version of Apache Beam has not been sufficiently tested on Python 3.9. '
+ 'You may encounter bugs or missing features.',
+ "apache_beam",
+ ),
+ (
+ 'This version of Apache Beam has not been sufficiently tested on Python 3.10. '
+ 'You may encounter bugs or missing features.',
+ "apache_beam",
+ ),
+ (
+ "Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since"
+ " Python 3.3,and in 3.9 it will stop working",
+ "apache_beam",
+ ),
+ (
+ 'pyarrow.HadoopFileSystem is deprecated as of 2.0.0, please use pyarrow.fs.HadoopFileSystem instead.',
+ "papermill",
+ ),
+ (
+ "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that "
+ "adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
+ "apache_beam",
+ ),
+ (
+ "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that "
+ "adheres to: 'pyarrow<5.1.0,>=5.0.0; extra == \"pandas\"'",
+ "snowflake",
+ ),
+ ("dns.hash module will be removed in future versions. Please use hashlib instead.", "dns"),
+ ("PKCS#7 support in pyOpenSSL is deprecated. You should use the APIs in cryptography.", "eventlet"),
+ ("PKCS#12 support in pyOpenSSL is deprecated. You should use the APIs in cryptography.", "eventlet"),
+ (
+ "the imp module is deprecated in favour of importlib; see the module's documentation"
+ " for alternative uses",
+ "hdfs",
+ ),
+ ("This operator is deprecated. Please use `airflow.providers.tableau.operators.tableau`.", "salesforce"),
+ (
+ "You have an incompatible version of 'pyarrow' installed (4.0.1), please install a version that"
+ " adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
+ "snowflake",
+ ),
+ (
+ "You have an incompatible version of 'pyarrow' installed (6.0.1), please install a version that"
+ " adheres to: 'pyarrow<5.1.0,>=5.0.0; extra == \"pandas\"'",
+ "snowflake",
+ ),
+ (
+ "You have an incompatible version of 'pyarrow' installed (6.0.1), please install a version that"
+ " adheres to: 'pyarrow<3.1.0,>=3.0.0; extra == \"pandas\"'",
+ "snowflake",
+ ),
+ ("SelectableGroups dict interface is deprecated. Use select.", "kombu"),
+ ("The module cloudant is now deprecated. The replacement is ibmcloudant.", "cloudant"),
+ ("This module is deprecated. Please use `airflow.operators.empty`.", "dbt"),
+ ("This module is deprecated. Please use `airflow.operators.empty`.", "jdbc"),
+ ("This module is deprecated. Please use `airflow.operators.empty`.", "azure"),
+ ("This module is deprecated. Please use `airflow.operators.empty`.", "qubole"),
+ ("This module is deprecated. Please use `airflow.operators.empty`.", "winrm"),
+ ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "dbt"),
+ ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "jdbc"),
+ ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "azure"),
+ ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "qubole"),
+ ("This class is deprecated. Please use `airflow.operators.empty.EmptyOperator`.", "winrm"),
+ (
+ "'nteract-scrapbook' package has been renamed to `scrapbook`. No new releases are "
+ "going out for this old package name.",
+ "scrapbook",
+ ),
+}
+
+KNOWN_COMMON_DEPRECATED_MESSAGES: Set[str] = {
+ "distutils Version classes are deprecated. Use packaging.version instead.",
+ "the imp module is deprecated in favour of importlib; "
+ "see the module's documentation for alternative uses",
+}
+
+# The set of warning messages generated by direct importing of some deprecated modules. We should only
+# ignore those messages when the warnings are generated directly by importlib - which means that
+# we imported it directly during module walk by the importlib library
+KNOWN_DEPRECATED_DIRECT_IMPORTS: Set[str] = {
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.batch`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.container_instance`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.container_registry`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.container_volume`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.cosmos`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.data_factory`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.data_lake`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.hooks.fileshare`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.operators.batch`.",
+ "This module is deprecated. "
+ "Please use `airflow.providers.microsoft.azure.operators.container_instances`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.operators.cosmos`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.secrets.key_vault`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.sensors.cosmos`.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.dynamodb`.",
+ "This module is deprecated. Please use `airflow.providers.microsoft.azure.transfers.local_to_wasb`.",
+ "This module is deprecated. Please use `airflow.providers.tableau.operators.tableau_refresh_workbook`.",
+ "This module is deprecated. Please use `airflow.providers.tableau.sensors.tableau_job_status`.",
+ "This module is deprecated. Please use `airflow.providers.tableau.hooks.tableau`.",
+ "This module is deprecated. Please use `kubernetes.client.models.V1Volume`.",
+ "This module is deprecated. Please use `kubernetes.client.models.V1VolumeMount`.",
+ (
+ "This module is deprecated. Please use `kubernetes.client.models.V1ResourceRequirements`"
+ " and `kubernetes.client.models.V1ContainerPort`."
+ ),
+ "This module is deprecated. Please use `kubernetes.client.models.V1EnvVar`.",
+ 'numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header,'
+ ' got 216 from PyObject',
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.step_function`.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.step_function`.",
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.ec2`.',
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.ec2`.',
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.s3`.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.s3`.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.dms`.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.dms`.",
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.emr`.',
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.emr`.',
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.redshift_cluster` "
+ "or `airflow.providers.amazon.aws.hooks.redshift_sql` as appropriate.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.operators.redshift_sql` "
+ "or `airflow.providers.amazon.aws.operators.redshift_cluster` as appropriate.",
+ "This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.redshift_cluster`.",
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.sagemaker`.',
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.sagemaker`.',
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.hooks.emr`.',
+ 'This module is deprecated. Please use `airflow.providers.opsgenie.hooks.opsgenie`.',
+ 'This module is deprecated. Please use `airflow.providers.opsgenie.operators.opsgenie`.',
+ 'This module is deprecated. Please use `airflow.hooks.redshift_sql` '
+ 'or `airflow.hooks.redshift_cluster` as appropriate.',
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.operators.redshift_sql` or '
+ '`airflow.providers.amazon.aws.operators.redshift_cluster` as appropriate.',
+ 'This module is deprecated. Please use `airflow.providers.amazon.aws.sensors.redshift_cluster`.',
+ "This module is deprecated. Please use airflow.providers.amazon.aws.transfers.sql_to_s3`.",
+ "This module is deprecated. Please use `airflow.providers.tableau.sensors.tableau`.",
+}
+
+
+def filter_known_warnings(warn: warnings.WarningMessage) -> bool:
+ msg_string = str(warn.message).replace("\n", " ")
+ for m in KNOWN_DEPRECATED_MESSAGES:
+ expected_package_string = "/" + m[1] + "/"
+ if msg_string == m[0] and warn.filename.find(expected_package_string) != -1:
+ return False
+ return True
+
+
+def filter_direct_importlib_warning(warn: warnings.WarningMessage) -> bool:
+ msg_string = str(warn.message).replace("\n", " ")
+ for m in KNOWN_DEPRECATED_DIRECT_IMPORTS:
+ if msg_string == m and warn.filename.find("/importlib/") != -1:
+ return False
+ return True
+
+
+def filter_known_common_deprecated_messages(warn: warnings.WarningMessage) -> bool:
+ msg_string = str(warn.message).replace("\n", " ")
+ for m in KNOWN_COMMON_DEPRECATED_MESSAGES:
+ if msg_string == m:
+ return False
+ return True
+
+
+def get_all_providers() -> List[str]:
+ """
+ Returns all providers for regular packages.
+ :return: list of providers that are considered for provider packages
+ """
+ from setup import PROVIDERS_REQUIREMENTS
+
+ return list(PROVIDERS_REQUIREMENTS.keys())
+
+
+def import_all_classes(
+ walkable_paths_and_prefixes: Dict[str, str],
+ prefix: str,
+ provider_ids: Optional[List[str]] = None,
+ print_imports: bool = False,
+ print_skips: bool = False,
+) -> Tuple[List[str], List[WarningMessage]]:
+ """
+ Imports all classes in providers packages. This method loads and imports
+ all the classes found in providers, so that we can find all the subclasses
+ of operators/sensors etc.
+
+ :param walkable_paths_and_prefixes: dict of paths with accompanying prefixes to look the provider
+ packages in
+ :param prefix: prefix to add
+ :param provider_ids - provider ids that should be loaded.
+ :param print_imports - if imported class should also be printed in output
+ :param print_skips - if skipped classes should also be printed in output
+ :return: tuple of list of all imported classes and all warnings generated
+ """
+ console.print()
+ console.print(f"Walking all package with prefixes in {walkable_paths_and_prefixes}")
+ console.print()
+ imported_classes = []
+ tracebacks: List[Tuple[str, str]] = []
+ printed_packages: Set[str] = set()
+
+ def mk_prefix(provider_id):
+ return f'{prefix}{provider_id}'
+
+ if provider_ids:
+ provider_prefixes = [mk_prefix(provider_id) for provider_id in provider_ids]
+ else:
+ provider_prefixes = [prefix]
+
+ def onerror(_):
+ nonlocal tracebacks
+ exception_string = traceback.format_exc()
+ for provider_prefix in provider_prefixes:
+ if provider_prefix in exception_string:
+ start_index = exception_string.find(provider_prefix)
+ end_index = exception_string.find("\n", start_index + len(provider_prefix))
+ package = exception_string[start_index:end_index]
+ tracebacks.append((package, exception_string))
+ break
+
+ all_warnings: List[WarningMessage] = []
+ for path, prefix in walkable_paths_and_prefixes.items():
+ for modinfo in pkgutil.walk_packages(path=[path], prefix=prefix, onerror=onerror):
+ if not any(modinfo.name.startswith(provider_prefix) for provider_prefix in provider_prefixes):
+ if print_skips:
+ console.print(f"Skipping module: {modinfo.name}")
+ continue
+ if print_imports:
+ package_to_print = ".".join(modinfo.name.split(".")[:-1])
+ if package_to_print not in printed_packages:
+ printed_packages.add(package_to_print)
+ console.print(f"Importing package: {package_to_print}")
+ try:
+ with warnings.catch_warnings(record=True) as w:
+ warnings.filterwarnings("always", category=DeprecationWarning)
+ _module = importlib.import_module(modinfo.name)
+ for attribute_name in dir(_module):
+ class_name = modinfo.name + "." + attribute_name
+ attribute = getattr(_module, attribute_name)
+ if isclass(attribute):
+ imported_classes.append(class_name)
+ if w:
+ all_warnings.extend(w)
+ except Exception:
+ exception_str = traceback.format_exc()
+ tracebacks.append((modinfo.name, exception_str))
+ if tracebacks:
+ console.print(
+ """
+[red]ERROR: There were some import errors[/]
+
+[yellow]If the job is about installing providers in 2.1.0, most likely you are using features that[/]
+[yellow]are not available in Airflow 2.1.0 and you mast implement them in backwards-compatible way![/]
+
+""",
+ )
+ console.print("[red]----------------------------------------[/]")
+ for package, trace in tracebacks:
+ console.print(f"Exception when importing: {package}\n\n")
+ console.print(trace)
+ console.print("[red]----------------------------------------[/]")
+ sys.exit(1)
+ else:
+ return imported_classes, all_warnings
+
+
+def get_provider_packages() -> List[str]:
+ """
+ Returns all provider packages.
+
+ """
+ from setup import PROVIDERS_REQUIREMENTS
+
+ return list(PROVIDERS_REQUIREMENTS.keys())
+
+
+def is_imported_from_same_module(the_class: str, imported_name: str) -> bool:
+ """
+ Is the class imported from another module?
+
+ :param the_class: the class object itself
+ :param imported_name: name of the imported class
+ :return: true if the class was imported from another module
+ """
+ return ".".join(imported_name.split(".")[:-1]) == the_class.__module__
+
+
+def is_example_dag(imported_name: str) -> bool:
+ """
+ Is the class an example_dag class?
+
+ :param imported_name: name where the class is imported from
+ :return: true if it is an example_dags class
+ """
+ return ".example_dags." in imported_name
+
+
+def is_from_the_expected_base_package(the_class: Type, expected_package: str) -> bool:
+ """
+ Returns true if the class is from the package expected.
+ :param the_class: the class object
+ :param expected_package: package expected for the class
+ :return:
+ """
+ return the_class.__module__.startswith(expected_package)
+
+
+def inherits_from(the_class: Type, expected_ancestor: Optional[Type] = None) -> bool:
+ """
+ Returns true if the class inherits (directly or indirectly) from the class specified.
+ :param the_class: The class to check
+ :param expected_ancestor: expected class to inherit from
+ :return: true is the class inherits from the class expected
+ """
+ if expected_ancestor is None:
+ return False
+ import inspect
+
+ mro = inspect.getmro(the_class)
+ return the_class is not expected_ancestor and expected_ancestor in mro
+
+
+def is_class(the_class: Type) -> bool:
+ """
+ Returns true if the object passed is a class
+ :param the_class: the class to pass
+ :return: true if it is a class
+ """
+ import inspect
+
+ return inspect.isclass(the_class)
+
+
+def package_name_matches(the_class: Type, expected_pattern: Optional[str] = None) -> bool:
+ """
+ In case expected_pattern is set, it checks if the package name matches the pattern.
+ .
+ :param the_class: imported class
+ :param expected_pattern: the pattern that should match the package
+ :return: true if the expected_pattern is None or the pattern matches the package
+ """
+ return expected_pattern is None or re.match(expected_pattern, the_class.__module__) is not None
+
+
+def convert_classes_to_table(entity_type: EntityType, entities: List[str], full_package_name: str) -> str:
+ """
+ Converts new entities to a Markdown table.
+
+ :param entity_type: entity type to convert to markup
+ :param entities: list of entities
+ :param full_package_name: name of the provider package
+ :return: table of new classes
+ """
+ from tabulate import tabulate
+
+ headers = [f"New Airflow 2.0 {entity_type.value.lower()}: `{full_package_name}` package"]
+ table = [(get_class_code_link(full_package_name, class_name, "main"),) for class_name in entities]
+ return tabulate(table, headers=headers, tablefmt="pipe")
+
+
+def get_details_about_classes(
+ entity_type: EntityType,
+ entities: Set[str],
+ wrong_entities: List[Tuple[type, str]],
+ full_package_name: str,
+) -> EntityTypeSummary:
+ """
+ Get details about entities.
+
+ :param entity_type: type of entity (Operators, Hooks etc.)
+ :param entities: set of entities found
+ :param wrong_entities: wrong entities found for that type
+ :param full_package_name: full package name
+ :return:
+ """
+ all_entities = list(entities)
+ all_entities.sort()
+ TOTALS[entity_type] += len(all_entities)
+ return EntityTypeSummary(
+ entities=all_entities,
+ new_entities_table=convert_classes_to_table(
+ entity_type=entity_type,
+ entities=all_entities,
+ full_package_name=full_package_name,
+ ),
+ wrong_entities=wrong_entities,
+ )
+
+
+def strip_package_from_class(base_package: str, class_name: str) -> str:
+ """
+ Strips base package name from the class (if it starts with the package name).
+ """
+ if class_name.startswith(base_package):
+ return class_name[len(base_package) + 1 :]
+ else:
+ return class_name
+
+
+def convert_class_name_to_url(base_url: str, class_name) -> str:
+ """
+ Converts the class name to URL that the class can be reached
+
+ :param base_url: base URL to use
+ :param class_name: name of the class
+ :return: URL to the class
+ """
+ return base_url + os.path.sep.join(class_name.split(".")[:-1]) + ".py"
+
+
+def get_class_code_link(base_package: str, class_name: str, git_tag: str) -> str:
+ """
+ Provides a Markdown link for the class passed as parameter.
+
+ :param base_package: base package to strip from most names
+ :param class_name: name of the class
+ :param git_tag: tag to use for the URL link
+ :return: URL to the class
+ """
+ url_prefix = f'https://github.com/apache/airflow/blob/{git_tag}/'
+ return (
+ f'[{strip_package_from_class(base_package, class_name)}]'
+ f'({convert_class_name_to_url(url_prefix, class_name)})'
+ )
+
+
+def print_wrong_naming(entity_type: EntityType, wrong_classes: List[Tuple[type, str]]):
+ """
+ Prints wrong entities of a given entity type if there are any
+ :param entity_type: type of the class to print
+ :param wrong_classes: list of wrong entities
+ """
+ if wrong_classes:
+ console.print(f"\n[red]There are wrongly named entities of type {entity_type}:[/]\n")
+ for wrong_entity_type, message in wrong_classes:
+ console.print(f"{wrong_entity_type}: {message}")
+
+
+def find_all_entities(
+ imported_classes: List[str],
+ base_package: str,
+ ancestor_match: Type,
+ sub_package_pattern_match: str,
+ expected_class_name_pattern: str,
+ unexpected_class_name_patterns: Set[str],
+ exclude_class_type: Optional[Type] = None,
+ false_positive_class_names: Optional[Set[str]] = None,
+) -> VerifiedEntities:
+ """
+ Returns set of entities containing all subclasses in package specified.
+
+ :param imported_classes: entities imported from providers
+ :param base_package: base package name where to start looking for the entities
+ :param sub_package_pattern_match: this string is expected to appear in the sub-package name
+ :param ancestor_match: type of the object the method looks for
+ :param expected_class_name_pattern: regexp of class name pattern to expect
+ :param unexpected_class_name_patterns: set of regexp of class name pattern that are not expected
+ :param exclude_class_type: exclude class of this type (Sensor are also Operators, so
+ they should be excluded from the list)
+ :param false_positive_class_names: set of class names that are wrongly recognised as badly named
+ """
+ found_entities: Set[str] = set()
+ wrong_entities: List[Tuple[type, str]] = []
+ for imported_name in imported_classes:
+ module, class_name = imported_name.rsplit(".", maxsplit=1)
+ the_class = getattr(importlib.import_module(module), class_name)
+ if (
+ is_class(the_class=the_class)
+ and not is_example_dag(imported_name=imported_name)
+ and is_from_the_expected_base_package(the_class=the_class, expected_package=base_package)
+ and is_imported_from_same_module(the_class=the_class, imported_name=imported_name)
+ and inherits_from(the_class=the_class, expected_ancestor=ancestor_match)
+ and not inherits_from(the_class=the_class, expected_ancestor=exclude_class_type)
+ and package_name_matches(the_class=the_class, expected_pattern=sub_package_pattern_match)
+ ):
+
+ if not false_positive_class_names or class_name not in false_positive_class_names:
+ if not re.match(expected_class_name_pattern, class_name):
+ wrong_entities.append(
+ (
+ the_class,
+ f"The class name {class_name} is wrong. "
+ f"It should match {expected_class_name_pattern}",
+ )
+ )
+ continue
+ if unexpected_class_name_patterns:
+ for unexpected_class_name_pattern in unexpected_class_name_patterns:
+ if re.match(unexpected_class_name_pattern, class_name):
+ wrong_entities.append(
+ (
+ the_class,
+ f"The class name {class_name} is wrong. "
+ f"It should not match {unexpected_class_name_pattern}",
+ )
+ )
+ continue
+ found_entities.add(imported_name)
+ return VerifiedEntities(all_entities=found_entities, wrong_entities=wrong_entities)
+
+
+def get_package_class_summary(
+ full_package_name: str, imported_classes: List[str]
+) -> Dict[EntityType, EntityTypeSummary]:
+ """
+ Gets summary of the package in the form of dictionary containing all types of entities
+ :param full_package_name: full package name
+ :param imported_classes: entities imported_from providers
+ :return: dictionary of objects usable as context for JINJA2 templates - or None if there are some errors
+ """
+ from airflow.hooks.base import BaseHook
+ from airflow.models.baseoperator import BaseOperator
+ from airflow.secrets import BaseSecretsBackend
+ from airflow.sensors.base import BaseSensorOperator
+
+ all_verified_entities: Dict[EntityType, VerifiedEntities] = {
+ EntityType.Operators: find_all_entities(
+ imported_classes=imported_classes,
+ base_package=full_package_name,
+ sub_package_pattern_match=r".*\.operators\..*",
+ ancestor_match=BaseOperator,
+ expected_class_name_pattern=OPERATORS_PATTERN,
+ unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN},
+ exclude_class_type=BaseSensorOperator,
+ false_positive_class_names={
+ 'CloudVisionAddProductToProductSetOperator',
+ 'CloudDataTransferServiceGCSToGCSOperator',
+ 'CloudDataTransferServiceS3ToGCSOperator',
+ 'BigQueryCreateDataTransferOperator',
+ 'CloudTextToSpeechSynthesizeOperator',
+ 'CloudSpeechToTextRecognizeSpeechOperator',
+ },
+ ),
+ EntityType.Sensors: find_all_entities(
+ imported_classes=imported_classes,
+ base_package=full_package_name,
+ sub_package_pattern_match=r".*\.sensors\..*",
+ ancestor_match=BaseSensorOperator,
+ expected_class_name_pattern=SENSORS_PATTERN,
+ unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN, SENSORS_PATTERN},
+ ),
+ EntityType.Hooks: find_all_entities(
+ imported_classes=imported_classes,
+ base_package=full_package_name,
+ sub_package_pattern_match=r".*\.hooks\..*",
+ ancestor_match=BaseHook,
+ expected_class_name_pattern=HOOKS_PATTERN,
+ unexpected_class_name_patterns=ALL_PATTERNS - {HOOKS_PATTERN},
+ ),
+ EntityType.Secrets: find_all_entities(
+ imported_classes=imported_classes,
+ sub_package_pattern_match=r".*\.secrets\..*",
+ base_package=full_package_name,
+ ancestor_match=BaseSecretsBackend,
+ expected_class_name_pattern=SECRETS_PATTERN,
+ unexpected_class_name_patterns=ALL_PATTERNS - {SECRETS_PATTERN},
+ ),
+ EntityType.Transfers: find_all_entities(
+ imported_classes=imported_classes,
+ base_package=full_package_name,
+ sub_package_pattern_match=r".*\.transfers\..*",
+ ancestor_match=BaseOperator,
+ expected_class_name_pattern=TRANSFERS_PATTERN,
+ unexpected_class_name_patterns=ALL_PATTERNS - {OPERATORS_PATTERN, TRANSFERS_PATTERN},
+ ),
+ }
+ for entity in EntityType:
+ print_wrong_naming(entity, all_verified_entities[entity].wrong_entities)
+
+ entities_summary: Dict[EntityType, EntityTypeSummary] = {}
+
+ for entity_type in EntityType:
+ entities_summary[entity_type] = get_details_about_classes(
+ entity_type,
+ all_verified_entities[entity_type].all_entities,
+ all_verified_entities[entity_type].wrong_entities,
+ full_package_name,
+ )
+
+ return entities_summary
+
+
+def is_camel_case_with_acronyms(s: str):
+ """
+ Checks if the string passed is Camel Case (with capitalised acronyms allowed).
+ :param s: string to check
+ :return: true if the name looks cool as Class name.
+ """
+ return s != s.lower() and s != s.upper() and "_" not in s and s[0].upper() == s[0]
+
+
+def check_if_classes_are_properly_named(
+ entity_summary: Dict[EntityType, EntityTypeSummary]
+) -> Tuple[int, int]:
+ """
+ Check if all entities in the dictionary are named properly. It prints names at the output
+ and returns the status of class names.
+
+ :param entity_summary: dictionary of class names to check, grouped by types.
+ :return: Tuple of 2 ints = total number of entities and number of badly named entities
+ """
+ total_class_number = 0
+ badly_named_class_number = 0
+ for entity_type, class_suffix in EXPECTED_SUFFIXES.items():
+ for class_full_name in entity_summary[entity_type].entities:
+ _, class_name = class_full_name.rsplit(".", maxsplit=1)
+ error_encountered = False
+ if not is_camel_case_with_acronyms(class_name):
+ console.print(
+ f"[red]The class {class_full_name} is wrongly named. The "
+ f"class name should be CamelCaseWithACRONYMS ![/]"
+ )
+ error_encountered = True
+ if not class_name.endswith(class_suffix):
+ console.print(
+ f"[red]The class {class_full_name} is wrongly named. It is one of the {entity_type.value}"
+ f" so it should end with {class_suffix}[/]"
+ )
+ error_encountered = True
+ total_class_number += 1
+ if error_encountered:
+ badly_named_class_number += 1
+ return total_class_number, badly_named_class_number
+
+
+def verify_provider_classes_for_single_provider(imported_classes: List[str], provider_package_id: str):
+ """Verify naming of provider classes for single provider."""
+ full_package_name = f"airflow.providers.{provider_package_id}"
+ entity_summaries = get_package_class_summary(full_package_name, imported_classes)
+ total, bad = check_if_classes_are_properly_named(entity_summaries)
+ bad += sum(len(entity_summary.wrong_entities) for entity_summary in entity_summaries.values())
+ if bad != 0:
+ console.print()
+ console.print(f"[red]There are {bad} errors of {total} entities for {provider_package_id}[/]")
+ console.print()
+ return total, bad
+
+
+def summarise_total_vs_bad_and_warnings(total: int, bad: int, warns: List[warnings.WarningMessage]) -> bool:
+ """Summarises Bad/Good class names for providers and warnings"""
+ raise_error = False
+ if bad == 0:
+ console.print()
+ console.print(f"[green]OK: All {total} entities are properly named[/]")
+ console.print()
+ console.print("Totals:")
+ console.print()
+ for entity in EntityType:
+ console.print(f"{entity.value}: {TOTALS[entity]}")
+ console.print()
+ else:
+ console.print()
+ console.print(
+ f"[red]ERROR! There are in total: {bad} entities badly named out of {total} entities[/]"
+ )
+ console.print()
+ raise_error = True
+ if warns:
+ console.print()
+ console.print("[red]Unknown warnings generated:[/]")
+ console.print()
+ for w in warns:
+ one_line_message = str(w.message).replace('\n', ' ')
+ console.print(f"{w.filename}:{w.lineno}:[yellow]{one_line_message}[/]")
+ console.print()
+ console.print(f"[red]ERROR! There were {len(warns)} warnings generated during the import[/]")
+ console.print()
+ console.print("[yellow]Ideally, fix it, so that no warnings are generated during import.[/]")
+ console.print("[yellow]There are three cases that are legitimate deprecation warnings though:[/]")
+ console.print("[yellow] 1) when you deprecate whole module or class and replace it in provider[/]")
+ console.print("[yellow] 2) when 3rd-party module generates Deprecation and you cannot upgrade it[/]")
+ console.print(
+ "[yellow] 3) when many 3rd-party module generates same Deprecation warning that "
+ "comes from another common library[/]"
+ )
+ console.print()
+ console.print(
+ "[yellow]In case 1), add the deprecation message to "
+ "the KNOWN_DEPRECATED_DIRECT_IMPORTS in ./scripts/ci/in_container/verify_providers.py[/]"
+ )
+ console.print(
+ "[yellow]In case 2), add the deprecation message together with module it generates to "
+ "the KNOWN_DEPRECATED_MESSAGES in ./scripts/ci/in_container/verify_providers.py[/]"
+ )
+ console.print(
+ "[yellow]In case 3), add the deprecation message to "
+ "the KNOWN_COMMON_DEPRECATED_MESSAGES in ./scripts/ci/in_container/verify_providers.py[/]"
+ )
+ console.print()
+ raise_error = True
+ else:
+ console.print()
+ console.print("[green]OK: No warnings generated[/]")
+ console.print()
+
+ if raise_error:
+ console.print("[red]Please fix the problems listed above [/]")
+ return False
+ return True
+
+
+def get_providers_paths() -> List[str]:
+ import airflow.providers
+
+ # handle providers in sources
+ paths = [str(PROVIDERS_PATH)]
+ # as well as those installed via packages
+ paths.extend(airflow.providers.__path__) # type: ignore[attr-defined]
+ return paths
+
+
+def add_all_namespaced_packages(
+ walkable_paths_and_prefixes: Dict[str, str], provider_path: str, provider_prefix: str
+):
+ """
+ We need to find namespace packages ourselves as "walk_packages" does not support namespaced packages
+ # and PEP420
+
+ :param walkable_paths_and_prefixes: pats
+ :param provider_path:
+ :param provider_prefix:
+ """
+ main_path = Path(provider_path).absolute()
+ for candidate_path in main_path.rglob("*"):
+ if candidate_path.name == "__pycache__":
+ continue
+ if candidate_path.is_dir() and not (candidate_path / "__init__.py").exists():
+ subpackage = str(candidate_path.relative_to(main_path)).replace(os.sep, ".")
+ walkable_paths_and_prefixes[str(candidate_path)] = provider_prefix + subpackage + "."
+
+
+def verify_provider_classes():
+ provider_ids = get_all_providers()
+ walkable_paths_and_prefixes: Dict[str, str] = {}
+ provider_prefix = "airflow.providers."
+ for provider_path in get_providers_paths():
+ walkable_paths_and_prefixes[provider_path] = provider_prefix
+ add_all_namespaced_packages(walkable_paths_and_prefixes, provider_path, provider_prefix)
+ imported_classes, warns = import_all_classes(
+ walkable_paths_and_prefixes=walkable_paths_and_prefixes,
+ provider_ids=provider_ids,
+ print_imports=True,
+ prefix="airflow.providers.",
+ )
+ total = 0
+ bad = 0
+ for provider_package_id in provider_ids:
+ inc_total, inc_bad = verify_provider_classes_for_single_provider(
+ imported_classes, provider_package_id
+ )
+ total += inc_total
+ bad += inc_bad
+ warns = list(filter(filter_known_warnings, warns))
+ warns = list(filter(filter_direct_importlib_warning, warns))
+ warns = list(filter(filter_known_common_deprecated_messages, warns))
+ if not summarise_total_vs_bad_and_warnings(total, bad, warns):
+ sys.exit(1)
+
+ if len(imported_classes) == 0:
+ console.print("[red]Something is seriously wrong - no classes imported[/]")
+ sys.exit(1)
+ if warns:
+ console.print("[yellow]There were warnings generated during the import[/]")
+ for w in warns:
+ one_line_message = str(w.message).replace('\n', ' ')
+ print(f"[yellow]{w.filename}:{w.lineno}: {one_line_message}[/]")
+
+ console.print()
+ console.print(
+ "[green]SUCCESS: All provider packages are importable and no unknown warnings generated![/]\n"
+ )
+ console.print(f"Imported {len(imported_classes)} classes.")
+ console.print()
+
+
+def run_provider_discovery():
+ console.print("[bright_blue]List all providers[/]\n")
+ subprocess.run(["airflow", "providers", "list"], check=True)
+ console.print("[bright_blue]List all hooks[/]\n")
+ subprocess.run(["airflow", "providers", "hooks"], check=True)
+ console.print("[bright_blue]List all behaviours[/]\n")
+ subprocess.run(["airflow", "providers", "behaviours"], check=True)
+ console.print("[bright_blue]List all widgets[/]\n")
+ subprocess.run(["airflow", "providers", "widgets"], check=True)
+ console.print("[bright_blue]List all extra links[/]\n")
+ subprocess.run(["airflow", "providers", "links"], check=True)
+ if os.environ.get("USE_AIRFLOW_VERSION", None) == "2.1.0":
+ console.print("[info]Skip commands not available in Airflow 2.1.0[/]")
+ else:
+ console.print("[bright_blue]List all logging[/]\n")
+ subprocess.run(["airflow", "providers", "logging"], check=True)
+ console.print("[bright_blue]List all secrets[/]\n")
+ subprocess.run(["airflow", "providers", "secrets"], check=True)
+ console.print("[bright_blue]List all auth backends[/]\n")
+ subprocess.run(["airflow", "providers", "auth"], check=True)
+
+
+if __name__ == '__main__':
+ sys.path.insert(0, str(AIRFLOW_SOURCES_ROOT))
+ verify_provider_classes()
+ run_provider_discovery()
[airflow] 01/07: Mark image as refreshed when pulled on CI (#23410)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2a835cd134df5e8e44c805240d71bda638a1f87c
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Mon May 2 18:13:44 2022 +0200
Mark image as refreshed when pulled on CI (#23410)
One of the recent changes in Breeze (#23395) caused unnecessary
rebuilding of image when "build-docs" is run. On CI we build
image once and reuse it. However in case of Buld docs we missed
information that the image is "fresh" and we started rebuilding
it. This change marks the image as "refreshed" when it is pulled
with `--tag-as-latest` flag (which happens in CI).
(cherry picked from commit ce4a60ce279f691f428aa5f671180407e722ccbf)
---
.../build_image/ci/build_ci_image.py | 12 +++--
.../src/airflow_breeze/commands/common_options.py | 3 +-
dev/breeze/src/airflow_breeze/utils/pulll_image.py | 5 +-
images/breeze/output-pull-image.svg | 55 +++++++++++-----------
images/breeze/output-pull-prod-image.svg | 55 +++++++++++-----------
5 files changed, 70 insertions(+), 60 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
index 9c698a8dab..d2c12a23bf 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
@@ -177,10 +177,7 @@ def build_ci_image(
)
if not dry_run:
if build_result.returncode == 0:
- ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
- ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
- touch_cache_file(f"built_{ci_image_params.python}", root_dir=ci_image_cache_dir)
- calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir, update=True)
+ mark_image_as_refreshed(ci_image_params)
else:
get_console().print("[error]Error when building image![/]")
return (
@@ -194,6 +191,13 @@ def build_ci_image(
return build_result.returncode, f"Image build: {ci_image_params.python}"
+def mark_image_as_refreshed(ci_image_params: BuildCiParams):
+ ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
+ ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
+ touch_cache_file(f"built_{ci_image_params.python}", root_dir=ci_image_cache_dir)
+ calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir, update=True)
+
+
def build_ci_image_in_parallel(
verbose: bool, dry_run: bool, parallelism: int, python_version_list: List[str], **kwargs
):
diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py
index a2d2e8c0f9..2b0831e44a 100644
--- a/dev/breeze/src/airflow_breeze/commands/common_options.py
+++ b/dev/breeze/src/airflow_breeze/commands/common_options.py
@@ -294,7 +294,8 @@ option_wait_for_image = click.option(
)
option_tag_as_latest = click.option(
'--tag-as-latest',
- help='Tags the image as latest after pulling.',
+ help='Tags the image as latest and update checksum of all files after pulling. '
+ 'Used in CI to pull the image built in another job.',
is_flag=True,
envvar='TAG_AS_LATEST',
)
diff --git a/dev/breeze/src/airflow_breeze/utils/pulll_image.py b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
index 357fa1b77e..28179a2329 100644
--- a/dev/breeze/src/airflow_breeze/utils/pulll_image.py
+++ b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
@@ -19,6 +19,7 @@ import multiprocessing as mp
import time
from typing import List, Tuple, Union
+from airflow_breeze.build_image.ci.build_ci_image import mark_image_as_refreshed
from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
from airflow_breeze.utils.console import get_console
@@ -80,7 +81,7 @@ def run_pull_image(
poll_time: float,
) -> Tuple[int, str]:
"""
- Pull image soecified.
+ Pull image specified.
:param image_params: Image parameters.
:param dry_run: whether it's dry run
:param verbose: whether it's verbose
@@ -138,6 +139,8 @@ def run_pull_image(
dry_run=dry_run,
check=False,
)
+ if command_result.returncode == 0 and isinstance(image_params, BuildCiParams):
+ mark_image_as_refreshed(image_params)
return command_result.returncode, f"Image Python {image_params.python}"
if wait_for_image:
if verbose or dry_run:
diff --git a/images/breeze/output-pull-image.svg b/images/breeze/output-pull-image.svg
index b432347677..ed075ebadc 100644
--- a/images/breeze/output-pull-image.svg
+++ b/images/breeze/output-pull-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="934" viewBox="0 0 1720.0 934"
+<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-2708726962-terminal-wrapper span {
+ .rich-svg-921580930-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-2708726962-terminal-wrapper a {
+ .rich-svg-921580930-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-2708726962-terminal-body .blink {
- animation: rich-svg-2708726962-blinker 1s infinite;
+ .rich-svg-921580930-terminal-body .blink {
+ animation: rich-svg-921580930-blinker 1s infinite;
}
- @keyframes rich-svg-2708726962-blinker {
+ @keyframes rich-svg-921580930-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-2708726962-terminal-wrapper {
+ .rich-svg-921580930-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-2708726962-terminal {
+ .rich-svg-921580930-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-2708726962-terminal:after {
+ .rich-svg-921580930-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-2708726962-terminal-header {
+ .rich-svg-921580930-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-2708726962-terminal-title-tab {
+ .rich-svg-921580930-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-2708726962-terminal-traffic-lights {
+ .rich-svg-921580930-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-2708726962-terminal-body {
+ .rich-svg-921580930-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-2708726962-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-921580930-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-2708726962-terminal-wrapper">
- <div class="rich-svg-2708726962-terminal">
- <div class="rich-svg-2708726962-terminal-header">
- <svg class="rich-svg-2708726962-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-921580930-terminal-wrapper">
+ <div class="rich-svg-921580930-terminal">
+ <div class="rich-svg-921580930-terminal-header">
+ <svg class="rich-svg-921580930-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-2708726962-terminal-title-tab">Command: pull-image</div>
+ <div class="rich-svg-921580930-terminal-title-tab">Command: pull-image</div>
</div>
- <div class="rich-svg-2708726962-terminal-body">
+ <div class="rich-svg-921580930-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze pull-image [OPTIONS] [EXTRA_PYTEST_ARGS]... </span></div>
<div><span class="r2"> </span></div>
@@ -126,7 +126,8 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verify-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Verify image. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--wait-for-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Wait until image is available. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Tags the image as latest after pulling. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Tags the image as latest and update checksum of all files after pulling. Used in CI to pull </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r1">the image built in another job. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--run-in-parallel</span><span class="r1"> </span><span class="r1">Run the operation in parallel on all or selected subset of Python versions. </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-pull-prod-image.svg b/images/breeze/output-pull-prod-image.svg
index a12f75146f..0253a869fa 100644
--- a/images/breeze/output-pull-prod-image.svg
+++ b/images/breeze/output-pull-prod-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="934" viewBox="0 0 1720.0 934"
+<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-3990808719-terminal-wrapper span {
+ .rich-svg-1620392287-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-3990808719-terminal-wrapper a {
+ .rich-svg-1620392287-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-3990808719-terminal-body .blink {
- animation: rich-svg-3990808719-blinker 1s infinite;
+ .rich-svg-1620392287-terminal-body .blink {
+ animation: rich-svg-1620392287-blinker 1s infinite;
}
- @keyframes rich-svg-3990808719-blinker {
+ @keyframes rich-svg-1620392287-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-3990808719-terminal-wrapper {
+ .rich-svg-1620392287-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-3990808719-terminal {
+ .rich-svg-1620392287-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-3990808719-terminal:after {
+ .rich-svg-1620392287-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-3990808719-terminal-header {
+ .rich-svg-1620392287-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-3990808719-terminal-title-tab {
+ .rich-svg-1620392287-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-3990808719-terminal-traffic-lights {
+ .rich-svg-1620392287-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-3990808719-terminal-body {
+ .rich-svg-1620392287-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-3990808719-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
+ .rich-svg-1620392287-terminal-body .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r7 {color: #78780e; text-decoration-color: #78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml">
- <div class="rich-svg-3990808719-terminal-wrapper">
- <div class="rich-svg-3990808719-terminal">
- <div class="rich-svg-3990808719-terminal-header">
- <svg class="rich-svg-3990808719-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <div class="rich-svg-1620392287-terminal-wrapper">
+ <div class="rich-svg-1620392287-terminal">
+ <div class="rich-svg-1620392287-terminal-header">
+ <svg class="rich-svg-1620392287-terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div class="rich-svg-3990808719-terminal-title-tab">Command: pull-prod-image</div>
+ <div class="rich-svg-1620392287-terminal-title-tab">Command: pull-prod-image</div>
</div>
- <div class="rich-svg-3990808719-terminal-body">
+ <div class="rich-svg-1620392287-terminal-body">
<div><span class="r2"> </span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze pull-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]... </span></div>
<div><span class="r2"> </span></div>
@@ -126,7 +126,8 @@
<div><span class="r4">│</span><span class="r1"> </span><span class="r4">[default: 3.7] </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verify-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Verify image. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--wait-for-image</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Wait until image is available. </span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Tags the image as latest after pulling. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r1">Tags the image as latest and update checksum of all files after pulling. Used in CI to pull </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r1">the image built in another job. </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--run-in-parallel</span><span class="r1"> </span><span class="r1">Run the operation in parallel on all or selected subset of Python versions. </span><span class="r1"> </span><span class="r4">│</span></div>
[airflow] 04/07: Move non-opencontainer labeling of the image to breeze from Dockerfile (#23379)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6a5c04ce254d237122571359229b3704df2665c6
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Wed May 4 00:15:52 2022 +0200
Move non-opencontainer labeling of the image to breeze from Dockerfile (#23379)
* Extract "extra" labeling of the image to breeze from Dockerfile
Fixes: #21046
* Add more ArtifictHub-specific labels
Co-authored-by: Kamil Breguła <ka...@apache.org>
(cherry picked from commit 5b1ab96865a6e8f18784f88f88f8b17981450cc6)
---
Dockerfile | 5 +---
.../build_image/prod/build_prod_params.py | 35 ++++++++++++++++++++--
2 files changed, 33 insertions(+), 7 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index cab6cbadff..8a156174a4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1571,9 +1571,6 @@ LABEL org.apache.airflow.distro="debian" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.ref.name="airflow" \
org.opencontainers.image.title="Production Airflow Image" \
- org.opencontainers.image.description="Reference, production-ready Apache Airflow image" \
- io.artifacthub.package.license='Apache-2.0' \
- io.artifacthub.package.readme-url='${AIRFLOW_IMAGE_README_URL}'
-
+ org.opencontainers.image.description="Reference, production-ready Apache Airflow image"
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
CMD []
diff --git a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
index aa781b486a..0260907fbd 100644
--- a/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
+++ b/dev/breeze/src/airflow_breeze/build_image/prod/build_prod_params.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
"""Parameters to build PROD image."""
+import json
import os
import re
import sys
@@ -53,7 +54,6 @@ class BuildProdParams:
airflow_is_in_context: bool = False
install_packages_from_context: bool = False
upgrade_to_newer_dependencies: bool = False
- airflow_version: str = get_airflow_version()
python: str = "3.7"
airflow_branch_for_pypi_preloading: str = AIRFLOW_BRANCH
install_airflow_reference: str = ""
@@ -87,6 +87,13 @@ class BuildProdParams:
debian_version: str = "bullseye"
answer: Optional[str] = None
+ @property
+ def airflow_version(self) -> str:
+ if self.install_airflow_version:
+ return self.install_airflow_version
+ else:
+ return get_airflow_version()
+
@property
def airflow_branch(self) -> str:
return self.airflow_branch_for_pypi_preloading
@@ -172,7 +179,6 @@ class BuildProdParams:
]
)
extra_build_flags.extend(self.args_for_remote_install)
- self.airflow_version = self.install_airflow_reference
elif len(self.install_airflow_version) > 0:
if not re.match(r'^[0-9\.]+((a|b|rc|alpha|beta|pre)[0-9]+)?$', self.install_airflow_version):
get_console().print(
@@ -185,8 +191,15 @@ class BuildProdParams:
["--build-arg", f"AIRFLOW_VERSION_SPECIFICATION==={self.install_airflow_version}"]
)
extra_build_flags.extend(["--build-arg", f"AIRFLOW_VERSION={self.install_airflow_version}"])
+ constraints_base = (
+ f"https://raw.githubusercontent.com/{self.github_repository}/"
+ f"{self.airflow_constraints_reference}"
+ )
+ constraints_location = (
+ f"{constraints_base}/constraints-{self.install_airflow_version}/constraints-{self.python}.txt"
+ )
+ self.airflow_constraints_location = constraints_location
extra_build_flags.extend(self.args_for_remote_install)
- self.airflow_version = self.install_airflow_version
else:
extra_build_flags.extend(
[
@@ -204,6 +217,22 @@ class BuildProdParams:
f"AIRFLOW_CONSTRAINTS_REFERENCE={self.default_constraints_branch}",
]
)
+
+ maintainers = json.dumps([{"name": "Apache Airflow PMC", "email": "dev@airflow.apache.org"}])
+ logo_url = "https://github.com/apache/airflow/raw/main/docs/apache-airflow/img/logos/wordmark_1.png"
+ readme_url = "https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md"
+ extra_build_flags.extend(
+ [
+ "--label",
+ "io.artifacthub.package.license=Apache-2.0",
+ "--label",
+ f"io.artifacthub.package.readme-url={readme_url}",
+ "--label",
+ f"io.artifacthub.package.maintainers={maintainers}",
+ "--label",
+ f"io.artifacthub.package.logo-url={logo_url}",
+ ]
+ )
return extra_build_flags
@property
[airflow] 03/07: Unify approach for user questions asked in Breeze (#23335)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ffe5526c6a699557796a1347f7f662bab53682d8
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue May 3 22:22:42 2022 +0200
Unify approach for user questions asked in Breeze (#23335)
This change documents and unifies the approach we've taken for
the user inut handling when it comes to confirmation questions.
(cherry picked from commit ebc1f14db3a1b14f2535462e97a6407f48b19f7c)
---
breeze | 14 ++-
.../doc/adr/0012-asking-user-for-confirmation.md | 99 ++++++++++++++++++++++
.../build_image/ci/build_ci_image.py | 12 ++-
.../src/airflow_breeze/commands/ci_image_tools.py | 2 -
.../src/airflow_breeze/commands/common_options.py | 9 +-
.../commands/configuration_and_maintenance.py | 21 +++--
.../airflow_breeze/commands/custom_param_types.py | 13 +++
.../airflow_breeze/commands/developer_commands.py | 3 -
.../commands/production_image_tools.py | 2 -
.../airflow_breeze/commands/release_management.py | 6 +-
dev/breeze/src/airflow_breeze/utils/confirm.py | 42 ++++++---
.../src/airflow_breeze/utils/md5_build_check.py | 3 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 13 +++
dev/breeze/src/airflow_breeze/utils/reinstall.py | 11 ++-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 4 +-
15 files changed, 205 insertions(+), 49 deletions(-)
diff --git a/breeze b/breeze
index b433fac7f9..d84f4bb8f0 100755
--- a/breeze
+++ b/breeze
@@ -21,19 +21,22 @@ MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
BREEZE_BINARY=breeze
COLOR_RED=$'\e[31m'
+COLOR_BLUE=$'\e[34m'
COLOR_RESET=$'\e[0m'
COLOR_YELLOW=$'\e[33m'
function manual_instructions() {
- echo "Please run those commands manually (you might need to restart shell between them):"
+ echo
+ echo "${COLOR_BLUE}Please run those commands manually (you might need to restart shell between them):i${COLOR_RESET}"
echo
echo " pip -m install pipx"
echo " pipx ensurepath"
echo " pipx install -e '${MY_DIR}/dev/breeze/'"
echo " breeze setup-autocomplete --force"
echo
- echo " After that, both pipx and breeze should be available ono your path"
- exit
+ echo " After that, both pipx and breeze should be available on your path"
+ echo
+ exit 1
}
function check_breeze_installed() {
@@ -61,7 +64,10 @@ function check_breeze_installed() {
if [[ ${breeze_on_path} != 0 ]]; then
echo "${COLOR_RED}The '${BREEZE_BINARY}' is not on path. Breeze should be installed and 'breeze' should be available on your PATH!${COLOR_RESET}"
export TIMEOUT=0
- if "${MY_DIR}/scripts/tools/confirm" "Installing breeze?"; then
+ echo
+ echo "${COLOR_YELLOW}Installing Breeze. This will install breeze via pipx and modify your local rc file for ${SHELL}${COLOR_RESET}"
+ echo
+ if "${MY_DIR}/scripts/tools/confirm" "installing and modifying the startup scripts"; then
pipx ensurepath --force
pipx install -e "${MY_DIR}/dev/breeze/" --force
${BREEZE_BINARY} setup-autocomplete --force --answer yes
diff --git a/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md b/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md
new file mode 100644
index 0000000000..6b63780b23
--- /dev/null
+++ b/dev/breeze/doc/adr/0012-asking-user-for-confirmation.md
@@ -0,0 +1,99 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [12. Asking user for confirmation](#12-asking-user-for-confirmation)
+ - [Status](#status)
+ - [Context](#context)
+ - [Decision](#decision)
+ - [Consequences](#consequences)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# 12. Asking user for confirmation
+
+Date: 2022-04-28
+
+## Status
+
+Accepted
+
+## Context
+
+When we run a number of breeze commands, we sometimes ask the user for confirmation. This happens in a few
+cases:
+
+1. When the user attempts to run a potentially disruptive operation (cleanup, delete, modifying startup
+ scripts)
+2. When the user attempts to run a potentially long operation or operation that needs fast internet access
+3. When an action generates an auxiliary action that we determined we need to run in order to make sure
+ we use the latest version of the environment (like upgrading breeze dependencies or pulling and building
+ latest image).
+
+In case 1) - we should present the user with information on what is going to change and the user should
+not be surprised with the effect of it.
+
+In case 2) - we should not break the workflow of the user, unless this is really necessary or the user
+chooses to break his workflow. This is important in situation when the user needs to perform an action
+quickly and disrupting the workflow would be annoying and undesired. In this case the confirmation of
+the "long" action should not be "necessary" - when user does not expect the action to wait for confirmation.
+
+In case 3) - we want to gently nudge the user to run an action when we guess it is advised. We should not
+break the regular workflows, but we want the user to "eventually" run the upgrade to use the same
+environment as others (and avoiding the problem where thing "work for me but not the others".
+
+Breeze is also used for unattended actions - and then any questions would prevent the actions from running,
+so it should be possible to disable the questions and assume an answer in those cases.
+
+## Decision
+
+We take the following approach with the confirmation:
+
+1) If there is a potentially disruptive command - we always ask the user for confirmation and explicit
+confirmation is necessary. Default action in this case is "quit" so if the user just presses enter
+the operation should quit without making any change.
+
+2) If there is a long-running (but not disruptive) operation that user directly requested (for example
+building and image by `build-image` command), there should be no question asked unless there is some
+condition that could change the length of the action (for example when we realise that the user have not
+rebased to the latest main and rebuilding the image might take far longer than initially anticipated by
+the user). In case of such unexpected condition the default answer should be "quit" as well without timeout.
+
+3) If there is an auxiliary actions (for example building image) that would prevent (or delay) another action
+of the user - the default action should be "no" (no upgrade) and there should be a short timeout (few seconds)
+so that the original action can be executed without too much waiting.
+In all cases an upgrade needs an explicit 'y' answer by the user.
+
+Pressing "ENTER" when asked triggers the default action if it is specified.
+
+User can "force" any answer by adding `--answer <ANSWER>` command line switch to any command line that
+might ask a question - and it will force `<ANSWER>` as an immediate action to all questions.
+
+Default action (if used) is capitalised in allowed answers. Timeout is displayed when present.
+
+## Consequences
+
+As a consequence, the user will not accidentally make any disruptive changes without confirming it. The user
+will not have a broken workflow even if upgrade is recommended - they will just get a short annoying delay and
+question asked whether to upgrade, but the upgrade will not run without the user's consent. The short
+delay will introduce a gentle "nagging" of the user to upgrade their breeze or image which will lead to
+"eventually consistent" versions of both for the users.
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
index d2c12a23bf..3b09d61602 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
@@ -27,7 +27,7 @@ from airflow_breeze.build_image.ci.build_ci_params import (
)
from airflow_breeze.utils.cache import touch_cache_file
from airflow_breeze.utils.ci_group import ci_group
-from airflow_breeze.utils.confirm import Answer, user_confirm
+from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
construct_docker_build_command,
@@ -69,16 +69,20 @@ def should_we_run_the_build(build_ci_params: BuildCiParams, verbose: bool) -> bo
):
return False
try:
- answer = user_confirm(message="Do you want to build image?", timeout=5, default_answer=Answer.NO)
+ answer = user_confirm(
+ message="Do you want to build the image?", timeout=STANDARD_TIMEOUT, default_answer=Answer.NO
+ )
if answer == answer.YES:
if is_repo_rebased(build_ci_params.github_repository, build_ci_params.airflow_branch):
return True
else:
get_console().print(
- "\n[warning]This might take a lot of time, w" "e think you should rebase first.[/]\n"
+ "\n[warning]This might take a lot of time, we think you should rebase first.[/]\n"
)
answer = user_confirm(
- "But if you really, really want - you can do it", timeout=5, default_answer=Answer.NO
+ "But if you really, really want - you can do it. Are you really sure?",
+ timeout=STANDARD_TIMEOUT,
+ default_answer=Answer.NO,
)
if answer == Answer.YES:
return True
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py b/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
index 15fd2d52f5..90761417ea 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_image_tools.py
@@ -65,7 +65,6 @@ from airflow_breeze.commands.common_options import (
)
from airflow_breeze.commands.main import main
from airflow_breeze.utils.ci_group import ci_group
-from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
from airflow_breeze.utils.python_versions import get_python_version_list
@@ -219,7 +218,6 @@ def build_image(
get_console().print(f"[error]Error when building image! {info}")
sys.exit(return_code)
- set_forced_answer(answer)
parameters_passed = filter_out_none(**kwargs)
if build_multiple_images:
with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py
index 2b0831e44a..7193743fad 100644
--- a/dev/breeze/src/airflow_breeze/commands/common_options.py
+++ b/dev/breeze/src/airflow_breeze/commands/common_options.py
@@ -19,7 +19,12 @@ import multiprocessing as mp
import click
-from airflow_breeze.commands.custom_param_types import BetterChoice, CacheableChoice, CacheableDefault
+from airflow_breeze.commands.custom_param_types import (
+ AnswerChoice,
+ BetterChoice,
+ CacheableChoice,
+ CacheableDefault,
+)
from airflow_breeze.global_constants import (
ALLOWED_BACKENDS,
ALLOWED_BUILD_CACHE,
@@ -50,7 +55,7 @@ option_dry_run = click.option(
option_answer = click.option(
"-a",
"--answer",
- type=BetterChoice(['y', 'n', 'q', 'yes', 'no', 'quit']),
+ type=AnswerChoice(choices=['y', 'n', 'q', 'yes', 'no', 'quit']),
help="Force answer to questions.",
envvar='ANSWER',
)
diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
index 47811c3660..5f4d12c83b 100644
--- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
+++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance.py
@@ -40,7 +40,7 @@ from airflow_breeze.commands.main import main
from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_ALL
from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.cache import check_if_cache_exists, delete_cache, touch_cache_file
-from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm
+from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
check_docker_resources,
@@ -130,7 +130,6 @@ CONFIGURATION_AND_MAINTENANCE_PARAMETERS = {
@option_dry_run
@option_github_repository
def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, answer: Optional[str]):
- set_forced_answer(answer)
if all:
get_console().print(
"\n[info]Removing cache of parameters, clean up docker cache "
@@ -162,7 +161,7 @@ def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, ans
'--force',
]
docker_rmi_command_to_execute.extend(images)
- given_answer = user_confirm("Are you sure?", timeout=None)
+ given_answer = user_confirm("Are you sure with the removal?")
if given_answer == Answer.YES:
run_command(docker_rmi_command_to_execute, verbose=verbose, dry_run=dry_run, check=False)
elif given_answer == Answer.QUIT:
@@ -170,14 +169,14 @@ def cleanup(verbose: bool, dry_run: bool, github_repository: str, all: bool, ans
else:
get_console().print("[light_blue]No locally downloaded images to remove[/]\n")
get_console().print("Pruning docker images")
- given_answer = user_confirm("Are you sure?", timeout=None)
+ given_answer = user_confirm("Are you sure with the removal?")
if given_answer == Answer.YES:
system_prune_command_to_execute = ['docker', 'system', 'prune']
run_command(system_prune_command_to_execute, verbose=verbose, dry_run=dry_run, check=False)
elif given_answer == Answer.QUIT:
sys.exit(0)
get_console().print(f"Removing build cache dir ${BUILD_CACHE_DIR}")
- given_answer = user_confirm("Are you sure?", timeout=None)
+ given_answer = user_confirm("Are you sure with the removal?")
if given_answer == Answer.YES:
if not dry_run:
shutil.rmtree(BUILD_CACHE_DIR, ignore_errors=True)
@@ -215,7 +214,7 @@ def self_upgrade(force: bool, use_current_airflow_sources: bool):
if force:
reinstall_breeze(breeze_sources)
else:
- ask_to_reinstall_breeze(breeze_sources)
+ ask_to_reinstall_breeze(breeze_sources, timeout=None)
else:
warn_non_editable()
sys.exit(1)
@@ -235,7 +234,6 @@ def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Option
"""
Enables autocompletion of breeze commands.
"""
- set_forced_answer(answer)
# Determine if the shell is bash/zsh/powershell. It helps to build the autocomplete path
detected_shell = os.environ.get('SHELL')
detected_shell = None if detected_shell is None else detected_shell.split(os.sep)[-1]
@@ -247,8 +245,10 @@ def setup_autocomplete(verbose: bool, dry_run: bool, force: bool, answer: Option
AIRFLOW_SOURCES_ROOT / "dev" / "breeze" / "autocomplete" / f"{NAME}-complete-{detected_shell}.sh"
)
get_console().print(f"[info]Activation command script is available here: {autocomplete_path}[/]\n")
- get_console().print(f"[info]We need to add above script to your {detected_shell} profile.[/]\n")
- given_answer = user_confirm("Should we proceed ?", default_answer=Answer.NO, timeout=3)
+ get_console().print(f"[warning]We need to add above script to your {detected_shell} profile.[/]\n")
+ given_answer = user_confirm(
+ "Should we proceed with modifying the script?", default_answer=Answer.NO, timeout=STANDARD_TIMEOUT
+ )
if given_answer == Answer.YES:
if detected_shell == 'bash':
script_path = str(Path('~').expanduser() / '.bash_completion')
@@ -388,8 +388,7 @@ def change_config(
@option_dry_run
@option_answer
def free_space(verbose: bool, dry_run: bool, answer: str):
- set_forced_answer(answer)
- if user_confirm("Are you sure to run free-space and perform cleanup?", timeout=None) == Answer.YES:
+ if user_confirm("Are you sure to run free-space and perform cleanup?") == Answer.YES:
run_command(["sudo", "swapoff", "-a"], verbose=verbose, dry_run=dry_run)
run_command(["sudo", "rm", "-f", "/swapfile"], verbose=verbose, dry_run=dry_run)
run_command(["sudo", "apt-get", "clean"], verbose=verbose, dry_run=dry_run, check=False)
diff --git a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py b/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
index 4737aa2737..45c4ee0ff8 100644
--- a/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
+++ b/dev/breeze/src/airflow_breeze/commands/custom_param_types.py
@@ -26,6 +26,7 @@ from airflow_breeze.utils.cache import (
read_from_cache_file,
write_to_cache_file,
)
+from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.recording import output_file_for_recording
@@ -53,6 +54,18 @@ class BetterChoice(click.Choice):
return f"[{choices_str}]"
+class AnswerChoice(BetterChoice):
+ """
+ Stores forced answer if it has been selected
+ """
+
+ name = "AnswerChoice"
+
+ def convert(self, value, param, ctx):
+ set_forced_answer(value)
+ return super().convert(value, param, ctx)
+
+
@dataclass
class CacheableDefault:
value: Any
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 3338a32cb6..598190d5d9 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -53,7 +53,6 @@ from airflow_breeze.global_constants import (
from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
from airflow_breeze.shell.enter_shell import enter_shell, find_airflow_container
from airflow_breeze.shell.shell_params import ShellParams
-from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.docker_command_utils import (
construct_env_variables_docker_compose_command,
@@ -230,7 +229,6 @@ def shell(
extra_args: Tuple,
):
"""Enter breeze.py environment. this is the default command use when no other is selected."""
- set_forced_answer(answer)
if verbose or dry_run:
get_console().print("\n[success]Welcome to breeze.py[/]\n")
get_console().print(f"\n[success]Root of Airflow Sources = {AIRFLOW_SOURCES_ROOT}[/]\n")
@@ -295,7 +293,6 @@ def start_airflow(
extra_args: Tuple,
):
"""Enter breeze.py environment and starts all Airflow components in the tmux session."""
- set_forced_answer(answer)
enter_shell(
verbose=verbose,
dry_run=dry_run,
diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py b/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
index 52fd3ec2a3..fcd09549cd 100644
--- a/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
+++ b/dev/breeze/src/airflow_breeze/commands/production_image_tools.py
@@ -66,7 +66,6 @@ from airflow_breeze.commands.custom_param_types import BetterChoice
from airflow_breeze.commands.main import main
from airflow_breeze.global_constants import ALLOWED_INSTALLATION_METHODS, DEFAULT_EXTRAS
from airflow_breeze.utils.ci_group import ci_group
-from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.pulll_image import run_pull_image, run_pull_in_parallel
from airflow_breeze.utils.python_versions import get_python_version_list
@@ -271,7 +270,6 @@ def build_prod_image(
get_console().print(f"[error]Error when building image! {info}")
sys.exit(return_code)
- set_forced_answer(answer)
parameters_passed = filter_out_none(**kwargs)
if build_multiple_images:
with ci_group(f"Building images sequentially {python_versions}", enabled=with_ci_group):
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management.py b/dev/breeze/src/airflow_breeze/commands/release_management.py
index 33765f9afd..4655b6197c 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management.py
@@ -48,7 +48,7 @@ from airflow_breeze.global_constants import (
)
from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.ci_group import ci_group
-from airflow_breeze.utils.confirm import Answer, set_forced_answer, user_confirm
+from airflow_breeze.utils.confirm import Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.constraints import run_generate_constraints, run_generate_constraints_in_parallel
from airflow_breeze.utils.docker_command_utils import (
@@ -192,7 +192,6 @@ def prepare_provider_documentation(
skip_package_verification: bool,
packages: List[str],
):
- set_forced_answer(answer)
shell_params = ShellParams(
verbose=verbose,
mount_sources=MOUNT_ALL,
@@ -317,16 +316,13 @@ def generate_constraints(
generate_constraints_mode: str,
with_ci_group: bool,
):
- set_forced_answer(answer)
if run_in_parallel:
given_answer = user_confirm(
f"Did you build all CI images {python_versions} with --upgrade-to-newer-dependencies flag set?",
- timeout=None,
)
else:
given_answer = user_confirm(
f"Did you build CI image {python} with --upgrade-to-newer-dependencies flag set?",
- timeout=None,
)
if given_answer != Answer.YES:
if run_in_parallel:
diff --git a/dev/breeze/src/airflow_breeze/utils/confirm.py b/dev/breeze/src/airflow_breeze/utils/confirm.py
index 45a9026edb..8cfb1c69fd 100644
--- a/dev/breeze/src/airflow_breeze/utils/confirm.py
+++ b/dev/breeze/src/airflow_breeze/utils/confirm.py
@@ -19,11 +19,13 @@ import sys
from enum import Enum
from typing import Optional
+STANDARD_TIMEOUT = 10
+
class Answer(Enum):
- YES = (0,)
- NO = (1,)
- QUIT = 2
+ YES = "y"
+ NO = "n"
+ QUIT = "q"
forced_answer: Optional[str] = None
@@ -36,7 +38,7 @@ def set_forced_answer(answer: Optional[str]):
def user_confirm(
message: str,
- timeout: Optional[float],
+ timeout: Optional[float] = None,
default_answer: Optional[Answer] = Answer.NO,
quit_allowed: bool = True,
) -> Answer:
@@ -46,8 +48,7 @@ def user_confirm(
:rtype: object
:param message: message to display to the user (should end with the question mark)
:param timeout: time given user to answer
- :param default_answer: default value returned on timeout. If no default - the question is
- repeated until user answers.
+ :param default_answer: default value returned on timeout. If no default - is set, the timeout is ignored.
:param quit_allowed: whether quit answer is allowed
:return:
"""
@@ -61,11 +62,31 @@ def user_confirm(
user_status = force
print(f"Forced answer for '{message}': {force}")
else:
+ if default_answer:
+ # Capitalise default answer
+ allowed_answers = allowed_answers.replace(
+ default_answer.value, default_answer.value.upper()
+ )
+ timeout_answer = default_answer.value
+ else:
+ timeout = None
+ timeout_answer = ""
+ message_prompt = f'\n{message} \nPress {allowed_answers}'
+ if default_answer and timeout:
+ message_prompt += (
+ f". Auto-select {timeout_answer} in {timeout} seconds "
+ f"(add `--answer {default_answer.value}` to avoid delay next time)"
+ )
+ message_prompt += ": "
user_status = inputimeout(
- prompt=f'\n{message} \nPress {allowed_answers}'
- + (f" in {timeout} seconds: " if timeout is not None else ": "),
+ prompt=message_prompt,
timeout=timeout,
)
+ if user_status == '':
+ if default_answer:
+ return default_answer
+ else:
+ continue
if user_status.upper() in ['Y', 'YES']:
return Answer.YES
elif user_status.upper() in ['N', 'NO']:
@@ -75,9 +96,10 @@ def user_confirm(
else:
print(f"Wrong answer given {user_status}. Should be one of {allowed_answers}. Try again.")
except TimeoutOccurred:
- if default_answer is not None:
+ if default_answer:
return default_answer
- print(f"Timeout after {timeout} seconds. Try again.")
+ # timeout should only occur when default_answer is set so this should never happened
+ continue
except KeyboardInterrupt:
if quit_allowed:
return Answer.QUIT
diff --git a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py
index 0071c0c8e3..eedec9296d 100644
--- a/dev/breeze/src/airflow_breeze/utils/md5_build_check.py
+++ b/dev/breeze/src/airflow_breeze/utils/md5_build_check.py
@@ -98,7 +98,8 @@ def md5sum_check_if_build_is_needed(md5sum_cache_dir: Path, verbose: bool) -> bo
modified_files, not_modified_files = calculate_md5_checksum_for_files(md5sum_cache_dir, update=False)
if len(modified_files) > 0:
get_console().print(
- '[warning]The following files are modified since last time image was built: [/]\n\n'
+ f'[warning]The following important files are modified in {AIRFLOW_SOURCES_ROOT} '
+ f'since last time image was built: [/]\n\n'
)
for file in modified_files:
get_console().print(f" * [info]{file}[/]")
diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py
index d3003305f8..74ae2831a8 100644
--- a/dev/breeze/src/airflow_breeze/utils/path_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py
@@ -26,6 +26,7 @@ from pathlib import Path
from typing import Optional
from airflow_breeze import NAME
+from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.reinstall import (
ask_to_reinstall_breeze,
@@ -120,6 +121,16 @@ def get_used_sources_setup_metadata_hash() -> str:
return get_sources_setup_metadata_hash(get_used_airflow_sources())
+def set_forced_answer_for_upgrade_check():
+ """When we run upgrade check --answer is not parsed yet, so we need to guess it."""
+ if "--answer n" in " ".join(sys.argv).lower() or os.environ.get('ANSWER', '').lower().startswith("n"):
+ set_forced_answer("no")
+ if "--answer y" in " ".join(sys.argv).lower() or os.environ.get('ANSWER', '').lower().startswith("y"):
+ set_forced_answer("yes")
+ if "--answer q" in " ".join(sys.argv).lower() or os.environ.get('ANSWER', '').lower().startswith("q"):
+ set_forced_answer("quit")
+
+
def print_warning_if_setup_changed() -> bool:
"""
Prints warning if detected airflow sources are not the ones that Breeze was installed with.
@@ -132,7 +143,9 @@ def print_warning_if_setup_changed() -> bool:
if installation_sources is not None:
breeze_sources = installation_sources / "dev" / "breeze"
warn_dependencies_changed()
+ set_forced_answer_for_upgrade_check()
ask_to_reinstall_breeze(breeze_sources)
+ set_forced_answer(None)
return True
return False
diff --git a/dev/breeze/src/airflow_breeze/utils/reinstall.py b/dev/breeze/src/airflow_breeze/utils/reinstall.py
index ca869fd643..0df3a6a0ea 100644
--- a/dev/breeze/src/airflow_breeze/utils/reinstall.py
+++ b/dev/breeze/src/airflow_breeze/utils/reinstall.py
@@ -18,9 +18,10 @@
import subprocess
import sys
from pathlib import Path
+from typing import Optional
from airflow_breeze import NAME
-from airflow_breeze.utils.confirm import Answer, user_confirm
+from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm
from airflow_breeze.utils.console import get_console
@@ -41,15 +42,17 @@ def reinstall_breeze(breeze_sources: Path):
sys.exit(0)
-def ask_to_reinstall_breeze(breeze_sources: Path):
+def ask_to_reinstall_breeze(breeze_sources: Path, timeout: Optional[int] = STANDARD_TIMEOUT):
"""
Ask the user to reinstall Breeze (and do so if confirmed).
:param breeze_sources: breeze sources to reinstall Breeze from.
+ :param timeout to answer - if set (when upgrade is auxiliary) then default answer is no,
+ otherwise it is yes
"""
answer = user_confirm(
f"Do you want to reinstall Breeze from {breeze_sources.parent.parent}?",
- timeout=3,
- default_answer=Answer.NO,
+ default_answer=Answer.NO if timeout else Answer.YES,
+ timeout=timeout,
)
if answer == Answer.YES:
reinstall_breeze(breeze_sources)
diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py
index b215cb523b..1412655f2b 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py
@@ -186,7 +186,9 @@ def get_filesystem_type(filepath):
def instruct_build_image(python: str):
"""Print instructions to the user that they should build the image"""
get_console().print(f'[warning]\nThe CI image for ' f'python version {python} may be outdated[/]\n')
- print(f"\n[info]Please run at the earliest convenience:[/]\n\nbreeze build-image --python {python}\n\n")
+ get_console().print(
+ f"\n[info]Please run at the earliest convenience:[/]\n\nbreeze build-image --python {python}\n\n"
+ )
@contextlib.contextmanager