You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/12/30 12:20:07 UTC
(airflow) 04/27: Fix using extras when `--use-airflow-version` is used in Breeze (#36280)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-8-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 8986f8f3beba11cf568e0f014883b997e4042f7b
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Dec 18 20:18:30 2023 +0100
Fix using extras when `--use-airflow-version` is used in Breeze (#36280)
When we are installing a released version of Airflow in Breeze, we can
pass additional extras to install (For example, we need to pass celery
extra in order to start airflow with celery executor.
The extras could be specified as:
```
breeze start-airflow --use-airflow-version 2.8.0rc4 \
--executor CeleryExecutor --airflow-extras "celery"
```
However recent refactors caused a problem that the extras added were
specified after version (which is rejected by newer versions of `pip`).
This PR fixes it and also moves the place where CeleryExecutor use
triggers adding celery extra when`--use-airflow-version` is used.
The warning about this is better visible when moving to Shell Params.
(cherry picked from commit 329780649543ab7b9d593a2e2428073fbd4cf274)
---
dev/breeze/src/airflow_breeze/params/shell_params.py | 7 +++++++
.../src/airflow_breeze/utils/docker_command_utils.py | 14 --------------
scripts/in_container/install_airflow_and_providers.py | 2 +-
3 files changed, 8 insertions(+), 15 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py
index 58c49ecf9b..e90d348046 100644
--- a/dev/breeze/src/airflow_breeze/params/shell_params.py
+++ b/dev/breeze/src/airflow_breeze/params/shell_params.py
@@ -328,6 +328,13 @@ class ShellParams:
if self.executor == "CeleryExecutor":
compose_file_list.append(DOCKER_COMPOSE_DIR / "integration-celery.yml")
+ if self.use_airflow_version:
+ current_extras = self.airflow_extras
+ if "celery" not in current_extras.split(","):
+ get_console().print(
+ "[warning]Adding `celery` extras as it is implicitly needed by celery executor"
+ )
+ self.airflow_extras = ",".join(current_extras.split(",") + ["celery"])
compose_file_list.append(DOCKER_COMPOSE_DIR / "base.yml")
self.add_docker_in_docker(compose_file_list)
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 2ad25acb7c..bc54b9bf69 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -751,20 +751,6 @@ def enter_shell(shell_params: ShellParams, output: Output | None = None) -> RunC
f"Changing the executor to {SEQUENTIAL_EXECUTOR}.\n"
)
shell_params.executor = SEQUENTIAL_EXECUTOR
-
- if shell_params.executor == "CeleryExecutor" and shell_params.use_airflow_version:
- if shell_params.airflow_extras and "celery" not in shell_params.airflow_extras.split():
- get_console().print(
- f"\n[warning]CeleryExecutor requires airflow_extras: celery. "
- f"Adding celery to extras: '{shell_params.airflow_extras}'.\n"
- )
- shell_params.airflow_extras += ",celery"
- elif not shell_params.airflow_extras:
- get_console().print(
- "\n[warning]CeleryExecutor requires airflow_extras: celery. "
- "Setting airflow extras to 'celery'.\n"
- )
- shell_params.airflow_extras = "celery"
if shell_params.restart:
bring_compose_project_down(preserve_volumes=False, shell_params=shell_params)
if shell_params.include_mypy_volume:
diff --git a/scripts/in_container/install_airflow_and_providers.py b/scripts/in_container/install_airflow_and_providers.py
index 9eadad9789..58f2d86af3 100755
--- a/scripts/in_container/install_airflow_and_providers.py
+++ b/scripts/in_container/install_airflow_and_providers.py
@@ -253,7 +253,7 @@ def find_installation_spec(
)
else:
console.print(f"\nInstalling airflow via apache-airflow=={use_airflow_version}")
- airflow_package_spec = f"apache-airflow=={use_airflow_version}{airflow_extras}"
+ airflow_package_spec = f"apache-airflow{airflow_extras}=={use_airflow_version}"
airflow_constraints_location = get_airflow_constraints_location(
airflow_skip_constraints=airflow_skip_constraints,
airflow_constraints_mode=airflow_constraints_mode,