You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/12/01 14:33:25 UTC
(airflow) branch main updated: Add support for chicken-egg providers to dockerhub release process (#36002)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8829d1732c Add support for chicken-egg providers to dockerhub release process (#36002)
8829d1732c is described below
commit 8829d1732c7f210c9d2ab0cc20ebfb1861ae9f84
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Dec 1 15:33:18 2023 +0100
Add support for chicken-egg providers to dockerhub release process (#36002)
---
.github/workflows/release_dockerhub_image.yml | 17 ++++++
Dockerfile | 7 +++
airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 58 ++++++++++++++++++++
.../commands/release_management_commands.py | 33 ++++++++----
.../commands/release_management_commands_config.py | 1 +
.../src/airflow_breeze/utils/common_options.py | 8 +++
dev/breeze/src/airflow_breeze/utils/versions.py | 6 +++
...tput_release-management_release-prod-images.svg | 62 ++++++++++++++--------
...tput_release-management_release-prod-images.txt | 2 +-
.../docker/install_from_docker_context_files.sh | 7 +++
10 files changed, 170 insertions(+), 31 deletions(-)
diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml
index 6889539387..3d6d4e065e 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -35,6 +35,7 @@ concurrency:
cancel-in-progress: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ VERBOSE: true
jobs:
build-info:
timeout-minutes: 10
@@ -46,6 +47,7 @@ jobs:
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
+ chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }}
limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }}
env:
@@ -107,6 +109,20 @@ jobs:
run: >
echo ${{ secrets.DOCKERHUB_TOKEN }} |
docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }}
+ - name: "Prepare chicken-eggs provider packages"
+ # In case of provider packages which use latest dev0 version of providers, we should prepare them
+ # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
+ # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
+ shell: bash
+ run: >
+ breeze release-management prepare-provider-packages
+ --package-format wheel
+ --version-suffix-for-pypi dev0 ${{ needs.build-info.outputs.chicken-egg-providers }}
+ if: needs.build-info.outputs.chicken-egg-providers != ''
+ - name: "Copy dist packages to docker-context files"
+ shell: bash
+ run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
+ if: needs.build-info.outputs.chicken-egg-providers != ''
- name: >
Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
run: >
@@ -116,6 +132,7 @@ jobs:
${{ needs.build-info.outputs.skipLatest }}
${{ needs.build-info.outputs.limitPlatform }}
--limit-python ${{ matrix.python-version }}
+ --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}"
env:
COMMIT_SHA: ${{ github.sha }}
- name: >
diff --git a/Dockerfile b/Dockerfile
index 549fe0668c..b2789e3b25 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -627,6 +627,13 @@ function install_airflow_and_providers_from_docker_context_files(){
reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==$ver"
fi
+ if [[ -z "${reinstalling_apache_airflow_package}" && ${AIRFLOW_VERSION=} != "" ]]; then
+ # When we install only provider packages from docker-context files, we need to still
+ # install airflow from PyPI when AIRFLOW_VERSION is set. This handles the case where
+ # pre-release dockerhub image of airflow is built, but we want to install some providers from
+ # docker-context files
+ reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}"
+ fi
# Find Apache Airflow packages in docker-context files
local reinstalling_apache_airflow_providers_packages
reinstalling_apache_airflow_providers_packages=$(ls \
diff --git a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
index 0fd2e16215..40cbd5ff61 100644
--- a/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
+++ b/airflow/providers/MANAGING_PROVIDERS_LIFECYCLE.rst
@@ -109,6 +109,64 @@ breeze and I'll run unit tests for my Hook.
root@fafd8d630e46:/opt/airflow# python -m pytest tests/providers/<NEW_PROVIDER>/hook/<NEW_PROVIDER>.py
+Adding chicken-egg providers
+----------------------------
+
+Sometimes we want to release provider that depends on the version of airflow that has not yet been released
+- for example when we released ``common.io`` provider it had ``apache-airflow>=2.8.0`` dependency. This
+creates problem when generating constraints and building docker image for pre-release versions of
+Airflow - because ``pip`` does not recognize the ``.dev0`` or ``.b1`` suffixes of those packages as
+valid in the ``>=X.Y.Z`` comparison. So when you want to install a provider package with
+``apache-airflow>=2.8.0`` requirement and you have ``2.8.0.dev0`` airflow package, ``pip`` will not
+install the package, because it does not recognize ``2.8.0.dev0`` as a valid version for ``>=2.8.0``
+dependency. This is because ``pip`` currently implements the minimum version selection algorithm
+requirement specified in packaging as described in the packaging version specification
+https://packaging.python.org/en/latest/specifications/version-specifiers/#handling-of-pre-releases
+Currently ``pip`` only allows to include pre-release versions for all installed packages using ``--pre``
+flag, but it does not have the possibility of selectively using this flag to only one package.
+In order to implement our desired behaviour, we need the case where only ``apache-airflow`` is considered
+as pre-release version while all the other dependencies only have stable versions and this is currently
+not possible.
+
+To work around this limitation, we have introduced the concept of "chicken-egg" providers. Those providers
+are providers that are released together with the version of Airflow they depend on. They are released
+with the same version number as the Airflow version they depend on, but with a different suffix. For example
+``apache-airflow-providers-common-io==2.8.0.dev0`` is a chicken-egg provider for ``apache-airflow==2.8.0.dev0``.
+
+However - we should not release those providers to ``pypi``, so in order to allow our CI to work with
+pre-release versions and perform both - constraint generation and image releasing, we introduced workarounds
+in our tooling where in case we build a pre-release version of Airflow, we will locally build the
+chicken-egg providers from sources and they are installed from local directory instead of from PyPI.
+
+This is controlled by ``chicken_egg_providers`` property in Selective Checks - and our CI will automatically
+build and use those chicken-egg providers during the CI process if pre-release version of Airflow is built.
+
+The code responsible for that is in ``src/airflow_breeze/utils/selective_checks.py``:
+
+.. code-block:: python
+
+ @cached_property
+ def chicken_egg_providers(self) -> str:
+ """Space separated list of providers with chicken-egg problem and should be built from sources."""
+ return "common.io"
+
+This list should be kept here until the official version of Airflow the chicken-egg-providers depend on
+is released and the version of airflow is updated in the ``main`` and ``v2-X-Y`` branch to ``2.X+1.0.dev0``
+and ``2.X.1.dev0`` respectively. After that the chicken-egg providers will be correctly installed because
+both ``2.X.1.dev0`` and ``2.X+1.0.dev0`` are considered by ``pip`` as ``>2.X.0`` (unlike ``2.X.0.dev0``).
+
+This workaround might be removed if ``pip`` implements the possibility of selectively using ``--pre`` flag
+for only one package (Which is foreseen as a possibility in the packaging specification but not implemented
+by ``pip``).
+
+.. note::
+
+ The current solution of building pre-release images will not work well if the chicken-egg-provider is
+ pre-installed package because slim imges will not use the chicken-egg-provider. This could be solved
+ by adding ``--chicken-egg-providers`` flag to slim image building step in ``released_dockerhub_image.yml``
+ but it would also require filtering out the non-pre-installed packages from it, so the current solution
+ is to assume pre-installed packages are not chicken-egg providers.
+
Integration tests
-----------------
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 42a55111ea..3822d9da4b 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -71,6 +71,7 @@ from airflow_breeze.utils.common_options import (
option_airflow_extras,
option_airflow_site_directory,
option_answer,
+ option_chicken_egg_providers,
option_commit_sha,
option_debug_resources,
option_dry_run,
@@ -139,6 +140,7 @@ from airflow_breeze.utils.run_utils import (
run_compile_www_assets,
)
from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
+from airflow_breeze.utils.versions import is_pre_release
option_debug_release_management = click.option(
"--debug",
@@ -685,14 +687,7 @@ def run_generate_constraints_in_parallel(
@option_image_tag_for_running
@option_debug_release_management
@option_airflow_constraints_mode_ci
-@click.option(
- "--chicken-egg-providers",
- default="",
- help="List of chicken-egg provider packages - "
- "those that have airflow_version >= current_version and should "
- "be installed in CI from locally built packages with >= current_version.dev0 ",
- envvar="CHICKEN_EGG_PROVIDERS",
-)
+@option_chicken_egg_providers
@option_github_repository
@option_verbose
@option_dry_run
@@ -1207,6 +1202,18 @@ def add_back_references(
start_generating_back_references(site_path, list(expand_all_provider_packages(doc_packages)))
+def _add_chicken_egg_providers_to_build_args(
+ python_build_args: dict[str, str], chicken_egg_providers: str, airflow_version: str
+):
+ if chicken_egg_providers and is_pre_release(airflow_version):
+ get_console().print(
+ f"[info]Adding chicken egg providers to build args as {airflow_version} is "
+ f"pre release and we have chicken-egg packages '{chicken_egg_providers}' defined[/]"
+ )
+ python_build_args["INSTALL_PACKAGES_FROM_CONTEXT"] = "true"
+ python_build_args["DOCKER_CONTEXT_FILES"] = "./docker-context-files"
+
+
@release_management.command(
name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)."
)
@@ -1243,6 +1250,7 @@ def add_back_references(
"rc/alpha/beta images are built.",
)
@option_commit_sha
+@option_chicken_egg_providers
@option_verbose
@option_dry_run
def release_prod_images(
@@ -1253,10 +1261,11 @@ def release_prod_images(
limit_python: str | None,
commit_sha: str | None,
skip_latest: bool,
+ chicken_egg_providers: str,
):
perform_environment_checks()
check_remote_ghcr_io_commands()
- if not re.match(r"^\d*\.\d*\.\d*$", airflow_version):
+ if is_pre_release(airflow_version):
get_console().print(
f"[warning]Skipping latest image tagging as this is a pre-release version: {airflow_version}"
)
@@ -1308,6 +1317,9 @@ def release_prod_images(
get_console().print(f"[info]Building slim {airflow_version} image for Python {python}[/]")
python_build_args = deepcopy(slim_build_args)
slim_image_name = f"{dockerhub_repo}:slim-{airflow_version}-python{python}"
+ _add_chicken_egg_providers_to_build_args(
+ python_build_args, chicken_egg_providers, airflow_version
+ )
docker_buildx_command = [
"docker",
"buildx",
@@ -1337,6 +1349,9 @@ def release_prod_images(
}
if commit_sha:
regular_build_args["COMMIT_SHA"] = commit_sha
+ _add_chicken_egg_providers_to_build_args(
+ regular_build_args, chicken_egg_providers, airflow_version
+ )
docker_buildx_command = [
"docker",
"buildx",
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index ff0c9cdcd0..fb60e12463 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -161,6 +161,7 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"--limit-platform",
"--skip-latest",
"--commit-sha",
+ "--chicken-egg-providers",
],
}
],
diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py
index cf59df8bc0..d053a00514 100644
--- a/dev/breeze/src/airflow_breeze/utils/common_options.py
+++ b/dev/breeze/src/airflow_breeze/utils/common_options.py
@@ -787,3 +787,11 @@ option_restart = click.option(
is_flag=True,
envvar="RESTART",
)
+option_chicken_egg_providers = click.option(
+ "--chicken-egg-providers",
+ default="",
+ help="List of chicken-egg provider packages - "
+ "those that have airflow_version >= current_version and should "
+ "be installed in CI from locally built packages with >= current_version.dev0 ",
+ envvar="CHICKEN_EGG_PROVIDERS",
+)
diff --git a/dev/breeze/src/airflow_breeze/utils/versions.py b/dev/breeze/src/airflow_breeze/utils/versions.py
index 88c5986f7d..70dc6ad77d 100644
--- a/dev/breeze/src/airflow_breeze/utils/versions.py
+++ b/dev/breeze/src/airflow_breeze/utils/versions.py
@@ -34,3 +34,9 @@ def get_version_tag(version: str, provider_package_id: str, version_suffix: str
if version_suffix is None:
version_suffix = ""
return f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}"
+
+
+def is_pre_release(version: str) -> bool:
+ from packaging.version import Version
+
+ return Version(version).is_prerelease
diff --git a/images/breeze/output_release-management_release-prod-images.svg b/images/breeze/output_release-management_release-prod-images.svg
index 76922057e1..f1e6d24f2c 100644
--- a/images/breeze/output_release-management_release-prod-images.svg
+++ b/images/breeze/output_release-management_release-prod-images.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 660.0" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 782.0" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -45,7 +45,7 @@
<defs>
<clipPath id="breeze-release-management-release-prod-images-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="609.0" />
+ <rect x="0" y="0" width="1463.0" height="731.0" />
</clipPath>
<clipPath id="breeze-release-management-release-prod-images-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -119,9 +119,24 @@
<clipPath id="breeze-release-management-release-prod-images-line-23">
<rect x="0" y="562.7" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-24">
+ <rect x="0" y="587.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-25">
+ <rect x="0" y="611.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-26">
+ <rect x="0" y="635.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-27">
+ <rect x="0" y="660.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-release-prod-images-line-28">
+ <rect x="0" y="684.7" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="658" rx="8"/><text class="breeze-release-management-release-prod-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management release-prod-images</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="780" rx="8"/><text class="breeze-release-management-release-prod-images-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management release-prod-images</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -137,25 +152,30 @@
</text><text class="breeze-release-management-release-prod-images-r1" x="12.2" y="93.2" textLength="841.8" clip-path="url(#breeze-release-management-release-prod-images-line-3)">Release production images to DockerHub (needs DockerHub permissions).</text><text class="breeze-release-management-release-prod-images-r1" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-3)">
</text><text class="breeze-release-management-release-prod-images-r1" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-4)">
</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="142" textLength="24.4" clip-path="url(#breeze-release-management-release-prod-images-line-5)">╭─</text><text class="breeze-release-management-release-prod-images-r5" x="24.4" y="142" textLength="317.2" clip-path="url(#breeze-release-management-release-prod-images-line-5)"> Release PROD IMAGE flags </text><text class="breeze-release-management-release-prod-images-r5" x="341.6" y="142" t [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-6)">│</text><text class="breeze-release-management-release-prod-images-r6" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-6)">*</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="166.4" textLength="12.2" clip-path="url(#breeze-release-m [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-7)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-7)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="190.8" textLength="122" clip-path="url(#breeze-release-ma [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-8)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-8)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="215.2" textLength="61" clip-path="url(#breeze-release-man [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-9)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-9)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="239.6" textLength="73.2" clip-path="url(#breeze-release-m [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-10)">│</text><text class="breeze-release-management-release-prod-images-r1" x="317.2" y="264" textLength="1122.4" clip-path="url(#breeze-release-management-release-prod-images-line-10)">available python versions)                  [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-11)">│</text><text class="breeze-release-management-release-prod-images-r7" x="317.2" y="288.4" textLength="1122.4" clip-path="url(#breeze-release-management-release-prod-images-line-11)">(3.8 | 3.9 | 3.10 | 3.11)              [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-12)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-12)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="312.8" textLength="73.2" clip-path="url(#breeze-release [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-13)">│</text><text class="breeze-release-management-release-prod-images-r7" x="317.2" y="337.2" textLength="1122.4" clip-path="url(#breeze-release-management-release-prod-images-line-13)">(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)           [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-14)">│</text><text class="breeze-release-management-release-prod-images-r5" x="317.2" y="361.6" textLength="1122.4" clip-path="url(#breeze-release-management-release-prod-images-line-14)">[default: linux/amd64,linux/arm64]                [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-15)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="386" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-15)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="386" textLength="61" clip-path="url(#breeze-release-managem [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-16)">│</text><text class="breeze-release-management-release-prod-images-r1" x="317.2" y="410.4" textLength="1122.4" clip-path="url(#breeze-release-management-release-prod-images-line-16)">should only be used if you release image for previous branches. Automatically& [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-17)">│</text><text class="breeze-release-management-release-prod-images-r1" x="317.2" y="434.8" textLength="1122.4" clip-path="url(#breeze-release-management-release-prod-images-line-17)">rc/alpha/beta images are built.               [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-18)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-18)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="459.2" textLength="85.4" clip-path="url(#breeze-release [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="483.6" textLength="1464" clip-path="url(#breeze-release-management-release-prod-images-line-19)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-release-prod-images-r1" x="1464" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-19)">
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="508" textLength="24.4" clip-path="url(#breeze-release-management-release-prod-images-line-20)">╭─</text><text class="breeze-release-management-release-prod-images-r5" x="24.4" y="508" textLength="195.2" clip-path="url(#breeze-release-management-release-prod-images-line-20)"> Common options </text><text class="breeze-release-management-release-prod-images-r5" x="219.6" y="508" textLength="1220" c [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-21)">│</text><text class="breeze-release-management-release-prod-images-r4" x="24.4" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-21)">-</text><text class="breeze-release-management-release-prod-images-r4" x="36.6" y="532.4" textLength="97.6" clip-path="url(#breeze-relea [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-22)">│</text><text class="breeze-release-management-release-prod-images-r4" x="24.4" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-22)">-</text><text class="breeze-release-management-release-prod-images-r4" x="36.6" y="556.8" textLength="48.8" clip-path="url(#breeze-relea [...]
-</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-23)">│</text><text class="breeze-release-management-release-prod-images-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-23)">-</text><text class="breeze-release-management-release-prod-images-r4" x="36.6" y="581.2" textLength="61" clip-path="url(#breeze-release [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-6)">│</text><text class="breeze-release-management-release-prod-images-r6" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-6)">*</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="166.4" textLength="12.2" clip-path="url(#breeze-release-m [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-7)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="190.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-7)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="190.8" textLength="122" clip-path="url(#breeze-release-ma [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-8)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="215.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-8)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="215.2" textLength="61" clip-path="url(#breeze-release-man [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-9)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="239.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-9)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="239.6" textLength="73.2" clip-path="url(#breeze-release-m [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="264" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-10)">│</text><text class="breeze-release-management-release-prod-images-r1" x="390.4" y="264" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-10)">all available python versions)               &# [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-11)">│</text><text class="breeze-release-management-release-prod-images-r7" x="390.4" y="288.4" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-11)">(3.8 | 3.9 | 3.10 | 3.11)              [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-12)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="312.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-12)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="312.8" textLength="73.2" clip-path="url(#breeze-release [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-13)">│</text><text class="breeze-release-management-release-prod-images-r1" x="390.4" y="337.2" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-13)">built.                     & [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-14)">│</text><text class="breeze-release-management-release-prod-images-r7" x="390.4" y="361.6" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-14)">(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)           [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="386" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-15)">│</text><text class="breeze-release-management-release-prod-images-r5" x="390.4" y="386" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-15)">[default: linux/amd64,linux/arm64]                &# [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-16)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-16)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="410.4" textLength="61" clip-path="url(#breeze-release-m [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-17)">│</text><text class="breeze-release-management-release-prod-images-r1" x="390.4" y="434.8" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-17)">updated). This should only be used if you release image for previous bran [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-18)">│</text><text class="breeze-release-management-release-prod-images-r1" x="390.4" y="459.2" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-18)">Automatically set when rc/alpha/beta images are built.        & [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-19)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-19)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="483.6" textLength="85.4" clip-path="url(#breeze-release [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-20)">│</text><text class="breeze-release-management-release-prod-images-r4" x="61" y="508" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-20)">-</text><text class="breeze-release-management-release-prod-images-r4" x="73.2" y="508" textLength="97.6" clip-path="url(#breeze-release-manag [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-21)">│</text><text class="breeze-release-management-release-prod-images-r1" x="390.4" y="532.4" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-21)">current_version and should be installed in CI from locally built packages with [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-22)">│</text><text class="breeze-release-management-release-prod-images-r1" x="390.4" y="556.8" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-22)">current_version.dev0                    [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-23)">│</text><text class="breeze-release-management-release-prod-images-r7" x="390.4" y="581.2" textLength="1049.2" clip-path="url(#breeze-release-management-release-prod-images-line-23)">(TEXT)                     & [...]
</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="605.6" textLength="1464" clip-path="url(#breeze-release-management-release-prod-images-line-24)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-release-prod-images-r1" x="1464" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-24)">
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="630" textLength="24.4" clip-path="url(#breeze-release-management-release-prod-images-line-25)">╭─</text><text class="breeze-release-management-release-prod-images-r5" x="24.4" y="630" textLength="195.2" clip-path="url(#breeze-release-management-release-prod-images-line-25)"> Common options </text><text class="breeze-release-management-release-prod-images-r5" x="219.6" y="630" textLength="1220" c [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-26)">│</text><text class="breeze-release-management-release-prod-images-r4" x="24.4" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-26)">-</text><text class="breeze-release-management-release-prod-images-r4" x="36.6" y="654.4" textLength="97.6" clip-path="url(#breeze-relea [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-27)">│</text><text class="breeze-release-management-release-prod-images-r4" x="24.4" y="678.8" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-27)">-</text><text class="breeze-release-management-release-prod-images-r4" x="36.6" y="678.8" textLength="48.8" clip-path="url(#breeze-relea [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-28)">│</text><text class="breeze-release-management-release-prod-images-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-28)">-</text><text class="breeze-release-management-release-prod-images-r4" x="36.6" y="703.2" textLength="61" clip-path="url(#breeze-release [...]
+</text><text class="breeze-release-management-release-prod-images-r5" x="0" y="727.6" textLength="1464" clip-path="url(#breeze-release-management-release-prod-images-line-29)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-release-prod-images-r1" x="1464" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-release-prod-images-line-29)">
</text>
</g>
</g>
diff --git a/images/breeze/output_release-management_release-prod-images.txt b/images/breeze/output_release-management_release-prod-images.txt
index 0397b63e19..58827ed619 100644
--- a/images/breeze/output_release-management_release-prod-images.txt
+++ b/images/breeze/output_release-management_release-prod-images.txt
@@ -1 +1 @@
-cfbfe8b19fee91fd90718f98ef2fd078
+4c5c364cc00355ac02debacc48447be8
diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh
index cdb66875ac..8d9c6cdd85 100644
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -65,6 +65,13 @@ function install_airflow_and_providers_from_docker_context_files(){
reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==$ver"
fi
+ if [[ -z "${reinstalling_apache_airflow_package}" && ${AIRFLOW_VERSION=} != "" ]]; then
+ # When we install only provider packages from docker-context files, we need to still
+ # install airflow from PyPI when AIRFLOW_VERSION is set. This handles the case where
+ # pre-release dockerhub image of airflow is built, but we want to install some providers from
+ # docker-context files
+ reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}"
+ fi
# Find Apache Airflow packages in docker-context files
local reinstalling_apache_airflow_providers_packages
reinstalling_apache_airflow_providers_packages=$(ls \