You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/05/02 10:42:58 UTC
(airflow) branch main updated: Remove special case for building PROD image in non-main branch (#50115)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9deb2a71700 Remove special case for building PROD image in non-main branch (#50115)
9deb2a71700 is described below
commit 9deb2a71700d36ebab7edd9a45845988ef76d370
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri May 2 12:42:51 2025 +0200
Remove special case for building PROD image in non-main branch (#50115)
When we removed "chicken-egg-providers" from the build in #49799 we
should be able now to build PROD image in v3*test branch without special
case of only using PyPI providers. This should work because in
v3-*test branch we automatically detect (by presence of relesed
tag) whether we should build provider from sources or use it from
PyPI. In normal circumstances, we would need use providers fro
PyPI, but sometimes (For example for 3.0.1) we need to use some
providers from sources before they get released (for example
common.messaging 1.0.1 that is the lowest possible common.messaging
for Airflow 3.0.1.
This PR removes that special case and switches to the same method
of building PROD image in main and in release branch.
---
.github/workflows/additional-ci-image-checks.yml | 2 +-
.github/workflows/additional-prod-image-tests.yml | 2 --
.github/workflows/ci-image-build.yml | 2 +-
.github/workflows/ci-image-checks.yml | 2 +-
.github/workflows/ci.yml | 1 -
.github/workflows/prod-image-build.yml | 37 +++++++----------------
.github/workflows/prod-image-extra-checks.yml | 10 +-----
.github/workflows/push-image-cache.yml | 2 +-
8 files changed, 16 insertions(+), 42 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
index c4faa7dac05..86740494b64 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -37,7 +37,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml
index 6fe821ad390..56badcd10ad 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -68,7 +68,6 @@ jobs:
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
use-uv: "false"
- build-provider-distributions: ${{ inputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
@@ -84,7 +83,6 @@ jobs:
default-python-version: ${{ inputs.default-python-version }}
branch: ${{ inputs.default-branch }}
use-uv: "false"
- build-provider-distributions: ${{ inputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
index b4392930567..1df2e26c090 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -77,7 +77,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
diff --git a/.github/workflows/ci-image-checks.yml b/.github/workflows/ci-image-checks.yml
index 0e007f54d9f..93b8fd1ece2 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -41,7 +41,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
canary-run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 66a4da10ee0..79434589238 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -689,7 +689,6 @@ jobs:
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
branch: ${{ needs.build-info.outputs.default-branch }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
- build-provider-distributions: ${{ needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.docker-cache }}
diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
index 1c8f7504c25..844b0d46e7d 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -85,17 +85,13 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch:
description: "Branch used to construct constraints URL from."
required: true
type: string
- build-provider-distributions:
- description: "Whether to build provider distributions (true/false). If false providers are from PyPI"
- required: true
- type: string
upgrade-to-newer-dependencies:
description: "Whether to attempt to upgrade image to newer dependencies (true/false)"
required: true
@@ -143,14 +139,22 @@ jobs:
with:
use-uv: ${{ inputs.use-uv }}
if: inputs.upload-package-artifact == 'true'
- - name: "Prepare providers packages"
+ - name: "Prepare providers packages - all providers built from sources"
shell: bash
run: >
breeze release-management prepare-provider-distributions
--distributions-list-file ./prod_image_installed_providers.txt
--distribution-format wheel --include-not-ready-providers --skip-tag-check
if: >
- inputs.upload-package-artifact == 'true' && inputs.build-provider-distributions == 'true'
+ inputs.upload-package-artifact == 'true' && inputs.branch == 'main'
+ - name: "Prepare providers packages with only new versions of providers"
+ shell: bash
+ run: >
+ breeze release-management prepare-provider-distributions
+ --distributions-list-file ./prod_image_installed_providers.txt
+ --distribution-format wheel --include-not-ready-providers
+ if: >
+ inputs.upload-package-artifact == 'true' && inputs.branch != 'main'
- name: "Prepare airflow package"
shell: bash
run: >
@@ -262,25 +266,6 @@ jobs:
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
- if: inputs.build-provider-distributions == 'true'
- - name: "Build PROD images with PyPi providers ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
- shell: bash
- run: >
- breeze prod-image build
- --builder airflow_cache
- --commit-sha "${{ github.sha }}"
- --install-distributions-from-context
- --airflow-constraints-mode constraints
- --use-constraints-for-context-distributions
- env:
- PUSH: ${{ inputs.push-image }}
- DOCKER_CACHE: ${{ inputs.docker-cache }}
- DISABLE_AIRFLOW_REPO_CACHE: ${{ inputs.disable-airflow-repo-cache }}
- DEBIAN_VERSION: ${{ inputs.debian-version }}
- INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
- INCLUDE_NOT_READY_PROVIDERS: "true"
- if: inputs.build-provider-distributions != 'true'
- name: "Verify PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
run: breeze prod-image verify
- name: "Export PROD docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml
index 287348d7754..356900c9222 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -33,17 +33,13 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
use-uv:
description: "Whether to use uv to build the image (true/false)"
required: true
type: string
- build-provider-distributions:
- description: "Whether to build provider distributions (true/false). If false providers are from PyPI"
- required: true
- type: string
upgrade-to-newer-dependencies:
description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)"
required: true
@@ -78,7 +74,6 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: ${{ inputs.use-uv }}
- build-provider-distributions: ${{ inputs.build-provider-distributions }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
@@ -87,8 +82,6 @@ jobs:
pip-image:
uses: ./.github/workflows/prod-image-build.yml
- # Skip testing PIP image on release branches as all images there are built with pip
- if: ${{ inputs.use-uv == 'true' }}
with:
runs-on-as-json-public: ${{ inputs.runs-on-as-json-public }}
build-type: "pip"
@@ -102,7 +95,6 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: "false"
- build-provider-distributions: ${{ inputs.build-provider-distributions }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml
index ba16ac4fcb3..7db441c3cc7 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -57,7 +57,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
branch:
- description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ description: "Branch used to run the CI jobs in (main/v*_*_test)."
required: true
type: string
constraints-branch: