You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/05/03 11:40:49 UTC
(airflow) branch v3-0-test updated: [v3-0-test] Better diagnostics and easier use of release dockerhub image workflow (#50144) (#50146)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 895370042cc [v3-0-test] Better diagnostics and easier use of release dockerhub image workflow (#50144) (#50146)
895370042cc is described below
commit 895370042ccf9f746ba87d89eeda5cdff56d1bf2
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat May 3 13:40:42 2025 +0200
[v3-0-test] Better diagnostics and easier use of release dockerhub image workflow (#50144) (#50146)
This change addresses a few challenges with release image workflows:
* normalizes version of airflow passed (so that your "." in ".rc" can
be removed automatically
* verifies that the version passed is one of the released versions of
apache-airflow
* replaces skipLatest field with automatic detection whether we are
releasing latest version or not (excluding pre-releases)
* allows to specify "amd only" flag to speed up releasing image in
case we want to test it and do not want to release multiplatform
image (and uses checkbox rather than text entry for it)
(cherry picked from commit 8622db70977c19406fc10043753da91ab3133a95)
Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
---
.github/workflows/release_dockerhub_image.yml | 51 +++++++++------
dev/README_RELEASE_AIRFLOW.md | 8 +--
dev/images/release_prod_image.png | Bin 309833 -> 77069 bytes
scripts/ci/airflow_version_check.py | 90 ++++++++++++++++++++++++++
4 files changed, 126 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml
index 710cc905ff6..ef0e0d9f485 100644
--- a/.github/workflows/release_dockerhub_image.yml
+++ b/.github/workflows/release_dockerhub_image.yml
@@ -21,12 +21,12 @@ on: # yamllint disable-line rule:truthy
workflow_dispatch:
inputs:
airflowVersion:
- description: 'Airflow version'
+ description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)'
required: true
- skipLatest:
- description: 'Skip Latest: Set to true if not latest.'
- default: ''
- required: false
+ amdOnly:
+ type: boolean
+ description: 'Limit to amd64 only (faster testing)'
+ default: false
permissions:
contents: read
packages: read
@@ -40,17 +40,26 @@ jobs:
build-info:
timeout-minutes: 10
name: "Build Info"
- runs-on: ["ubuntu-22.04"]
+ runs-on: ["ubuntu-24.04"]
outputs:
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
- skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }}
- limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }}
+ skipLatest: ${{ steps.selective-checks.outputs.skip-latest }}
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
VERBOSE: true
+ AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
+ AMD_ONLY: ${{ github.event.inputs.amdOnly }}
steps:
+ - name: "Input parameters summary"
+ shell: bash
+ run: |
+ echo "Input parameters summary"
+ echo "========================="
+ echo "Airflow version: '${AIRFLOW_VERSION}'"
+ echo "Skip latest: '${SKIP_LATEST}'"
+ echo "AMD only: '${AMD_ONLY}'"
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -60,10 +69,15 @@ jobs:
persist-credentials: false
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
+ - name: "Install uv"
+ run: curl -LsSf https://astral.sh/uv/install.sh | sh
+ - name: "Check airflow version"
+ shell: bash
+ run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}"
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
- use-uv: "false"
+ use-uv: "true"
- name: Selective checks
id: selective-checks
env:
@@ -150,8 +164,10 @@ jobs:
REPOSITORY: ${{ github.repository }}
PYTHON_VERSION: ${{ matrix.python-version }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
- SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }}
- LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }}
+ SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }}
+ LIMIT_PLATFORM: >
+ ${{ (github.repository == 'apache/airflow' || github.event.inputs.amdOnly == 'true')
+ && '--limit-platform linux/amd64' || '' }}
run: >
breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}"
--airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} ${LIMIT_PLATFORM}
@@ -163,14 +179,13 @@ jobs:
REPOSITORY: ${{ github.repository }}
PYTHON_VERSION: ${{ matrix.python-version }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
- SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }}
- LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }}
+ SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }}
+ LIMIT_PLATFORM: >
+ ${{ (github.repository == 'apache/airflow' || github.event.inputs.amdOnly == 'true')
+ && '--limit-platform linux/amd64' || '' }}
run: >
- breeze release-management release-prod-images
- --dockerhub-repo "${REPOSITORY}"
- --airflow-version "${AIRFLOW_VERSION}"
- ${SKIP_LATEST}
- ${LIMIT_PLATFORM}
+ breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}"
+ --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} ${LIMIT_PLATFORM}
--limit-python ${PYTHON_VERSION} --slim-images
- name: >
Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index 9bbe5abae3e..9998162763b 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -344,11 +344,9 @@ to have an environment prepared to build multi-platform images. You can achieve
Building the image is triggered by running the
[Release PROD Images](https://github.com/apache/airflow/actions/workflows/release_dockerhub_image.yml) workflow.
-When you trigger it you need to pass Airflow Version (including the right rc suffix). Make sure to use the
-``v2-*-stable`` branch for the workflow.
-
-You can leave the "skip latest" field empty.
-
+When you trigger it you need to pass Airflow Version (including the right `rc` suffix). The workflow will
+normalize and verify version passed. When you are testing or want to release image faster you can also
+select the check-box to limit the build to only AMD platform. This will speed up the build significantly.
diff --git a/dev/images/release_prod_image.png b/dev/images/release_prod_image.png
index 47ab4f35d0d..57386198b29 100644
Binary files a/dev/images/release_prod_image.png and b/dev/images/release_prod_image.png differ
diff --git a/scripts/ci/airflow_version_check.py b/scripts/ci/airflow_version_check.py
new file mode 100755
index 00000000000..20502aff400
--- /dev/null
+++ b/scripts/ci/airflow_version_check.py
@@ -0,0 +1,90 @@
+#! /usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+# "packaging>=23.2",
+# "requests>=2.28.1",
+# "rich>=13.3.5",
+# ]
+# ///
+from __future__ import annotations
+
+import sys
+
+import requests
+from packaging.version import Version
+from rich.console import Console
+
+console = Console(color_system="standard", stderr=True, width=400)
+
+
+def check_airflow_version(airflow_version: Version) -> tuple[str, bool]:
+ """
+ Check if the given version is a valid Airflow version and latest.
+
+ airflow_version: The Airflow version to check.
+ returns: tuple containing the version and a boolean indicating if it's latest.
+ """
+ latest = False
+ url = "https://pypi.org/pypi/apache-airflow/json"
+ max_versions_shown = 30
+ try:
+ response = requests.get(url)
+ response.raise_for_status()
+ data = response.json()
+ latest_version = Version(data["info"]["version"])
+ valid_versions = list(reversed(data["releases"].keys()))[:max_versions_shown]
+ if str(airflow_version) not in valid_versions:
+ console.print(f"[red]Version {airflow_version} is not a valid Airflow version")
+ console.print(
+ f"Available versions: (first available {max_versions_shown} versions):", valid_versions
+ )
+ sys.exit(1)
+ if airflow_version == latest_version:
+ latest = True
+ return str(airflow_version), latest
+ except Exception as e:
+ console.print(f"[red]Error fetching latest version: {e}")
+ sys.exit(1)
+
+
+def normalize_version(version: str) -> Version:
+ try:
+ return Version(version)
+ except Exception as e:
+ console.print(f"[red]Error normalizing version: {e}")
+ sys.exit(1)
+
+
+def print_both_outputs(output: str):
+ print(output)
+ console.print(output)
+
+
+if __name__ == "__main__":
+ if len(sys.argv) < 2:
+ console.print("[yellow]Usage: python normalize_airflow_version.py <version>")
+ sys.exit(1)
+ version = sys.argv[1]
+ parsed_version = normalize_version(version)
+ actual_version, is_latest = check_airflow_version(parsed_version)
+ print_both_outputs(f"airflow_version={actual_version}")
+ skip_latest = "false" if is_latest else "true"
+ print_both_outputs(f"skip_latest={skip_latest}")