You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/03/03 12:17:21 UTC
(airflow) 01/01: Refactor and extract production image checks to separate workflows
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch extract-prod-check-workflows
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4a29eb7fc92b2ca72184358af5128946001158b6
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Mar 3 11:37:35 2024 +0100
Refactor and extract production image checks to separate workflows
Our CI workflow yaml had become a huge flying-spaghetti-monster
piece of monolitic workflow, with lots of a duplication. This is
the first step (of quite a few) where we separate parts of the
workflow to micro-workflows.
GitHub Actions has the option to call workflows from other workflows
and (additionally to composite actions) such composite workflows
can implement a nice workflow code reuse.
This - first- attempt is to check if we can do it easily and it
should be followed by a series of similar extraction steps.
---
.github/workflows/build-images.yml | 15 +-
.github/workflows/ci.yml | 425 ++++-----------------
.github/workflows/prod-image-build.yml | 103 +++++
.github/workflows/prod-image-extra-checks.yml | 93 +++++
.../pre_commit_checkout_no_credentials.py | 2 +
5 files changed, 288 insertions(+), 350 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index da76fcbeff..6179f34c1d 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -42,11 +42,6 @@ env:
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
USE_SUDO: "true"
INCLUDE_SUCCESS_OUTPUTS: "true"
- # Version of Python used for reproducibility of the packages built
- # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
- # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
- # TODO: remove me when we switch to Python 3.9 as minimal version
- REPRODUCIBLE_PYTHON_VERSION: "3.9"
concurrency:
group: build-${{ github.event.pull_request.number || github.ref }}
@@ -60,6 +55,12 @@ jobs:
env:
TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
outputs:
+ # Version of Python used for reproducibility of the packages built
+ # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
+ # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
+ # TODO: remove me when we switch to Python 3.9 as minimal version
+ breeze-python-version: "3.9"
+ image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
all-python-versions-list-as-string: >-
@@ -306,7 +307,7 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
uses: ./.github/actions/build-prod-images
with:
@@ -388,7 +389,7 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
- name: >
- Build ARM CI images ${{ env.IMAGE_TAG }}
+ Build ARM CI images ${{ needs.build-info.outputs.image-tag }}
${{needs.build-info.outputs.all-python-versions-list-as-string}}
run: >
breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1cb19201dd..b9078345b8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -52,11 +52,6 @@ env:
INCLUDE_NOT_READY_PROVIDERS: "true"
AIRFLOW_ENABLE_AIP_44: "true"
MOUNT_SOURCES: "skip"
- # Version of Python used for reproducibility of the packages built
- # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
- # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
- # TODO: remove me when we switch to Python 3.9 as minimal version
- REPRODUCIBLE_PYTHON_VERSION: "3.9"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
@@ -74,6 +69,12 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
+ # Version of Python used for reproducibility of the packages built
+ # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing
+ # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible
+ # TODO: remove me when we switch to Python 3.9 as minimal version
+ breeze-python-version: "3.9"
+ image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
affected-providers-list-as-string: >-
${{ steps.selective-checks.outputs.affected-providers-list-as-string }}
@@ -302,7 +303,7 @@ jobs:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
BUILD_TIMEOUT_MINUTES: 70
- - name: Verify CI images ${{ matrix.python-version }}:${{ env.IMAGE_TAG }}
+ - name: Verify CI images ${{ matrix.python-version }}:${{ needs.build-info.outputs.image-tag }}
run: breeze ci-image verify --python ${{ matrix.python-version }}
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
@@ -471,7 +472,7 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
id: wait-for-images
run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest
env:
@@ -504,10 +505,10 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
+ - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze ci-image pull --run-in-parallel --tag-as-latest
- - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze ci-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
@@ -866,7 +867,7 @@ jobs:
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider documentation"
@@ -929,7 +930,7 @@ jobs:
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider packages: sdist"
@@ -1000,7 +1001,7 @@ jobs:
Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider packages: wheel"
@@ -1115,7 +1116,7 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- name: Setup git for tagging
run: |
git config --global user.email "name@example.com"
@@ -1849,342 +1850,78 @@ jobs:
echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l)
build-prod-images:
- strategy:
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: ${{needs.build-info.outputs.build-job-description}} PROD image (main) ${{matrix.python-version}}
- runs-on: ["ubuntu-22.04"]
+ name: Build PROD images (in-workflow)
needs: [build-info, build-ci-images]
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- BACKEND: sqlite
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-prod-images
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- with:
- build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
- chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
+ uses: ./.github/workflows/prod-image-build.yml
+ with:
+ build-type: "Regular"
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }}
+ build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
+ chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+ constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- build-prod-images-bullseye:
- strategy:
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: Build Bullseye PROD image (main) ${{matrix.python-version}}
- runs-on: ["ubuntu-22.04"]
+ prod-image-extra-checks-main:
+ name: PROD image checks (main)
+ # Here we just need to wait for CI images: we build provider packages and can use source constraints
needs: [build-info, build-ci-images]
- if: needs.build-info.outputs.canary-run == 'true'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- BACKEND: sqlite
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - uses: actions/checkout@v3
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: Build Bullseye PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-prod-images
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- with:
- build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
- chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
- DEBIAN_VERSION: "bullseye"
- # Do not override the "bookworm" image - just push a new bullseye image
- # TODO: improve caching for that build
- IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
+ uses: ./.github/workflows/prod-image-extra-checks.yml
+ with:
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
+ chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+ constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ if: needs.build-info.outputs.default-branch == 'main'
- build-prod-images-mysql-client:
- strategy:
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: Build MySQL Client PROD image (main) ${{ matrix.python-version}}
- runs-on: ["ubuntu-22.04"]
- needs: [build-info, build-ci-images]
- if: needs.build-info.outputs.canary-run == 'true'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- BACKEND: sqlite
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - uses: actions/checkout@v3
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- - name: Build MySQL Client PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-prod-images
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch == 'main'
- with:
- build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
- chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
- INSTALL_MYSQL_CLIENT_TYPE: "mysql"
- # Do not override the "mariadb" (original) image - just push a new mysql image
- # TODO: improve caching for that build
- IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
-
-
- build-prod-images-release-branch:
- strategy:
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: >
- ${{needs.build-info.outputs.build-job-description}} PROD image (v2_*_test) ${{matrix.python-version}}
- runs-on: ["ubuntu-22.04"]
+ prod-image-extra-checks-release-branch:
+ name: PROD image checks (release)
+ # Here we need to wait for generate-constraints to complete because we have to use PyPI constraints
needs: [build-info, generate-constraints]
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- BACKEND: sqlite
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - name: Build Release PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-prod-images
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- with:
- build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
- chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
-
- build-prod-images-bullseye-release-branch:
- strategy:
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: Build Bullseye PROD image (v2_*_test) ${{matrix.python-version}}
- runs-on: ["ubuntu-22.04"]
- needs: [build-info, generate-constraints]
- if: needs.build-info.outputs.canary-run == 'true'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- BACKEND: sqlite
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - uses: actions/checkout@v3
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - name: Build Bullseye Release PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-prod-images
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- with:
- build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
- chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
- DEBIAN_VERSION: "bullseye"
- # Do not override the "bookworm" image - just push a new bullseye image
- # TODO: improve caching for that build
- IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
+ uses: ./.github/workflows/prod-image-extra-checks.yml
+ with:
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
+ chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+ constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ if: needs.build-info.outputs.default-branch != 'main'
- build-prod-images-mysql-release-branch:
- strategy:
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: Build MySQL PROD image (v2_*_test) ${{matrix.python-version}}
- runs-on: ["ubuntu-22.04"]
- needs: [build-info, generate-constraints]
- if: needs.build-info.outputs.canary-run == 'true'
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- BACKEND: sqlite
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - uses: actions/checkout@v3
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- submodules: recursive
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- - name: Build Mysql PROD Image ${{matrix.python-version}}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-prod-images
- if: >
- needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.default-branch != 'main'
- with:
- build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
- chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
- INSTALL_MYSQL_CLIENT_TYPE: "mysql"
- # Do not override the "mariadb" image - just push a new mysql image
- # TODO: improve caching for that build
- IMAGE_TAG: "mysql-${{ github.event.pull_request.head.sha || github.sha }}"
wait-for-prod-images:
timeout-minutes: 80
name: "Wait for PROD images"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs: [build-info, wait-for-ci-images, build-prod-images, build-prod-images-release-branch]
- if: needs.build-info.outputs.prod-image-build == 'true'
+ runs-on: ["ubuntu-22.04"]
+ needs: [build-info, wait-for-ci-images, build-prod-images]
+ # This strange condition below is equivalent to: "all success or skipped"
+ # The "build-;rod-images" step might be skipped, in case production building happens in the
+ # "build-images" workflow, and in this case we sill want to wait for PROD images and run depending tests
+ if: >
+ always() && !failure() && !cancelled() &&
+ needs.build-info.outputs.prod-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
- # Force more parallelism for pull even on public images
+ # Force more parallelism for pull on public images
PARALLELISM: 6
steps:
- name: Cleanup repo
@@ -2198,7 +1935,7 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
# We wait for the images to be available either from "build-images.yml' run as pull_request_target
# or from build-prod-images (or build-prod-images-release-branch) above.
# We are utilising single job to wait for all images because this job merely waits
@@ -2225,7 +1962,9 @@ jobs:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- - name: Pull PROD image "${{needs.build-info.outputs.default-python-version}}":${{ env.IMAGE_TAG }}
+ - name: >
+ Pull PROD image
+ "${{needs.build-info.outputs.default-python-version}}":${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image pull --tag-as-latest
env:
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
@@ -2264,13 +2003,13 @@ jobs:
persist-credentials: false
- name: "Install Breeze"
uses: ./.github/actions/breeze
- - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
# Force more parallelism for pull even on public images
PARALLELISM: 6
- - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image verify --run-in-parallel
env:
PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
@@ -2306,7 +2045,7 @@ jobs:
- name: "Install Breeze"
uses: ./.github/actions/breeze
id: breeze
- - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }}
run: breeze prod-image pull --run-in-parallel --tag-as-latest
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
@@ -2438,7 +2177,7 @@ jobs:
Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
- python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
+ python-version: ${{ needs.build-info.outputs.breeze-python-version }}
env:
# Always use default Python version of CI image for preparing packages
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
@@ -2545,7 +2284,7 @@ jobs:
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: >
- Build CI ARM images ${{ env.IMAGE_TAG }}
+ Build CI ARM images ${{ needs.build-info.outputs.image-tag }}
${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
run: >
breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64"
diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
new file mode 100644
index 0000000000..05176dda61
--- /dev/null
+++ b/.github/workflows/prod-image-build.yml
@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Build PROD images (in-workflow)
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ build-type:
+ required: true
+ type: string
+ debian-version:
+ type: string
+ default: "bookworm"
+ install-mysql-client-type:
+ type: string
+ default: "mariadb"
+ image-tag:
+ required: true
+ type: string
+ in-workflow-build:
+ required: true
+ type: string
+ python-versions:
+ required: true
+ type: string
+ branch:
+ required: true
+ type: string
+ build-provider-packages:
+ required: true
+ type: string
+ upgrade-to-newer-dependencies:
+ required: true
+ type: string
+ breeze-python-version:
+ required: true
+ type: string
+ chicken-egg-providers:
+ required: true
+ type: string
+ constraints-branch:
+ required: true
+ type: string
+ docker-cache:
+ required: true
+ type: string
+ debug-resources:
+ required: true
+ type: string
+jobs:
+ build-prod-images:
+ strategy:
+ fail-fast: true
+ matrix:
+ python-version: ${{fromJson(inputs.python-versions)}}
+ timeout-minutes: 80
+ name: Build PROD ${{inputs.build-type}} image ${{matrix.python-version}}
+ runs-on: ["ubuntu-22.04"]
+ if: ${{ inputs.in-workflow-build == 'true' }}
+ env:
+ DEFAULT_BRANCH: ${{ inputs.branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
+ BACKEND: sqlite
+ VERSION_SUFFIX_FOR_PYPI: "dev0"
+ DEBUG_RESOURCES: ${{ inputs.debug-resources}}
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ persist-credentials: false
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ python-version: ${{ inputs.breeze-python-version }}
+ - name: Build PROD Image ${{ matrix.python-version }}:${{ inputs.image-tag }}
+ uses: ./.github/actions/build-prod-images
+ with:
+ build-provider-packages: ${{ inputs.build-provider-packages }}
+ chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
+ python-version: ${{ matrix.python-version }}
+ env:
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ inputs.cache-directive }}
+ DEBIAN_VERSION: ${{ inputs.debian-version }}
+ INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
+ INCLUDE_NOT_READY_PROVIDERS: "true"
diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml
new file mode 100644
index 0000000000..2d0be2fdcb
--- /dev/null
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: PROD images extra checks
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ python-versions:
+ required: true
+ type: string
+ branch:
+ required: true
+ type: string
+ image-tag:
+ required: true
+ type: string
+ build-provider-packages:
+ required: true
+ type: string
+ upgrade-to-newer-dependencies:
+ required: true
+ type: string
+ breeze-python-version:
+ required: true
+ type: string
+ chicken-egg-providers:
+ required: true
+ type: string
+ constraints-branch:
+ required: true
+ type: string
+ docker-cache:
+ required: true
+ type: string
+ debug-resources:
+ required: true
+ type: string
+ canary-run:
+ required: true
+ type: string
+jobs:
+ bullseye-image:
+ uses: ./.github/workflows/prod-image-build.yml
+ with:
+ build-type: "Bullseye"
+ image-tag: bullseye-${{ inputs.image-tag }}
+ debian-version: "bullseye"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ # Always build images during the extra checks
+ in-workflow-build: 'true'
+ build-provider-packages: ${{ inputs.build-provider-packages }}
+ upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ inputs.breeze-python-version }}
+ chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ docker-cache: ${{ inputs.docker-cache }}
+ debug-resources: ${{ inputs.debug-resources }}
+ if: ${{ inputs.canary-run == 'true' }}
+
+ myssql-client-image:
+ uses: ./.github/workflows/prod-image-build.yml
+ with:
+ build-type: "MySQL Client"
+ image-tag: mysql-${{ inputs.image-tag }}
+ install-mysql-client-type: "mysql"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ # Always build images during the extra checks
+ in-workflow-build: 'true'
+ build-provider-packages: ${{ inputs.build-provider-packages }}
+ upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ inputs.breeze-python-version }}
+ chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ docker-cache: ${{ inputs.docker-cache }}
+ debug-resources: ${{ inputs.debug-resources }}
+ if: ${{ inputs.canary-run == 'true' }}
diff --git a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
index 15c0650e18..02e8f0a20f 100755
--- a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
+++ b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py
@@ -39,6 +39,8 @@ def check_file(the_file: Path) -> int:
res = yaml.safe_load(the_file.read_text())
console.print(f"Checking file [yellow]{the_file}[/]")
for job in res["jobs"].values():
+ if job.get("steps") is None:
+ continue
for step in job["steps"]:
uses = step.get("uses")
pretty_step = yaml.safe_dump(step, indent=2)