You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/03/26 14:22:47 UTC
(airflow) 01/01: Improve finalization of tests
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch improve-test-finalization
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 19412f3feb30bdfea423fae20421362feddd9b98
Author: Idan Tepper <65...@users.noreply.github.com>
AuthorDate: Tue Mar 26 15:07:19 2024 +0200
Improve finalization of tests
Last step of the test preparation is pushing the latest images for
AMD to github registry. This should happen using the cache of ours
so we should use the `airflow_cache` builders to make it really fast.
For PROD image we also want to run them in parallel, because this
way we will reuse provider packages and airflow packages
built earlier in the same job.
Also while we are touching builders, we remove last place where
builders were passed by env variables.
---
.github/workflows/additional-ci-image-checks.yml | 5 +++++
.github/workflows/ci-image-build.yml | 6 ++----
.github/workflows/ci.yml | 1 +
.github/workflows/finalize-tests.yml | 11 +++++++----
.github/workflows/prod-image-build.yml | 5 ++---
.github/workflows/push-image-cache.yml | 15 +++++++--------
6 files changed, 24 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
index f73fe08ec7..79b7750bb2 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -33,6 +33,10 @@ on: # yamllint disable-line rule:truthy
description: "The list of python versions (stringified JSON array) to run the tests on."
required: true
type: string
+ python-versions-list-as-string:
+ description: "List of Python versions to test: space separated string"
+ required: true
+ type: string
branch:
description: "Branch used to run the CI jobs in (main/v2_*_test)."
required: true
@@ -107,6 +111,7 @@ jobs:
push-latest-images: "false"
platform: "linux/amd64"
python-versions: ${{ inputs.python-versions }}
+ python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
branch: ${{ inputs.branch }}
constraints-branch: ${{ inputs.constraints-branch }}
use-uv: "true"
diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
index 177cd39fbb..e384d7cf83 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -192,15 +192,13 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
${{ inputs.platform }}:${{ matrix.python-version }}:${{ inputs.image-tag }}
run: >
- breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}"
- --python "${{ matrix.python-version }}"
- --platform "${{ inputs.platform }}"
+ breeze ci-image build --builder airflow_cache --tag-as-latest --image-tag "${{ inputs.image-tag }}"
+ --python "${{ matrix.python-version }}" --platform "${{ inputs.platform }}"
env:
DOCKER_CACHE: ${{ inputs.docker-cache }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- BUILDER: "airflow_cache"
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
if: inputs.do-build == 'true'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 992a49ac43..9e93472ecd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -252,6 +252,7 @@ jobs:
runs-on: ${{ needs.build-info.outputs.runs-on }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
+ python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
default-python-version: ${{ needs.build-info.outputs.default-python-version }}
diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
index a10b1826d1..dcc40f83a2 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -121,7 +121,7 @@ jobs:
git push
push-buildx-cache-to-github-registry-amd:
- name: Push Regular Image Cache
+ name: Push Regular AMD Image Cache
needs: [update-constraints]
uses: ./.github/workflows/push-image-cache.yml
permissions:
@@ -129,11 +129,13 @@ jobs:
packages: write
secrets: inherit
with:
- cache-type: "Regular"
+ runs-on: '["ubuntu-22.04"]'
+ cache-type: "Regular AMD"
include-prod-images: "true"
push-latest-images: "true"
platform: "linux/amd64"
python-versions: ${{ inputs.python-versions }}
+ python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
branch: ${{ inputs.branch }}
constraints-branch: ${{ inputs.constraints-branch }}
use-uv: "true"
@@ -142,7 +144,7 @@ jobs:
if: inputs.canary-run == 'true'
push-buildx-cache-to-github-registry-arm:
- name: Push Regular Image Cache
+ name: Push Regular ARM Image Cache
needs: [update-constraints]
uses: ./.github/workflows/push-image-cache.yml
permissions:
@@ -151,11 +153,12 @@ jobs:
secrets: inherit
with:
runs-on: ${{ inputs.runs-on }}
- cache-type: "Regular"
+ cache-type: "Regular ARM"
include-prod-images: "true"
push-latest-images: "true"
platform: "linux/arm64"
python-versions: ${{ inputs.python-versions }}
+ python-versions-list-as-string: ${{ inputs.python-versions-list-as-string }}
branch: ${{ inputs.branch }}
constraints-branch: ${{ inputs.constraints-branch }}
use-uv: "true"
diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
index 27cd62d42c..9556633c40 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -252,7 +252,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
PUSH: ${{ inputs.push-image }}
DOCKER_CACHE: ${{ inputs.docker-cache }}
DEBIAN_VERSION: ${{ inputs.debian-version }}
- BUILDER: "airflow_cache"
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
@@ -260,8 +259,8 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
- name: "Build PROD images with PyPi providers ${{ matrix.python-version }}:${{ inputs.image-tag }}"
shell: bash
run: >
- breeze prod-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}"
- --commit-sha "${{ github.sha }}"
+ breeze prod-image build --builder airflow_cache --tag-as-latest
+ --image-tag "${{ inputs.image-tag }}" --commit-sha "${{ github.sha }}"
--install-packages-from-context --airflow-constraints-mode constraints
--use-constraints-for-context-packages --python "${{ matrix.python-version }}"
env:
diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml
index c89ef49302..ec0ae16c34 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -74,7 +74,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
jobs:
- push-image-cache:
+ push-ci-image-cache:
name: "Push CI ${{ inputs.cache-type }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
@@ -120,17 +120,13 @@ jobs:
if: always() && inputs.platform == 'linux/arm64'
- name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)"
run: >
- breeze ci-image build --push
+ breeze ci-image build --builder airflow_cache --push
--python "${{ matrix.python }}" --platform "${{ inputs.platform }}"
if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'
push-prod-image-cache:
name: "Push PROD ${{ inputs.cache-type }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
- strategy:
- fail-fast: false
- matrix:
- python: ${{ fromJson(inputs.python-versions) }}
env:
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
@@ -174,10 +170,13 @@ jobs:
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} ${{ inputs.platform }}"
run: >
- breeze prod-image build --builder airflow_cache
+ breeze prod-image build --builder airflow_cache --run-in-parallel
--prepare-buildx-cache --platform "${{ inputs.platform }}"
--install-packages-from-context --airflow-constraints-mode constraints-source-providers
+ --python-versions
--python ${{ matrix.python }}
+ env:
+ PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always() && inputs.platform == 'linux/arm64'
@@ -186,6 +185,6 @@ jobs:
# and we can do it after we stopped the ARM instance as it is not needed anymore
- name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)"
run: >
- breeze prod-image build --install-packages-from-context
+ breeze prod-image build --builder airflow_cache --install-packages-from-context
--push --python ${{ matrix.python}} --platform "${{ inputs.platform }}"
if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'