You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/03/25 22:15:57 UTC
(airflow) branch fix-image-cache-2 created (now 73e1923443)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch fix-image-cache-2
in repository https://gitbox.apache.org/repos/asf/airflow.git
at 73e1923443 Fix image cache optimizations - speeding up the build (#38442)
This branch includes the following new commits:
new 73e1923443 Fix image cache optimizations - speeding up the build (#38442)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow) 01/01: Fix image cache optimizations - speeding up the build (#38442)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch fix-image-cache-2
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 73e192344322947caf02da8c3ccf5550efaef4bd
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Mar 25 18:16:06 2024 +0100
Fix image cache optimizations - speeding up the build (#38442)
The recent refactors in workflows broke the way how cache had been used
in the CI builds. This PR brings back the optimizations by using the
cache and rebuilding it.
---
.github/workflows/additional-ci-image-checks.yml | 56 +++++++++---------
.github/workflows/build-images.yml | 3 +-
.github/workflows/ci-image-build.yml | 17 +++---
.github/workflows/ci.yml | 2 +
.github/workflows/finalize-tests.yml | 67 +++++++++++++--------
.github/workflows/prod-image-build.yml | 9 ++-
.github/workflows/prod-image-extra-checks.yml | 3 +
.github/workflows/push-image-cache.yml | 68 ++++++++++------------
.../airflow_breeze/utils/docker_command_utils.py | 6 +-
dev/breeze/src/airflow_breeze/utils/image.py | 14 +----
10 files changed, 128 insertions(+), 117 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml
index 84012fb24f..f73fe08ec7 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -89,42 +89,40 @@ jobs:
# delay cache refresh. It does not attempt to upgrade to newer dependencies.
# We only push CI cache as PROD cache usually does not gain as much from fresh cache because
# it uses prepared airflow and provider packages that invalidate the cache anyway most of the time
- # push-early-buildx-cache-to-github-registry:
- # name: Push Early Image Cache
- # uses: ./.github/workflows/push-image-cache.yml
- # permissions:
- # contents: read
- # # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
- # # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
- # # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
- # # "in-workflow-build" condition
- # packages: write
- # secrets: inherit
- # with:
- # runs-on: ${{ inputs.runs-on }}
- # cache-type: "Early"
- # include-prod-images: "false"
- # push-latest-images: "false"
- # image-tag: ${{ inputs.image-tag }}
- # python-versions: ${{ inputs.python-versions }}
- # branch: ${{ inputs.branch }}
- # use-uv: "true"
- # include-success-outputs: ${{ inputs.include-success-outputs }}
- # constraints-branch: ${{ inputs.constraints-branch }}
- # docker-cache: ${{ inputs.docker-cache }}
- # if: inputs.canary-run == 'true' && inputs.branch == 'main'
+ push-early-buildx-cache-to-github-registry:
+ name: Push Early Image Cache
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
+ # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
+ # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
+ # "in-workflow-build" condition
+ packages: write
+ secrets: inherit
+ with:
+ # Runs on Public runners
+ cache-type: "Early"
+ include-prod-images: "false"
+ push-latest-images: "false"
+ platform: "linux/amd64"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ use-uv: "true"
+ include-success-outputs: ${{ inputs.include-success-outputs }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true' && inputs.branch == 'main'
# Check that after earlier cache push, breeze command will build quickly
check-that-image-builds-quickly:
- timeout-minutes: 5
+ timeout-minutes: 11
name: Check that image builds quickly
runs-on: ["ubuntu-22.04"]
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
- PLATFORM: "linux/amd64"
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
PYTHON_VERSION: ${{ inputs.default-python-version }}
- IMAGE_TAG: ${{ inputs.image-tag }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: inputs.canary-run == 'true' && inputs.branch == 'main'
steps:
@@ -142,7 +140,7 @@ jobs:
- name: "Login to ghcr.io"
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- name: "Check that image builds quickly"
- run: breeze shell --max-time 120
+ run: breeze shell --max-time 600 --platform "linux/amd64"
# This is only a check if ARM images are successfully building when committer runs PR from
# Apache repository. This is needed in case you want to fix failing cache job in "canary" run
@@ -156,11 +154,11 @@ jobs:
packages: write
secrets: inherit
with:
- platform: "arm64"
push-image: "false"
runs-on: ${{ inputs.runs-on }}
image-tag: ${{ inputs.image-tag }}
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/arm64"
branch: ${{ inputs.branch }}
constraints-branch: ${{ inputs.constraints-branch }}
use-uv: "true"
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index eacbffffac..a1d5c0937c 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -180,13 +180,13 @@ jobs:
upload-constraints: "true"
use-uv: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
-
build-prod-images:
name: Build PROD images
permissions:
@@ -209,6 +209,7 @@ jobs:
push-image: "true"
use-uv: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
index 9eff0ced3a..177cd39fbb 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -48,9 +48,8 @@ on: # yamllint disable-line rule:truthy
default: "false"
type: string
platform:
- description: >
- Name of the platform for the build - 'amd64/arm64'
- default: "amd64"
+ description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+ required: true
type: string
push-image:
description: "Whether to push image to the registry (true/false)"
@@ -107,8 +106,8 @@ jobs:
timeout-minutes: 110
name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-CI ${{inputs.platform}} image\
-${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\
+CI ${{ inputs.platform }} image\
+${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
runs-on: ${{ fromJSON(inputs.runs-on) }}
env:
@@ -185,7 +184,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies != 'false'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.do-build == 'true' && inputs.platform == 'arm64'
+ if: inputs.do-build == 'true' && inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
if: inputs.do-build == 'true'
@@ -195,19 +194,19 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
run: >
breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}"
--python "${{ matrix.python-version }}"
- --platform "linux/${{ inputs.platform }}"
+ --platform "${{ inputs.platform }}"
env:
DOCKER_CACHE: ${{ inputs.docker-cache }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- BUILDER: ${{ inputs.platform == 'amd64' && 'default' || 'airflow_cache' }}
+ BUILDER: "airflow_cache"
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
if: inputs.do-build == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.do-build == 'true' && inputs.platform == 'arm64'
+ if: always() && inputs.do-build == 'true' && inputs.platform == 'linux/arm64'
- name: "Source constraints: ${{ matrix.python-version }}"
shell: bash
run: >
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3bbe7ea9e3..bcc78deb56 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -197,6 +197,7 @@ jobs:
runs-on: '["ubuntu-22.04"]'
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
upload-constraints: "true"
branch: ${{ needs.build-info.outputs.default-branch }}
@@ -498,6 +499,7 @@ jobs:
build-type: "Regular"
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
push-image: "true"
diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml
index 48971ff789..a10b1826d1 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -120,31 +120,48 @@ jobs:
run:
git push
- # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build
- # is executed as result of direct push to "main" or one of the "vX-Y-test" branches
- # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry
- # It will automatically check if a new python image was released and will pull the latest one if needed
- # push-buildx-cache-to-github-registry:
- # name: Push Regular Image Cache
- # needs: [update-constraints]
- # uses: ./.github/workflows/push-image-cache.yml
- # permissions:
- # contents: read
- # packages: write
- # secrets: inherit
- # with:
- # runs-on: ${{ inputs.runs-on }}
- # cache-type: "Regular"
- # include-prod-images: "true"
- # push-latest-images: "true"
- # use-uv: "true"
- # image-tag: ${{ inputs.image-tag }}
- # python-versions: ${{ inputs.python-versions }}
- # branch: ${{ inputs.branch }}
- # constraints-branch: ${{ inputs.constraints-branch }}
- # include-success-outputs: ${{ inputs.include-success-outputs }}
- # docker-cache: ${{ inputs.docker-cache }}
- # if: inputs.canary-run == 'true'
+ push-buildx-cache-to-github-registry-amd:
+ name: Push Regular Image Cache
+ needs: [update-constraints]
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ packages: write
+ secrets: inherit
+ with:
+ cache-type: "Regular"
+ include-prod-images: "true"
+ push-latest-images: "true"
+ platform: "linux/amd64"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ use-uv: "true"
+ include-success-outputs: ${{ inputs.include-success-outputs }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true'
+
+ push-buildx-cache-to-github-registry-arm:
+ name: Push Regular Image Cache
+ needs: [update-constraints]
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ packages: write
+ secrets: inherit
+ with:
+ runs-on: ${{ inputs.runs-on }}
+ cache-type: "Regular"
+ include-prod-images: "true"
+ push-latest-images: "true"
+ platform: "linux/arm64"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ use-uv: "true"
+ include-success-outputs: ${{ inputs.include-success-outputs }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true'
summarize-warnings:
timeout-minutes: 15
diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
index 6005e44d56..27cd62d42c 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -77,6 +77,10 @@ on: # yamllint disable-line rule:truthy
description: "JSON-formatted array of Python versions to build images from"
required: true
type: string
+ platform:
+ description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+ required: true
+ type: string
branch:
description: "Branch used to run the CI jobs in (main/v2_*_test)."
required: true
@@ -111,8 +115,8 @@ jobs:
timeout-minutes: 80
name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-PROD ${{inputs.build-type}} image\
-${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\
+PROD ${{ inputs.build-type }} image\
+${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
runs-on: ${{ fromJSON(inputs.runs-on) }}
env:
@@ -248,6 +252,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
PUSH: ${{ inputs.push-image }}
DOCKER_CACHE: ${{ inputs.docker-cache }}
DEBIAN_VERSION: ${{ inputs.debian-version }}
+ BUILDER: "airflow_cache"
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml
index 69dee4e4a9..faa904e438 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -59,6 +59,7 @@ jobs:
image-tag: bullseye-${{ inputs.image-tag }}
debian-version: "bullseye"
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
@@ -76,6 +77,7 @@ jobs:
image-tag: mysql-${{ inputs.image-tag }}
install-mysql-client-type: "mysql"
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
@@ -93,6 +95,7 @@ jobs:
image-tag: mysql-${{ inputs.image-tag }}
install-mysql-client-type: "mysql"
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml
index dbc8893a8f..c89ef49302 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -23,14 +23,14 @@ on: # yamllint disable-line rule:truthy
runs-on:
description: "The array of labels (in json form) determining type of the runner to use for the build."
required: false
- default: '["self-hosted", "Linux", "X64"]'
+ default: "[\"ubuntu-22.04\"]"
type: string
cache-type:
description: "Type of cache to push (Early / Regular)."
required: true
type: string
include-prod-images:
- description: "Whether to include prod images in the cache (true/false)."
+ description: "Whether to build PROD image cache additionally to CI image cache (true/false)."
required: true
type: string
push-latest-images:
@@ -45,12 +45,8 @@ on: # yamllint disable-line rule:truthy
description: "MySQL client type to use during build (mariadb/mysql)"
type: string
default: "mariadb"
- use-uv:
- description: "Whether to use uv to build the image (true/false)"
- required: true
- type: string
- image-tag:
- description: "Tag to set for the image"
+ platform:
+ description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
required: true
type: string
python-versions:
@@ -65,6 +61,10 @@ on: # yamllint disable-line rule:truthy
description: "Branch used to construct constraints URL from."
required: true
type: string
+ use-uv:
+ description: "Whether to use uv to build the image (true/false)"
+ required: true
+ type: string
include-success-outputs:
description: "Whether to include success outputs (true/false)."
required: true
@@ -74,25 +74,22 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
jobs:
- push-ci-image-cache:
- name: "Push CI image cache"
+ push-image-cache:
+ name: "Push CI ${{ inputs.cache-type }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
- python: ${{fromJson(inputs.python-versions)}}
+ python: ${{ fromJson(inputs.python-versions) }}
env:
- RUNS_ON: '["self-hosted", "Linux", "X64"]'
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
USE_UV: ${{ inputs.use-uv }}
- IMAGE_TAG: ${{ inputs.image-tag }}
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
UPGRADE_TO_NEWER_DEPENDENCIES: "false"
DOCKER_CACHE: ${{ inputs.docker-cache }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
- PLATFORM: "linux/amd64,linux/arm64"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.sha }}
VERBOSE: "true"
@@ -109,45 +106,40 @@ jobs:
uses: ./.github/actions/cleanup-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
- - name: "Cleanup dist and context file"
- run: rm -fv ./dist/* ./docker-context-files/*
- if: inputs.include-prod-images == 'true'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+ if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }} (AMD/ARM)"
+ - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }} ${{ inputs.platform }}"
run: >
- breeze ci-image build --builder airflow_cache
- --prepare-buildx-cache --run-in-parallel --platform "linux/amd64,linux/arm64"
- --python ${{ matrix.python }}
+ breeze ci-image build --builder airflow_cache --prepare-buildx-cache
+ --platform "${{ inputs.platform }}" --python ${{ matrix.python }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always()
- - name: "Push CI latest images: ${{ matrix.python }} (AMD ONLY)"
+ if: always() && inputs.platform == 'linux/arm64'
+ - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)"
run: >
- breeze ci-image build --tag-as-latest --push --python "${{matrix.python}}" --platform "linux/amd64"
- if: inputs.push-latest-images == 'true'
+ breeze ci-image build --push
+ --python "${{ matrix.python }}" --platform "${{ inputs.platform }}"
+ if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'
push-prod-image-cache:
- name: "Push PROD image cache"
+ name: "Push PROD ${{ inputs.cache-type }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
- python: ${{fromJson(inputs.python-versions)}}
+ python: ${{ fromJson(inputs.python-versions) }}
env:
- RUNS_ON: '["self-hosted", "Linux", "X64"]'
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
USE_UV: ${{ inputs.use-uv }}
- IMAGE_TAG: ${{ inputs.image-tag }}
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
UPGRADE_TO_NEWER_DEPENDENCIES: "false"
DOCKER_CACHE: ${{ inputs.docker-cache }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
- PLATFORM: "linux/amd64,linux/arm64"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.sha }}
VERBOSE: "true"
@@ -177,23 +169,23 @@ jobs:
run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+ if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
- - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} (AMD/ARM)"
+ - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} ${{ inputs.platform }}"
run: >
breeze prod-image build --builder airflow_cache
- --prepare-buildx-cache --run-in-parallel --platform "linux/amd64,linux/arm64"
+ --prepare-buildx-cache --platform "${{ inputs.platform }}"
--install-packages-from-context --airflow-constraints-mode constraints-source-providers
--python ${{ matrix.python }}
- if: inputs.include-prod-images == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always()
+ if: always() && inputs.platform == 'linux/arm64'
# We only push "AMD" images as it is really only needed for any kind of automated builds in CI
# and currently there is not an easy way to make multi-platform image from two separate builds
# and we can do it after we stopped the ARM instance as it is not needed anymore
- - name: "Push PROD latest image: ${{ matrix.python }} (AMD ONLY)"
+ - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)"
run: >
- breeze prod-image build --tag-as-latest --install-packages-from-context
- --push --python ${{ matrix.python}} --platform "linux/amd64"
- if: inputs.push-latest-images == 'true'
+ breeze prod-image build --install-packages-from-context
+ --push --python ${{ matrix.python}} --platform "${{ inputs.platform }}"
+ if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64'
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 3ad92a19e5..cb75be5f19 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -381,12 +381,15 @@ def prepare_base_build_command(image_params: CommonBuildParams) -> list[str]:
]
)
if not image_params.docker_host:
+ builder = get_and_use_docker_context(image_params.builder)
build_command_param.extend(
[
"--builder",
- get_and_use_docker_context(image_params.builder),
+ builder,
]
)
+ if builder != "default":
+ build_command_param.append("--load")
else:
build_command_param.append("build")
return build_command_param
@@ -658,6 +661,7 @@ def autodetect_docker_context():
def get_and_use_docker_context(context: str):
if context == "autodetect":
context = autodetect_docker_context()
+ run_command(["docker", "context", "create", context], check=False)
output = run_command(["docker", "context", "use", context], check=False)
if output.returncode != 0:
get_console().print(
diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py
index 5172afc15e..3adc920a4c 100644
--- a/dev/breeze/src/airflow_breeze/utils/image.py
+++ b/dev/breeze/src/airflow_breeze/utils/image.py
@@ -197,18 +197,8 @@ def tag_image_as_latest(image_params: CommonBuildParams, output: Output | None)
check=False,
)
if command.returncode != 0:
- return command
- if image_params.push:
- command = run_command(
- [
- "docker",
- "push",
- image_params.airflow_image_name + ":latest",
- ],
- output=output,
- capture_output=True,
- check=False,
- )
+ get_console(output=output).print(command.stdout)
+ get_console(output=output).print(command.stderr)
return command