You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/03/12 01:10:09 UTC
(airflow) 01/01: Extract CI image building to separate workflow
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch extract-ci-image-building
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 79a4d65b1e261e25d225c77cb2a5a2bd3c99536c
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Mar 12 02:00:24 2024 +0100
Extract CI image building to separate workflow
Similarly to #37865, we are extracting CI image buolding to a
reusable workflow. Previously we used composite action (and for
now it is still used by build-image workflow, but building CI images
in workflow should get better output and presentability with foldable
group of building images.
---
.github/workflows/ci-image-build.yml | 130 ++++++++++++++++++++++++
.github/workflows/ci.yml | 137 ++++++++++----------------
.github/workflows/prod-image-build.yml | 11 ---
.github/workflows/prod-image-extra-checks.yml | 10 --
4 files changed, 182 insertions(+), 106 deletions(-)
diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml
new file mode 100644
index 0000000000..918780a0ee
--- /dev/null
+++ b/.github/workflows/ci-image-build.yml
@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Build CI images
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ platform:
+ description: >
+ Name of the platform for the build - 'amd64/arm64'
+ required: true
+ type: string
+ debian-version:
+ description: "Base Debian distribution to use for the build (bookworm/bullseye)"
+ type: string
+ default: "bookworm"
+ install-mysql-client-type:
+ description: "MySQL client type to use during build (mariadb/mysql)"
+ type: string
+ default: "mariadb"
+ use-uv:
+ description: "Whether to use uv to build the image (true/false)"
+ required: true
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
+ required: true
+ type: string
+ python-versions:
+ description: "JSON-formatted array of Python versions to build images from"
+ required: true
+ type: string
+ branch:
+ description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ required: true
+ type: string
+ upgrade-to-newer-dependencies:
+ description: "Whether to attempt to upgrade image to newer dependencies (false/RANDOM_VALUE)"
+ required: true
+ type: string
+ breeze-python-version:
+ description: <
+ Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds)
+ required: true
+ type: string
+ constraints-branch:
+ description: "Branch used to construct constraints URL from."
+ required: true
+ type: string
+ docker-cache:
+ description: "Docker cache specification to build the image (registry, local, disabled)."
+ required: true
+ type: string
+jobs:
+ build-ci-images:
+ strategy:
+ fail-fast: true
+ matrix:
+ python-version: ${{fromJson(inputs.python-versions)}}
+ timeout-minutes: 110
+ name: Build CI ${{inputs.build-type}} image ${{matrix.python-version}}:${{inputs.image-tag}}
+ runs-on: ["ubuntu-22.04"]
+ env:
+ BACKEND: sqlite
+ DEFAULT_BRANCH: ${{ inputs.branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
+ VERSION_SUFFIX_FOR_PYPI: "dev0"
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ persist-credentials: false
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ python-version: ${{ inputs.breeze-python-version }}
+ - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
+ shell: bash
+ run: |
+ pip install rich>=12.4.4 pyyaml
+ python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
+ if: inputs.upgrade-to-newer-dependencies != 'false'
+ - name: "Start ARM instance"
+ run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+ if: ${{ inputs.platform }} == 'arm64'
+ - name: "Build & Push ${{ inputs.platform }}:${{ inputs.image-tag }} ${{ matrix.python-version }}"
+ shell: bash
+ run: >
+ breeze ci-image build --push --tag-as-latest --image-tag "${{ inputs.image-tag }}"
+ --python "${{ matrix.python-version }}"
+ --platform "linux/${{ inputs.platform }}"
+ env:
+ DOCKER_CACHE: ${{ inputs.cache-directive }}
+ INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ USE_UV: ${{ inputs.use-uv }}
+ BUILDER: ${{ 'default' if inputs.platform == 'amd64' else 'airflow_cache' }}
+ - name: "Stop ARM instance"
+ run: ./scripts/ci/images/ci_stop_arm_instance.sh
+ if: always() && ${{ inputs.platform }} == 'arm64'
+ - name: "Source constraints: ${{ matrix.python-version }}"
+ shell: bash
+ run: >
+ breeze release-management generate-constraints --python "${{ matrix.python-version }}"
+ --airflow-constraints-mode constraints-source-providers
+ - name: "Upload constraint artifacts"
+ uses: actions/upload-artifact@v4
+ with:
+ name: source-constraints-${{ matrix.python-version }}
+ path: ./files/constraints-*/constraints-source-providers-*.txt
+ retention-days: 7
+ if-no-files-found: error
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d3d69b2299..4dc8550ed8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -270,50 +270,30 @@ jobs:
- name: "Check that image builds quickly"
run: breeze shell --max-time 120
-
build-ci-images:
- strategy:
- fail-fast: true
- matrix:
- python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
- timeout-minutes: 80
- name: ${{needs.build-info.outputs.build-job-description}} CI image ${{ matrix.python-version }}
- runs-on: ["ubuntu-22.04"]
- needs: [build-info]
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- # Force more parallelism for build even on public images
- PARALLELISM: 6
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- USE_UV: "true"
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.in-workflow-build == 'true'
- - uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- if: needs.build-info.outputs.in-workflow-build == 'true'
- - name: Build CI Images ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
- uses: ./.github/actions/build-ci-images
- if: needs.build-info.outputs.in-workflow-build == 'true'
- with:
- python-version: ${{ matrix.python-version }}
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- BUILD_TIMEOUT_MINUTES: 70
- - name: Verify CI images ${{ matrix.python-version }}:${{ needs.build-info.outputs.image-tag }}
- run: breeze ci-image verify --python ${{ matrix.python-version }}
- env:
- PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- if: needs.build-info.outputs.in-workflow-build == 'true'
+ name: Build CI images (in-workflow)
+ needs: [build-info, build-ci-images]
+ uses: ./.github/workflows/ci-image-build.yml
+ permissions:
+ contents: read
+ # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
+ # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
+ # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
+ # "in-workflow-build" condition
+ packages: write
+ secrets: inherit
+ with:
+ build-type: "AMD"
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ use-uv: "true"
+ build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
+ constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
run-breeze-tests:
timeout-minutes: 10
@@ -459,7 +439,12 @@ jobs:
name: "Wait for CI images"
runs-on: ["ubuntu-22.04"]
needs: [build-info, build-ci-images]
- if: needs.build-info.outputs.ci-image-build == 'true'
+ # This strange condition below is equivalent to: "all success or skipped"
+ # The "build-ci-images" step might be skipped, in case production building happens in the
+ # "build-images" workflow, and in this case we still want to wait for CI images and run dependent jobs
+ if: >
+ always() && !failure() && !cancelled() &&
+ needs.build-info.outputs.ci-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
@@ -1892,14 +1877,12 @@ jobs:
branch: ${{ needs.build-info.outputs.default-branch }}
push-image: "true"
use-uv: "true"
- in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }}
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
if: needs.build-info.outputs.in-workflow-build == 'true'
prod-image-extra-checks-main:
@@ -1944,8 +1927,8 @@ jobs:
runs-on: ["ubuntu-22.04"]
needs: [build-info, wait-for-ci-images, build-prod-images]
# This strange condition below is equivalent to: "all success or skipped"
- # The "build-;rod-images" step might be skipped, in case production building happens in the
- # "build-images" workflow, and in this case we sill want to wait for PROD images and run depending tests
+ # The "build-prod-images" step might be skipped, in case production building happens in the
+ # "build-images" workflow, and in this case we still want to wait for PROD images and run dependent tests
if: >
always() && !failure() && !cancelled() &&
needs.build-info.outputs.prod-image-build == 'true'
@@ -2278,11 +2261,7 @@ jobs:
# There is no point in running this one in "canary" run, because the above step is doing the
# same build anyway.
build-ci-arm-images:
- timeout-minutes: 110
- name: >
- Build CI ARM images
- ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+ name: Build CI ARM images (in-workflow)
needs:
- build-info
- build-docs
@@ -2295,38 +2274,26 @@ jobs:
- tests-no-db
- tests-integration-postgres
- tests-integration-mysql
- env:
- DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
- DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- # Force more parallelism for build even on small instances
- PARALLELISM: 6
- USE_UV: "true"
+ uses: ./.github/workflows/ci-image-build.yml
+ permissions:
+ contents: read
+ # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs
+ # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo.
+ # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the
+ # "in-workflow-build" condition
+ packages: write
+ secrets: inherit
+ with:
+ platform: "arm64"
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ use-uv: "true"
+ build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+ upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }}
+ constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
if: >
needs.build-info.outputs.in-workflow-build == 'true' &&
needs.build-info.outputs.canary-run != 'true'
- steps:
- - name: Cleanup repo
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- - uses: actions/checkout@v4
- with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
- persist-credentials: false
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- - name: "Start ARM instance"
- run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- - name: >
- Build CI ARM images ${{ needs.build-info.outputs.image-tag }}
- ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
- run: |
- for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do
- breeze ci-image build --builder airflow_cache --platform "linux/arm64" --python ${python}
- done
- env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
- COMMIT_SHA: ${{ github.sha }}
- - name: "Stop ARM instance"
- run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always()
diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml
index 867cfd9aac..3c5b396764 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -46,12 +46,6 @@ on: # yamllint disable-line rule:truthy
description: "Whether to push image to the registry (true/false)"
required: true
type: string
- in-workflow-build:
- description: >
- Whether the builds are done in-workflow (true/false). If false (in PRs from fork) - they are done in
- in pull request target workflows for security.
- required: true
- type: string
python-versions:
description: "JSON-formatted array of Python versions to build images from"
required: true
@@ -85,10 +79,6 @@ on: # yamllint disable-line rule:truthy
description: "Docker cache specification to build the image (registry, local, disabled)."
required: true
type: string
- debug-resources:
- description: "Whether to debug resources while parallel builds are running (true/false)"
- required: true
- type: string
jobs:
build-prod-images:
strategy:
@@ -103,7 +93,6 @@ jobs:
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
- DEBUG_RESOURCES: ${{ inputs.debug-resources }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml
index ffe99c1e5b..9e333e477e 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -56,10 +56,6 @@ on: # yamllint disable-line rule:truthy
description: "Docker cache specification to build the image (registry, local, disabled)."
required: true
type: string
- debug-resources:
- description: "Whether to debug resources while parallel builds are running (true/false)"
- required: true
- type: string
jobs:
bullseye-image:
uses: ./.github/workflows/prod-image-build.yml
@@ -72,14 +68,12 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: "true"
- in-workflow-build: "true"
build-provider-packages: ${{ inputs.build-provider-packages }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ inputs.breeze-python-version }}
chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
- debug-resources: ${{ inputs.debug-resources }}
myssql-client-image:
uses: ./.github/workflows/prod-image-build.yml
@@ -92,14 +86,12 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: "true"
- in-workflow-build: "true"
build-provider-packages: ${{ inputs.build-provider-packages }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ inputs.breeze-python-version }}
chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
- debug-resources: ${{ inputs.debug-resources }}
pip-image:
uses: ./.github/workflows/prod-image-build.yml
@@ -112,11 +104,9 @@ jobs:
# Always build images during the extra checks and never push them
push-image: "false"
use-uv: "false"
- in-workflow-build: "true"
build-provider-packages: ${{ inputs.build-provider-packages }}
upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }}
breeze-python-version: ${{ inputs.breeze-python-version }}
chicken-egg-providers: ${{ inputs.chicken-egg-providers }}
constraints-branch: ${{ inputs.constraints-branch }}
docker-cache: ${{ inputs.docker-cache }}
- debug-resources: ${{ inputs.debug-resources }}