You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/02/17 09:53:16 UTC

(airflow) branch main updated: Distribute CI/PROD regular builds among multiple workers (#37503)

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new b3e971a330 Distribute CI/PROD regular builds among multiple workers (#37503)
b3e971a330 is described below

commit b3e971a33078340bf2c25b04c3920f12f058cf22
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Feb 17 10:53:10 2024 +0100

    Distribute CI/PROD regular builds among multiple workers (#37503)
    
    Seems that Public GitHub runners started to run out of space for
    our parallell builds. Buid parallelisation has been done in order
    to make a good use of small number of runners available, However since
    we have many runners available from Apache Software Foundation, we
    can easily just have one image per worker now without risking
    unnecessary queuing because there is no worker available.
    
    This way each build is separate and requires less disk space to complete
    the build.
---
 .github/actions/build-ci-images/action.yml   |  18 ++--
 .github/actions/build-prod-images/action.yml |  17 ++--
 .github/workflows/build-images.yml           |  31 ++++---
 .github/workflows/ci.yml                     | 119 +++++++++++++--------------
 .github/workflows/codeql-analysis.yml        |   4 +-
 .github/workflows/recheck-old-bug-report.yml |   2 +-
 .github/workflows/stale.yml                  |   2 +-
 7 files changed, 99 insertions(+), 94 deletions(-)

diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml
index 72a140604e..6cd5e8c458 100644
--- a/.github/actions/build-ci-images/action.yml
+++ b/.github/actions/build-ci-images/action.yml
@@ -18,6 +18,10 @@
 ---
 name: 'Build CI images'
 description: 'Build CI images'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    required: true
 runs:
   using: "composite"
   steps:
@@ -29,18 +33,20 @@ runs:
         pip install rich>=12.4.4 pyyaml
         python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
-    - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}"
+    - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ inputs.python-version }}"
       shell: bash
-      run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure
-    - name: "Source constraints"
+      run: >
+        breeze ci-image build --push --tag-as-latest --upgrade-on-failure
+        --python "${{ inputs.python-version }}"
+    - name: "Source constraints: ${{ inputs.python-version }}"
       shell: bash
       run: >
-        breeze release-management generate-constraints --run-in-parallel
+        breeze release-management generate-constraints --python "${{ inputs.python-version }}"
         --airflow-constraints-mode constraints-source-providers
     - name: "Upload constraint artifacts"
       uses: actions/upload-artifact@v4
       with:
-        name: source-constraints
-        path: ./files/constraints-*/constraints-*.txt
+        name: source-constraints-${{ inputs.python-version }}
+        path: ./files/constraints-*/constraints-source-providers-*.txt
         retention-days: 7
         if-no-files-found: error
diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml
index 47f3934566..5c241236c0 100644
--- a/.github/actions/build-prod-images/action.yml
+++ b/.github/actions/build-prod-images/action.yml
@@ -19,6 +19,9 @@
 name: 'Build PROD images'
 description: 'Build PROD images'
 inputs:
+  python-version:
+    description: 'Python versions to use'
+    required: true
   build-provider-packages:
     description: 'Whether to build provider packages from sources'
     required: true
@@ -64,7 +67,7 @@ runs:
     - name: "Download constraints from the CI build"
       uses: actions/download-artifact@v4
       with:
-        name: source-constraints
+        name: source-constraints-${{ inputs.python-version }}
         path: ./docker-context-files
       if: ${{ inputs.build-provider-packages == 'true' }}
     - name: "Download constraints from the Generate & Verify build"
@@ -73,21 +76,21 @@ runs:
         name: constraints
         path: ./docker-context-files
       if: ${{ inputs.build-provider-packages != 'true' }}
-    - name: "Build & Push PROD images with source providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+    - name: "Build & Push PROD images w/ source providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}"
       shell: bash
       run: >
-        breeze prod-image build --tag-as-latest --run-in-parallel --push
+        breeze prod-image build --tag-as-latest --push
         --install-packages-from-context --airflow-constraints-mode constraints-source-providers
-        --use-constraints-for-context-packages
+        --use-constraints-for-context-packages --python "${{ inputs.python-version }}"
       env:
         COMMIT_SHA: ${{ github.sha }}
       if: ${{ inputs.build-provider-packages == 'true' }}
-    - name: "Build & Push PROD images with PyPi providers ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}"
+    - name: "Build & Push PROD images with PyPi providers ${{ inputs.python-version }}:${{ env.IMAGE_TAG }}"
       shell: bash
       run: >
-        breeze prod-image build --tag-as-latest --run-in-parallel --push
+        breeze prod-image build --tag-as-latest --push
         --install-packages-from-context --airflow-constraints-mode constraints
-        --use-constraints-for-context-packages
+        --use-constraints-for-context-packages --python "${{ inputs.python-version }}"
       env:
         COMMIT_SHA: ${{ github.sha }}
       if: ${{ inputs.build-provider-packages != 'true' }}
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 5630c91119..da76fcbeff 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -60,7 +60,7 @@ jobs:
     env:
       TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
     outputs:
-      python-versions: "${{ steps.selective-checks.python-versions }}"
+      python-versions: ${{ steps.selective-checks.outputs.python-versions }}
       upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
       all-python-versions-list-as-string: >-
         ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
@@ -160,13 +160,16 @@ jobs:
           GITHUB_CONTEXT: ${{ toJson(github) }}
 
   build-ci-images:
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     permissions:
       contents: read
       packages: write
     timeout-minutes: 80
-    name: >
-      Build CI images ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build CI image ${{ matrix.python-version }}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info]
     if: |
       needs.build-info.outputs.ci-image-build == 'true' &&
@@ -222,9 +225,10 @@ jobs:
       #  BE RUN SAFELY AS PART OF DOCKER BUILD. BECAUSE IT RUNS INSIDE THE DOCKER CONTAINER AND IT IS
       #  ISOLATED FROM THE RUNNER.
       ####################################################################################################
-      - name: >
-          Build CI Images ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build CI Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-ci-images
+        with:
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -233,14 +237,16 @@ jobs:
           BUILD_TIMEOUT_MINUTES: 70
 
   build-prod-images:
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     permissions:
       contents: read
       packages: write
     timeout-minutes: 80
-    name: >
-      Build PROD images
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build PROD image ${{ matrix.python-version }}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, build-ci-images]
     if: |
       needs.build-info.outputs.prod-image-build == 'true' &&
@@ -301,13 +307,12 @@ jobs:
         uses: ./.github/actions/breeze
         with:
           python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }}
-      - name: >
-          Build PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 94267db857..a2773922c8 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -70,7 +70,7 @@ jobs:
     # option but to specify a hard-coded list here. This is "safe", the
     # runner checks if the user is an owner or collaborator of the repo
     # before running the workflow.
-    runs-on: "ubuntu-22.04"
+    runs-on: ["ubuntu-22.04"]
     env:
       GITHUB_CONTEXT: ${{ toJson(github) }}
     outputs:
@@ -267,11 +267,13 @@ jobs:
         run: breeze shell --max-time 120
 
   build-ci-images:
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
-    name: >-
-      ${{needs.build-info.outputs.build-job-description}} CI images
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: ${{needs.build-info.outputs.build-job-description}} CI image ${{ matrix.python-version }}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info]
     env:
       DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
@@ -289,19 +291,19 @@ jobs:
           ref: ${{ needs.build-info.outputs.targetCommitSha }}
           persist-credentials: false
         if: needs.build-info.outputs.in-workflow-build == 'true'
-      - name: >
-          Build CI Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build CI Images ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-ci-images
         if: needs.build-info.outputs.in-workflow-build == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
           PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           BUILD_TIMEOUT_MINUTES: 70
-      - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
-        run: breeze ci-image verify --run-in-parallel
+      - name: Verify CI images ${{ matrix.python-version }}:${{ env.IMAGE_TAG }}
+        run: breeze ci-image verify --python ${{ matrix.python-version }}
         env:
           PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
@@ -449,7 +451,7 @@ jobs:
   wait-for-ci-images:
     timeout-minutes: 120
     name: "Wait for CI images"
-    runs-on: "ubuntu-22.04"
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, build-ci-images]
     if: needs.build-info.outputs.ci-image-build == 'true'
     env:
@@ -1840,11 +1842,12 @@ jobs:
           echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l)
 
   build-prod-images:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
-    name: >
-      ${{needs.build-info.outputs.build-job-description}} PROD images (main)
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: ${{needs.build-info.outputs.build-job-description}} PROD image (main) ${{matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, build-ci-images]
     env:
       DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
@@ -1853,8 +1856,6 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      # Force more parallelism for build even on public images
-      PARALLELISM: 6
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1875,9 +1876,7 @@ jobs:
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch == 'main'
-      - name: >
-          Build PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
@@ -1885,6 +1884,7 @@ jobs:
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -1892,11 +1892,12 @@ jobs:
           DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
 
   build-prod-images-bullseye:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
-    name: >
-      Build Bullseye PROD images (main)
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build Bullseye PROD image (main) ${{matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, build-ci-images]
     if: needs.build-info.outputs.canary-run == 'true'
     env:
@@ -1906,8 +1907,6 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      # Force more parallelism for build even on public images
-      PARALLELISM: 6
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1929,9 +1928,7 @@ jobs:
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch == 'main'
-      - name: >
-          Build Bullseye PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build Bullseye PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
@@ -1939,6 +1936,7 @@ jobs:
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -1950,11 +1948,12 @@ jobs:
           IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
 
   build-prod-images-mysql-client:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
-    name: >
-      Build MySQL Client PROD images (main)
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build MySQL Client PROD image (main) ${{ matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, build-ci-images]
     if: needs.build-info.outputs.canary-run == 'true'
     env:
@@ -1964,8 +1963,6 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      # Force more parallelism for build even on public images
-      PARALLELISM: 6
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1987,9 +1984,7 @@ jobs:
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch == 'main'
-      - name: >
-          Build MySQL Client PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build MySQL Client PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
@@ -1997,6 +1992,7 @@ jobs:
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -2009,11 +2005,13 @@ jobs:
 
 
   build-prod-images-release-branch:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
     name: >
-      ${{needs.build-info.outputs.build-job-description}} PROD images (v2_*_test)
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+      ${{needs.build-info.outputs.build-job-description}} PROD image (v2_*_test) ${{matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, generate-constraints]
     env:
       DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
@@ -2022,8 +2020,6 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      # Force more parallelism for build even on public images
-      PARALLELISM: 6
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2044,9 +2040,7 @@ jobs:
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch != 'main'
-      - name: >
-          Build PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build Release PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
@@ -2054,6 +2048,7 @@ jobs:
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -2061,11 +2056,12 @@ jobs:
           DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
 
   build-prod-images-bullseye-release-branch:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
-    name: >
-      Build Bullseye PROD images (v2_*_test)
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build Bullseye PROD image (v2_*_test) ${{matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, generate-constraints]
     if: needs.build-info.outputs.canary-run == 'true'
     env:
@@ -2075,8 +2071,6 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      # Force more parallelism for build even on public images
-      PARALLELISM: 6
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2098,9 +2092,7 @@ jobs:
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch != 'main'
-      - name: >
-          Build Bullseye PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build Bullseye Release PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
@@ -2108,6 +2100,7 @@ jobs:
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
@@ -2119,11 +2112,12 @@ jobs:
           IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
 
   build-prod-images-mysql-release-branch:
+    strategy:
+      matrix:
+        python-version: ${{fromJson(needs.build-info.outputs.python-versions)}}
     timeout-minutes: 80
-    name: >
-      Build MySQL PROD images (v2_*_test)
-      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    name: Build MySQL PROD image (v2_*_test) ${{matrix.python-version}}
+    runs-on: ["ubuntu-22.04"]
     needs: [build-info, generate-constraints]
     if: needs.build-info.outputs.canary-run == 'true'
     env:
@@ -2133,8 +2127,6 @@ jobs:
       BACKEND: sqlite
       VERSION_SUFFIX_FOR_PYPI: "dev0"
       DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
-      # Force more parallelism for build even on public images
-      PARALLELISM: 6
     steps:
       - name: Cleanup repo
         run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -2156,9 +2148,7 @@ jobs:
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch != 'main'
-      - name: >
-          Build Bullseye PROD Images
-          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+      - name: Build Mysql PROD Image ${{matrix.python-version}}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
           needs.build-info.outputs.in-workflow-build == 'true' &&
@@ -2166,6 +2156,7 @@ jobs:
         with:
           build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
           chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }}
+          python-version: ${{ matrix.python-version }}
         env:
           UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
           DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 9182918701..ec608192a7 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -33,7 +33,7 @@ concurrency:
 jobs:
   selective-checks:
     name: Selective checks
-    runs-on: "ubuntu-22.04"
+    runs-on: ["ubuntu-22.04"]
     outputs:
       needs-python-scans: ${{ steps.selective-checks.outputs.needs-python-scans }}
       needs-javascript-scans: ${{ steps.selective-checks.outputs.needs-javascript-scans }}
@@ -54,7 +54,7 @@ jobs:
 
   analyze:
     name: Analyze
-    runs-on: "ubuntu-22.04"
+    runs-on: ["ubuntu-22.04"]
     needs: [selective-checks]
     strategy:
       fail-fast: false
diff --git a/.github/workflows/recheck-old-bug-report.yml b/.github/workflows/recheck-old-bug-report.yml
index 462c51362b..ee14cfde5f 100644
--- a/.github/workflows/recheck-old-bug-report.yml
+++ b/.github/workflows/recheck-old-bug-report.yml
@@ -26,7 +26,7 @@ permissions:
   issues: write
 jobs:
   recheck-old-bug-report:
-    runs-on: "ubuntu-22.04"
+    runs-on: ["ubuntu-22.04"]
     steps:
       - uses: actions/stale@v9
         with:
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 953369a0ae..2e03e9f33b 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -27,7 +27,7 @@ permissions:
   issues: write
 jobs:
   stale:
-    runs-on: "ubuntu-22.04"
+    runs-on: ["ubuntu-22.04"]
     steps:
       - uses: actions/stale@v9
         with: