You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/02 14:54:41 UTC
[airflow] branch unify-dash-names created (now caccd1a98b)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch unify-dash-names
in repository https://gitbox.apache.org/repos/asf/airflow.git
at caccd1a98b Unified "dash-name" convention for outputs in ci workflows.
This branch includes the following new commits:
new caccd1a98b Unified "dash-name" convention for outputs in ci workflows.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airflow] 01/01: Unified "dash-name" convention for outputs in ci workflows.
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch unify-dash-names
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit caccd1a98bb15a80c8c736b2efea009ea28abcbb
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Jul 2 16:43:22 2022 +0200
Unified "dash-name" convention for outputs in ci workflows.
There were errors with retieving constraints branch caused by
using different convention for output names (sometimes dash,
sometimes camelCase as suggested by most GitHub documents).
The "dash-name" looks much better and is far more readable so
we shoud unify all internal outputs to follow it
---
.github/workflows/build-images.yml | 153 ++++---
.github/workflows/ci.yml | 552 ++++++++++++-----------
dev/breeze/src/airflow_breeze/branch_defaults.py | 1 +
3 files changed, 375 insertions(+), 331 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index a2218833e7..154f8b0988 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -27,7 +27,6 @@ env:
ANSWER: "yes"
CHECK_IMAGE_FOR_REBUILD: "true"
SKIP_CHECK_REMOTE_IMAGE: "true"
- DEBIAN_VERSION: "bullseye"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -51,21 +50,22 @@ jobs:
name: "Build Info"
runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }}
env:
- targetBranch: ${{ github.event.pull_request.base.ref }}
+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
outputs:
- runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }}
- pythonVersions: "${{ steps.selective-checks.python-versions }}"
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
- allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
+ runs-on: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }}
+ python-versions: "${{ steps.selective-checks.python-versions }}"
+ upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
+ all-python-versions-list-as-string: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
+ default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
image-build: ${{ steps.dynamic-outputs.outputs.image-build }}
- cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
- targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }}
- defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
- targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha ||
+ cache-directive: ${{ steps.dynamic-outputs.outputs.cache-directive }}
+ target-branch: ${{ steps.dynamic-outputs.outputs.target-branch }}
+ default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+ default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ debian-version: ${{ steps.selective-checks.outputs.debian-version }}
+ target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
github.event.pull_request.head.sha ||
github.sha
}}"
@@ -75,7 +75,7 @@ jobs:
run: |
TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV
- echo "::set-output name=targetCommitSha::${TARGET_COMMIT_SHA}"
+ echo "::set-output name=target-commit-sha::${TARGET_COMMIT_SHA}"
if: github.event_name == 'pull_request_target'
# The labels in the event aren't updated when re-triggering the job, So lets hit the API to get
# up-to-date values
@@ -104,7 +104,7 @@ jobs:
ref: ${{ env.TARGET_COMMIT_SHA }}
persist-credentials: false
fetch-depth: 2
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ - name: Checkout main branch to 'main-airflow' folder to use breeze from there.
uses: actions/checkout@v3
with:
persist-credentials: false
@@ -116,6 +116,34 @@ jobs:
python-version: 3.7
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
+ - name: "Retrieve defaults from branch_defaults.py"
+ # We cannot "execute" the branch_defaults.py python code here because that would be
+ # a security problem (we cannot run any code that comes from the sources coming from the PR.
+ # Therefore, we extract the branches via embedded Python code
+ id: defaults
+ run: |
+ python - <<EOF >>$GITHUB_ENV
+ from pathlib import Path
+ import re
+ import sys
+
+ DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
+ BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
+ CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'
+ DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$'
+
+ branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+
+ output = f"DEFAULT_BRANCH={branch}\n"
+ f"DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}"
+ f"DEBIAN_VERSION={debian_version}"
+
+ print(output)
+ # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
+ print(output, file=sys.stderr)
+ EOF
- run: ./scripts/ci/install_breeze.sh
- name: Selective checks
id: selective-checks
@@ -128,16 +156,16 @@ jobs:
run: |
set -x
if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
- echo "::set-output name=targetBranch::${targetBranch}"
+ echo "::set-output name=target-branch::${TARGET_BRANCH}"
else
# Direct push to branch, or scheduled build
- echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}"
+ echo "::set-output name=target-branch::${GITHUB_REF#refs/heads/}"
fi
if [[ "${{ github.event_name }}" == 'schedule' ]]; then
- echo "::set-output name=cacheDirective::disabled"
+ echo "::set-output name=cache-directive::disabled"
else
- echo "::set-output name=cacheDirective:registry"
+ echo "::set-output name=cache-directive:registry"
fi
if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then
@@ -158,14 +186,17 @@ jobs:
permissions:
packages: write
timeout-minutes: 80
- name: "Build CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: "Build CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: |
needs.build-info.outputs.image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
steps:
@@ -173,7 +204,7 @@ jobs:
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v3
with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
submodules: recursive
- name: "Retrieve DEFAULTS from the _initialization.sh"
@@ -204,9 +235,9 @@ jobs:
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
- name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+ Override "scripts/ci" with the "${{ needs.build-info.outputs.target-branch }}" branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
@@ -218,7 +249,7 @@ jobs:
mv -v "main-airflow/dev" "."
- uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -237,13 +268,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--show-diff-on-failure --color always || true
- name: >
Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: breeze build-image --push-image --tag-as-latest --run-in-parallel
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
if: failure() || cancelled()
run: breeze build-image --push-image --empty-image --run-in-parallel
@@ -253,7 +284,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
if: failure() || cancelled()
run: >
breeze find-newer-dependencies --max-age 1
- --python "${{ needs.build-info.outputs.defaultPythonVersion }}"
+ --python "${{ needs.build-info.outputs.default-python-version }}"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
@@ -262,21 +293,24 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
permissions:
packages: write
timeout-minutes: 80
- name: "Build PROD images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: "Build PROD images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-ci-images]
if: |
needs.build-info.outputs.image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v3
with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
submodules: recursive
- name: "Retrieve DEFAULTS from the _initialization.sh"
@@ -296,20 +330,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
cut -d "=" -f 3 | sed s'/["}]//g')
echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+ Checkout "${{ needs.build-info.outputs.target-branch }}" branch to 'main-airflow' folder
to use ci/scripts from there.
uses: actions/checkout@v3
with:
path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
+ ref: "${{ needs.build-info.outputs.target-branch }}"
persist-credentials: false
submodules: recursive
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
- name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+ Override "scripts/ci" with the "${{ needs.build-info.outputs.target-branch }}" branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
@@ -321,7 +355,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
mv -v "main-airflow/dev" "."
- uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -340,11 +374,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--show-diff-on-failure --color always || true
- name: >
Pull CI image for PROD build:
- ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+ ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
# Always use default Python version of CI image for preparing packages
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
@@ -360,7 +394,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: mv -v ./dist/*.whl ./docker-context-files
- name: >
Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-prod-image
--run-in-parallel
@@ -370,10 +404,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--disable-airflow-repo-cache
--airflow-is-in-context
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
if: failure() || cancelled()
run: breeze build-prod-image --cleanup-context --push-image --empty-image --run-in-parallel
@@ -386,15 +420,18 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
build-ci-images-arm:
timeout-minutes: 120
- name: "Build ARM CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: "Build ARM CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-prod-images]
if: |
needs.build-info.outputs.image-build == 'true' &&
- needs.build-info.outputs.upgradeToNewerDependencies != 'false' &&
+ needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
steps:
@@ -402,7 +439,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v2
with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
submodules: recursive
- name: "Retrieve DEFAULTS from the _initialization.sh"
@@ -422,16 +459,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+ Checkout "${{ needs.build-info.outputs.target-branch }}" branch to 'main-airflow' folder
to use ci/scripts from there.
uses: actions/checkout@v2
with:
path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
+ ref: "${{ needs.build-info.outputs.target-branch }}"
persist-credentials: false
submodules: recursive
- name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+ Override "scripts/ci" with the "${{ needs.build-info.outputs.target-branch }}" branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
@@ -444,7 +481,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v2
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze free-space
@@ -452,15 +489,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: >
Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-image --run-in-parallel --parallelism 1
--builder airflow_cache --platform "linux/arm64"
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always()
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 41e894e5a5..a223fb6dd0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -125,36 +125,26 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
- defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
- defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
- debianVersion: ${{ steps.selective-checks.outputs.debian-version }}
- cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
- waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
- allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
- pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
- allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
- kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
- kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
- defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
- kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
- defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
- postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
- defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
- mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
- mssqlVersions: ${{ steps.selective-checks.outputs.mssql-versions }}
- defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
- helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
- defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
- kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
- defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
- testTypes: ${{ steps.selective-checks.outputs.test-types }}
- postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
- mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
- mssqlExclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
- sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
+ debian-version: ${{ steps.selective-checks.outputs.debian-version }}
+ cache-directive: ${{ steps.dynamic-outputs.outputs.cache-directive }}
+ upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
+ python-versions: ${{ steps.selective-checks.outputs.python-versions }}
+ python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
+ all-python-versions-list-as-string: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
+ default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
+ kubernetes-versions-list-as-string: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
+ postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
+ default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
+ mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
+ mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }}
+ default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
+ default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
+ default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
+ test-types: ${{ steps.selective-checks.outputs.test-types }}
+ postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
+ mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
+ mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
+ sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
@@ -166,16 +156,16 @@ jobs:
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+ default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
docs-filter: ${{ steps.selective-checks.outputs.docs-filter }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
- sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- runsOn: ${{ steps.set-runs-on.outputs.runsOn }}
- runCoverage: ${{ steps.set-run-coverage.outputs.runCoverage }}
- inWorkflowBuild: ${{ steps.set-in-workflow-build.outputs.inWorkflowBuild }}
- buildJobDescription: ${{ steps.set-in-workflow-build.outputs.buildJobDescription }}
- mergeRun: ${{ steps.set-merge-run.outputs.merge-run }}
+ source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
+ pull-request-labels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
+ runs-on: ${{ steps.set-runs-on.outputs.runs-on }}
+ run-coverage: ${{ steps.set-run-coverage.outputs.run-coverage }}
+ in-workflow-build: ${{ steps.set-in-workflow-build.outputs.in-workflow-build }}
+ build-job-description: ${{ steps.set-in-workflow-build.outputs.build-job-description }}
+ merge-run: ${{ steps.set-merge-run.outputs.merge-run }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -202,27 +192,39 @@ jobs:
python-version: 3.7
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: ./scripts/ci/install_breeze.sh
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+ - name: "Retrieve defaults from branch_defaults.py"
+ # We cannot "execute" the branch_defaults.py python code here because that would be
+ # a security problem (we cannot run any code that comes from the sources coming from the PR.
+ # Therefore, we extract the branches via embedded Python code
id: defaults
run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+ python - <<EOF >>$GITHUB_ENV
+ from pathlib import Path
+ import re
+ import sys
+
+ DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
+ BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
+ CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'
+ DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$'
+
+ branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+
+ output = f"DEFAULT_BRANCH={branch}\n"
+ f"DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}"
+ f"DEBIAN_VERSION={debian_version}"
+
+ print(output)
+ # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
+ print(output, file=sys.stderr)
+ EOF
+ - run: ./scripts/ci/install_breeze.sh
- name: Selective checks
id: selective-checks
env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ steps.source-run-info.outputs.pull-request-labels }}"
COMMIT_REF: "${{ github.sha }}"
run: breeze selective-check
# Avoid having to specify the runs-on logic every time. We use the custom
@@ -231,22 +233,22 @@ jobs:
- name: Set runs-on
id: set-runs-on
env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ steps.source-run-info.outputs.pull-request-labels }}"
run: |
if [[ ${PR_LABELS=} == *"use public runners"* ]]; then
echo "Forcing running on Public Runners via `use public runners` label"
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
+ echo "::set-output name=runs-on::\"ubuntu-20.04\""
elif [[ ${AIRFLOW_SELF_HOSTED_RUNNER} == "" ]]; then
echo "Regular PR running with Public Runner"
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
+ echo "::set-output name=runs-on::\"ubuntu-20.04\""
else
echo "Maintainer or main run running with self-hosted runner"
- echo "::set-output name=runsOn::\"self-hosted\""
+ echo "::set-output name=runs-on::\"self-hosted\""
fi
# Avoid having to specify the coverage logic every time.
- name: Set run coverage
id: set-run-coverage
- run: echo "::set-output name=runCoverage::true"
+ run: echo "::set-output name=run-coverage::true"
if: >
github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
github.event_name == 'push' &&
@@ -262,12 +264,12 @@ jobs:
if [[ ${GITHUB_EVENT_NAME} == "push" || ${GITHUB_EVENT_NAME} == "push" || \
${{steps.source-run-info.outputs.sourceHeadRepo}} == "apache/airflow" ]]; then
echo "Images will be built in current workflow"
- echo "::set-output name=inWorkflowBuild::true"
- echo "::set-output name=buildJobDescription::Build"
+ echo "::set-output name=in-workflow-build::true"
+ echo "::set-output name=build-job-description::Build"
else
echo "Images will be built in pull_request_target workflow"
- echo "::set-output name=inWorkflowBuild::false"
- echo "::set-output name=buildJobDescription::Skip Build (pull_request_target)"
+ echo "::set-output name=in-workflow-build::false"
+ echo "::set-output name=build-job-description::Skip Build (pull_request_target)"
fi
- name: Determine if this is merge run
id: set-merge-run
@@ -284,9 +286,9 @@ jobs:
run: |
set -x
if [[ "${{ github.event_name }}" == 'schedule' ]]; then
- echo "::set-output name=cacheDirective::disabled"
+ echo "::set-output name=cache-directive::disabled"
else
- echo "::set-output name=cacheDirective::registry"
+ echo "::set-output name=cache-directive::registry"
fi
if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then
@@ -300,7 +302,7 @@ jobs:
run: printenv
env:
dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }}
- PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }}
+ PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
build-ci-images:
@@ -308,35 +310,35 @@ jobs:
packages: write
timeout-minutes: 80
name: >
- ${{needs.build-info.outputs.buildJobDescription}} CI images
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.build-job-description}} CI images
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v3
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Free space"
run: breeze free-space
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
@@ -348,63 +350,65 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: >
breeze static-checks --type update-providers-dependencies --all-files
--show-diff-on-failure --color always || true
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: >
+ needs.build-info.outputs.in-workflow-build == 'true' &&
+ steps.selective-checks.outputs.default-branch == 'main'
- name: >
Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: breeze build-image --push-image --tag-as-latest --run-in-parallel
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Candidates for pip resolver backtrack triggers"
if: failure() || cancelled()
run: >
breeze find-newer-dependencies --max-age 1
- --python "${{ needs.build-info.outputs.defaultPythonVersion }}"
+ --python "${{ needs.build-info.outputs.default-python-version }}"
- name: "Fix ownership"
run: breeze fix-ownership
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
build-prod-images:
permissions:
packages: write
timeout-minutes: 80
name: >
- ${{needs.build-info.outputs.buildJobDescription}} PROD images
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.build-job-description}} PROD images
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-ci-images]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v3
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Free space"
run: breeze free-space
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
@@ -416,34 +420,36 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: >
breeze static-checks --type update-providers-dependencies --all-files
--show-diff-on-failure --color always || true
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: >
+ needs.build-info.outputs.in-workflow-build == 'true' &&
+ steps.selective-checks.outputs.default-branch == 'main'
- name: >
Pull CI image for PROD build:
- ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
+ ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
run: breeze pull-image --tag-as-latest
env:
# Always use default Python version of CI image for preparing packages
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Prepare providers packages"
run: >
breeze prepare-provider-packages
--package-list-file ./scripts/ci/installed_providers.txt
--package-format wheel --version-suffix-for-pypi dev0
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Prepare airflow package"
run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Move dist packages to docker-context files"
run: mv -v ./dist/*.whl ./docker-context-files
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-prod-image
--tag-as-latest
@@ -453,19 +459,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--disable-airflow-repo-cache
--airflow-is-in-context
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Fix ownership"
run: breeze fix-ownership
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
run-new-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
steps:
- name: Cleanup repo
@@ -475,7 +481,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
persist-credentials: false
- uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: python -m pip install --editable ./dev/breeze/
@@ -485,7 +491,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-ui:
timeout-minutes: 10
name: React UI tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.run-ui-tests == 'true'
steps:
@@ -512,7 +518,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-www:
timeout-minutes: 10
name: React WWW tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.run-www-tests == 'true'
steps:
@@ -540,7 +546,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
test-openapi-client-generation:
timeout-minutes: 10
name: "Test OpenAPI client generation"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.needs-api-codegen == 'true'
steps:
@@ -557,7 +563,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of production image building"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.image-build == 'true'
steps:
@@ -571,7 +577,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/requirements.txt
- name: "Test examples of PROD image building"
@@ -582,11 +588,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
wait-for-ci-images:
timeout-minutes: 120
name: "Wait for CI images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.image-build == 'true'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: sqlite
steps:
- name: Cleanup repo
@@ -598,7 +604,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -608,7 +614,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
id: wait-for-images
run: breeze pull-image --run-in-parallel --verify-image --wait-for-image --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Fix ownership"
run: breeze fix-ownership
@@ -617,11 +623,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
static-checks:
timeout-minutes: 30
name: "Static checks"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: Cleanup repo
@@ -633,7 +639,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: Cache pre-commit envs
@@ -671,10 +677,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
- name: Cleanup repo
@@ -686,7 +692,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: Cache pre-commit envs
@@ -725,12 +731,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
docs:
timeout-minutes: 45
name: "Build docs"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -741,7 +747,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
submodules: recursive
- uses: actions/setup-python@v4
with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
+ python-version: ${{needs.build-info.outputs.default-python-version}}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -782,11 +788,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
prepare-test-provider-packages-wheel:
timeout-minutes: 40
name: "Build and test provider packages wheel"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
@@ -799,7 +805,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -823,7 +829,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze verify-provider-packages --use-airflow-version wheel --use-packages-from-dist
--package-format wheel
env:
- SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgradeToNewerDependencies }}"
+ SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Remove airflow package and replace providers with 2.2-compliant versions"
run: |
rm -vf dist/apache_airflow-*.whl \
@@ -857,11 +863,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
prepare-test-provider-packages-sdist:
timeout-minutes: 80
name: "Build and test provider packages sdist"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
@@ -874,7 +880,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -902,7 +908,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze verify-provider-packages --use-airflow-version sdist --use-packages-from-dist
--package-format sdist
env:
- SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgradeToNewerDependencies }}"
+ SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
@@ -910,15 +916,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-helm:
timeout-minutes: 80
name: "Python unit tests for helm chart"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
MOUNT_SELECTED_LOCAL_SOURCES: "true"
TEST_TYPES: "Helm"
BACKEND: ""
DB_RESET: "false"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.default-python-version}}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule') &&
@@ -933,7 +939,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -947,7 +953,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Tests: Helm"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -965,7 +971,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-helm
path: "./files/coverage*.xml"
@@ -978,20 +984,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
timeout-minutes: 130
name: >
Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ postgres-version: ${{ fromJson(needs.build-info.outputs.postgres-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.postgres-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: postgres
POSTGRES_VERSION: ${{ matrix.postgres-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1004,7 +1010,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1018,10 +1024,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/testing/run_downgrade_test.sh
- name: "Test Offline SQL generation"
run: ./scripts/ci/testing/run_offline_sql_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1039,7 +1045,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
path: "./files/coverage*.xml"
@@ -1051,20 +1057,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-mysql:
timeout-minutes: 130
name: >
- MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ mysql-version: ${{ fromJson(needs.build-info.outputs.mysql-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.mysql-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: mysql
MYSQL_VERSION: ${{ matrix.mysql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1077,7 +1083,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1091,10 +1097,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/testing/run_downgrade_test.sh
- name: "Test Offline SQL generation"
run: ./scripts/ci/testing/run_offline_sql_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1112,7 +1118,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
path: "./files/coverage*.xml"
@@ -1124,20 +1130,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-mssql:
timeout-minutes: 130
name: >
- MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mssql-version: ${{ fromJson(needs.build-info.outputs.mssqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mssqlExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ mssql-version: ${{ fromJson(needs.build-info.outputs.mssql-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.mssql-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: mssql
MSSQL_VERSION: ${{ matrix.mssql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1150,7 +1156,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1162,10 +1168,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Test downgrade"
run: ./scripts/ci/testing/run_downgrade_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1183,7 +1189,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}}
path: "./files/coverage*.xml"
@@ -1195,18 +1201,18 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-sqlite:
timeout-minutes: 130
name: >
- Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: sqlite
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1219,7 +1225,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1231,10 +1237,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Test downgrade"
run: ./scripts/ci/testing/run_downgrade_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1252,7 +1258,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-sqlite-${{matrix.python-version}}
path: ./files/coverage*.xml
@@ -1264,15 +1270,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-quarantined:
timeout-minutes: 60
name: "Quarantined tests"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
continue-on-error: true
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
- POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
+ POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}}
TEST_TYPES: "Quarantined"
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
@@ -1284,7 +1290,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: "Set issue id for main"
@@ -1309,7 +1315,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Tests: Quarantined"
run: ./scripts/ci/testing/ci_run_quarantined_tests.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
- name: "Upload Quarantine test results"
uses: actions/upload-artifact@v3
if: always()
@@ -1333,7 +1339,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-quarantined-${{ matrix.backend }}
path: "./files/coverage*.xml"
@@ -1345,7 +1351,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
upload-coverage:
timeout-minutes: 15
name: "Upload coverage"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
continue-on-error: true
needs:
- build-info
@@ -1355,9 +1361,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- tests-mssql
- tests-quarantined
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
# Only upload coverage on merges to main
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1380,13 +1386,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
wait-for-prod-images:
timeout-minutes: 120
name: "Wait for PROD images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images, build-prod-images]
if: needs.build-info.outputs.image-build == 'true'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1397,7 +1403,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1415,7 +1421,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
# For the images to be available and test them.
run: breeze pull-prod-image --verify-image --wait-for-image --run-in-parallel
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Fix ownership"
run: breeze fix-ownership
@@ -1424,11 +1430,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Test docker-compose quick start"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
if: needs.build-info.outputs.image-build == 'true'
env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1440,7 +1446,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1459,25 +1465,25 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-kubernetes:
timeout-minutes: 240
name: Helm Chart; ${{matrix.executor}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
strategy:
matrix:
executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
KUBERNETES_MODE: "image"
EXECUTOR: ${{matrix.executor}}
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
+ KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
+ HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
+ ${{needs.build-info.outputs.python-versions-list-as-string}}
CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
+ ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.needs-helm-tests == 'true' ) &&
@@ -1492,7 +1498,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1501,20 +1507,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --run-in-parallel --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/kubernetes-bin"
key: "kubernetes-binaries
--${{ needs.build-info.outputs.defaultKindVersion }}\
--${{ needs.build-info.outputs.defaultHelmVersion }}"
+-${{ needs.build-info.outputs.default-kind-version }}\
+-${{ needs.build-info.outputs.default-helm-version }}"
restore-keys: "kubernetes-binaries"
- name: "Kubernetes Tests"
run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
- name: "Upload KinD logs"
uses: actions/upload-artifact@v3
if: failure() || cancelled()
@@ -1529,21 +1535,21 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-helm-executor-upgrade:
timeout-minutes: 150
name: Helm Chart Executor Upgrade
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
KUBERNETES_MODE: "image"
EXECUTOR: "KubernetesExecutor"
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
+ KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
+ HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
+ ${{needs.build-info.outputs.python-versions-list-as-string}}
CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
+ ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
if: >
needs.build-info.outputs.run-kubernetes-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -1557,7 +1563,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1566,31 +1572,31 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --run-in-parallel --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/.kubernetes_venv"
- key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}
+ key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\
+ -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ -${{needs.build-info.outputs.python-versions-list-as-string}}
-${{ hashFiles('setup.py','setup.cfg') }}"
- restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}"
+ restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\
+ -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ -${{needs.build-info.outputs.python-versions-list-as-string}}"
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/kubernetes-bin"
key: "kubernetes-binaries
- -${{ needs.build-info.outputs.defaultKindVersion }}\
- -${{ needs.build-info.outputs.defaultHelmVersion }}"
+ -${{ needs.build-info.outputs.default-kind-version }}\
+ -${{ needs.build-info.outputs.default-helm-version }}"
restore-keys: "kubernetes-binaries"
- name: "Kubernetes Helm Chart Executor Upgrade Tests"
run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
- name: "Upload KinD logs"
uses: actions/upload-artifact@v3
if: failure() || cancelled()
@@ -1607,7 +1613,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
contents: write
timeout-minutes: 40
name: "Constraints"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs:
- build-info
- wait-for-ci-images
@@ -1618,8 +1624,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- tests-mssql
- tests-postgres
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1631,7 +1637,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1640,7 +1646,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --run-in-parallel --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Generate constraints"
run: |
@@ -1649,24 +1655,24 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers
breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
uses: actions/checkout@v3
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
with:
path: "repo"
ref: ${{ steps.constraints-branch.outputs.branch }}
persist-credentials: false
- - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
+ - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
- name: "Push changes"
uses: ./.github/actions/github-push-action
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.constraints-branch.outputs.branch }}
@@ -1684,19 +1690,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
packages: write
timeout-minutes: 120
name: "Push Image Cache"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs:
- build-info
- constraints
- docs
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
strategy:
fail-fast: false
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
platform: ["linux/amd64", "linux/arm64"]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
steps:
- name: Cleanup repo
@@ -1708,7 +1714,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1720,7 +1726,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: breeze pull-image --tag-as-latest
env:
# Always use default Python version of CI image for preparing packages
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
@@ -1770,9 +1776,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
build-ci-arm-images:
timeout-minutes: 120
name: >
- ${{needs.build-info.outputs.buildJobDescription}} CI ARM images
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.build-job-description}} CI ARM images
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs:
- build-info
- wait-for-ci-images
@@ -1786,46 +1792,46 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
- if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
+ if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v2
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Setup python"
uses: actions/setup-python@v2
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Free space"
run: breeze free-space
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-image --run-in-parallel --parallelism 1
--builder airflow_cache --platform "linux/arm64"
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
- name: "Fix ownership"
run: breeze fix-ownership
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py
index f4903cd077..7afc40d7f9 100644
--- a/dev/breeze/src/airflow_breeze/branch_defaults.py
+++ b/dev/breeze/src/airflow_breeze/branch_defaults.py
@@ -38,3 +38,4 @@ Examples:
AIRFLOW_BRANCH = "main"
DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main"
+DEBIAN_VERSION = "bullseye"