You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/02 21:35:43 UTC
[airflow] branch main updated: Unified "dash-name" convention for outputs in ci workflows. (#24802)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7179227a16 Unified "dash-name" convention for outputs in ci workflows. (#24802)
7179227a16 is described below
commit 7179227a165fb8a4c3fdca37a45e8e1831cf9b6a
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Jul 2 23:35:23 2022 +0200
Unified "dash-name" convention for outputs in ci workflows. (#24802)
There were errors with retieving constraints branch caused by
using different convention for output names (sometimes dash,
sometimes camelCase as suggested by most GitHub documents).
The "dash-name" looks much better and is far more readable so
we shoud unify all internal outputs to follow it.
During that rename some old, unused outputs were removed,
also it turned out that the new selective-check can
replace previous "dynamic outputs" written in Bash as well.
Additionally, the "defaults" are now retrieved via Python script, not
bash script which will make it much more readable - both build_images
and ci.yaml use it in the right place - before replacing
the scripts and dev with the version coming in from PR in case
of build_images.yaml.
---
.github/workflows/build-images.yml | 262 +++++----
.github/workflows/ci.yml | 583 ++++++++++-----------
dev/breeze/src/airflow_breeze/branch_defaults.py | 1 +
.../src/airflow_breeze/utils/selective_checks.py | 4 +
4 files changed, 411 insertions(+), 439 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index a2218833e7..03e156dacf 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -27,7 +27,6 @@ env:
ANSWER: "yes"
CHECK_IMAGE_FOR_REBUILD: "true"
SKIP_CHECK_REMOTE_IMAGE: "true"
- DEBIAN_VERSION: "bullseye"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -51,21 +50,22 @@ jobs:
name: "Build Info"
runs-on: ${{ github.repository == 'apache/airflow' && 'self-hosted' || 'ubuntu-20.04' }}
env:
- targetBranch: ${{ github.event.pull_request.base.ref }}
+ TARGET_BRANCH: ${{ github.event.pull_request.base.ref }}
outputs:
- runsOn: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }}
- pythonVersions: "${{ steps.selective-checks.python-versions }}"
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
- allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
+ runs-on: ${{ github.repository == 'apache/airflow' && '["self-hosted"]' || '["ubuntu-20.04"]' }}
+ python-versions: "${{ steps.selective-checks.python-versions }}"
+ upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
+ all-python-versions-list-as-string: >-
+ ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
+ default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
- image-build: ${{ steps.dynamic-outputs.outputs.image-build }}
- cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
- targetBranch: ${{ steps.dynamic-outputs.outputs.targetBranch }}
- defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
- targetCommitSha: "${{steps.discover-pr-merge-commit.outputs.targetCommitSha ||
+ image-build: ${{ steps.selective-checks.outputs.image-build }}
+ cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
+ default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+ default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+ debian-version: ${{ steps.selective-checks.outputs.debian-version }}
+ target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
github.event.pull_request.head.sha ||
github.sha
}}"
@@ -75,14 +75,14 @@ jobs:
run: |
TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> $GITHUB_ENV
- echo "::set-output name=targetCommitSha::${TARGET_COMMIT_SHA}"
+ echo "::set-output name=target-commit-sha::${TARGET_COMMIT_SHA}"
if: github.event_name == 'pull_request_target'
# The labels in the event aren't updated when re-triggering the job, So lets hit the API to get
# up-to-date values
- name: Get latest PR labels
id: get-latest-pr-labels
run: |
- echo -n "::set-output name=pullRequestLabels::"
+ echo -n "::set-output name=pull-request-labels::"
gh api graphql --paginate -F node_id=${{github.event.pull_request.node_id}} -f query='
query($node_id: ID!, $endCursor: String) {
node(id:$node_id) {
@@ -104,11 +104,6 @@ jobs:
ref: ${{ env.TARGET_COMMIT_SHA }}
persist-credentials: false
fetch-depth: 2
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v3
- with:
- persist-credentials: false
- submodules: recursive
- name: "Setup python"
uses: actions/setup-python@v4
with:
@@ -116,56 +111,72 @@ jobs:
python-version: 3.7
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
+ - name: "Retrieve defaults from branch_defaults.py"
+ # We cannot "execute" the branch_defaults.py python code here because that would be
+ # a security problem (we cannot run any code that comes from the sources coming from the PR.
+ # Therefore, we extract the branches via embedded Python code
+ # we need to do it before next step replaces checked-out breeze and scripts code coming from
+ # the PR, because the PR defaults have to be retrieved here.
+ id: defaults
+ run: |
+ python - <<EOF >>$GITHUB_ENV
+ from pathlib import Path
+ import re
+ import sys
+
+ DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
+ BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
+ CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'
+ DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$'
+
+ branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+
+ output = f"""
+ DEFAULT_BRANCH={branch}
+ DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
+ DEBIAN_VERSION={debian_version}
+ """.strip()
+
+ print(output)
+ # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
+ print(output, file=sys.stderr)
+ EOF
+ - name: Checkout main branch to use breeze from there.
+ uses: actions/checkout@v3
+ with:
+ ref: "main"
+ persist-credentials: false
+ submodules: recursive
- run: ./scripts/ci/install_breeze.sh
- name: Selective checks
id: selective-checks
env:
- PR_LABELS: "$${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }}"
+ PR_LABELS: "$${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}"
COMMIT_REF: "${{ env.TARGET_COMMIT_SHA }}"
run: breeze selective-check
- - name: Compute dynamic outputs
- id: dynamic-outputs
- run: |
- set -x
- if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
- echo "::set-output name=targetBranch::${targetBranch}"
- else
- # Direct push to branch, or scheduled build
- echo "::set-output name=targetBranch::${GITHUB_REF#refs/heads/}"
- fi
-
- if [[ "${{ github.event_name }}" == 'schedule' ]]; then
- echo "::set-output name=cacheDirective::disabled"
- else
- echo "::set-output name=cacheDirective:registry"
- fi
-
- if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then
- echo "::set-output name=image-build::true"
- else
- echo "::set-output name=image-build::false"
- fi
- env:
- SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }}
- name: env
run: printenv
env:
- dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }}
- PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }}
+ PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
build-ci-images:
permissions:
packages: write
timeout-minutes: 80
- name: "Build CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: "Build CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: |
needs.build-info.outputs.image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
steps:
@@ -173,40 +184,24 @@ jobs:
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v3
with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+ - name: "Setup python"
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
- name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+ Checkout "main branch to 'main-airflow' folder
to use ci/scripts from there.
uses: actions/checkout@v3
with:
path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
+ ref: "main"
persist-credentials: false
submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+ Override "scripts/ci" with the "main" branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
@@ -218,7 +213,7 @@ jobs:
mv -v "main-airflow/dev" "."
- uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -235,15 +230,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: >
breeze static-checks --type update-providers-dependencies --all-files
--show-diff-on-failure --color always || true
- - name: >
+ if: needs.build-info.outputs.default-branch == 'main'
+ - name: >-
Build & Push AMD64 CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: breeze build-image --push-image --tag-as-latest --run-in-parallel
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
if: failure() || cancelled()
run: breeze build-image --push-image --empty-image --run-in-parallel
@@ -253,7 +249,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
if: failure() || cancelled()
run: >
breeze find-newer-dependencies --max-age 1
- --python "${{ needs.build-info.outputs.defaultPythonVersion }}"
+ --python "${{ needs.build-info.outputs.default-python-version }}"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
@@ -262,54 +258,41 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
permissions:
packages: write
timeout-minutes: 80
- name: "Build PROD images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: "Build PROD images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-ci-images]
if: |
needs.build-info.outputs.image-build == 'true' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v3
with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- cut -d "=" -f 3 | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+ - name: "Setup python"
+ uses: actions/setup-python@v4
+ with:
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
- name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+ Checkout "main" branch to 'main-airflow' folder
to use ci/scripts from there.
uses: actions/checkout@v3
with:
path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
+ ref: "main"
persist-credentials: false
submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v4
- with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+ Override "scripts/ci" with the "main" branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
@@ -321,7 +304,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
mv -v "main-airflow/dev" "."
- uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -334,17 +317,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ if: needs.build-info.outputs.default-branch == 'main'
- name: "Regenerate dependencies in case they was modified manually so that we can build an image"
run: >
breeze static-checks --type update-providers-dependencies --all-files
--show-diff-on-failure --color always || true
+ if: needs.build-info.outputs.default-branch == 'main'
- name: >
Pull CI image for PROD build:
- ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
+ ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --tag-as-latest
env:
# Always use default Python version of CI image for preparing packages
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
@@ -358,9 +343,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
- name: "Move dist packages to docker-context files"
run: mv -v ./dist/*.whl ./docker-context-files
- - name: >
+ - name: >-
Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-prod-image
--run-in-parallel
@@ -370,10 +355,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--disable-airflow-repo-cache
--airflow-is-in-context
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
if: failure() || cancelled()
run: breeze build-prod-image --cleanup-context --push-image --empty-image --run-in-parallel
@@ -386,15 +371,18 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
build-ci-images-arm:
timeout-minutes: 120
- name: "Build ARM CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: "Build ARM CI images ${{ needs.build-info.outputs.all-python-versions-list-as-string }}"
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-prod-images]
if: |
needs.build-info.outputs.image-build == 'true' &&
- needs.build-info.outputs.upgradeToNewerDependencies != 'false' &&
+ needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
github.event.pull_request.head.repo.full_name != 'apache/airflow'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+ DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
outputs: ${{toJSON(needs.build-info.outputs) }}
steps:
@@ -402,36 +390,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- uses: actions/checkout@v2
with:
- ref: ${{ needs.build-info.outputs.targetCommitSha }}
+ ref: ${{ needs.build-info.outputs.target-commit-sha }}
persist-credentials: false
submodules: recursive
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
- id: defaults
- run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
- name: >
- Checkout "${{ needs.build-info.outputs.targetBranch }}" branch to 'main-airflow' folder
+ Checkout "main" branch to 'main-airflow' folder
to use ci/scripts from there.
uses: actions/checkout@v2
with:
path: "main-airflow"
- ref: "${{ needs.build-info.outputs.targetBranch }}"
+ ref: "main"
persist-credentials: false
submodules: recursive
- name: >
- Override "scripts/ci" with the "${{ needs.build-info.outputs.targetBranch }}" branch
+ Override "scripts/ci" with the "main" branch
so that the PR does not override it
# We should not override those scripts which become part of the image as they will not be
# changed in the image built - we should only override those that are executed to build
@@ -444,7 +416,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v2
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
- run: ./scripts/ci/install_breeze.sh
- name: "Free space"
run: breeze free-space
@@ -452,15 +424,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- name: >
Build ARM CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-image --run-in-parallel --parallelism 1
--builder airflow_cache --platform "linux/arm64"
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
if: always()
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 41e894e5a5..5cfe017ff7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -125,36 +125,28 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
- defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
- defaultConstraintsBranch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
- debianVersion: ${{ steps.selective-checks.outputs.debian-version }}
- cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
- waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
- allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
- upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
- pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
- allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
- defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
- kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
- kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
- defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
- kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
- defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
- postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
- defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
- mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
- mssqlVersions: ${{ steps.selective-checks.outputs.mssql-versions }}
- defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
- helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
- defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
- kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
- defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
- testTypes: ${{ steps.selective-checks.outputs.test-types }}
- postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
- mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
- mssqlExclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
- sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
+ debian-version: ${{ steps.selective-checks.outputs.debian-version }}
+ cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
+ upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
+ python-versions: ${{ steps.selective-checks.outputs.python-versions }}
+ python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
+ all-python-versions-list-as-string: >-
+ ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
+ default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
+ kubernetes-versions-list-as-string: >-
+ ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
+ postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
+ default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
+ mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
+ mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }}
+ default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
+ default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
+ default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
+ test-types: ${{ steps.selective-checks.outputs.test-types }}
+ postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
+ mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
+ mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
+ sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }}
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
@@ -166,16 +158,16 @@ jobs:
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+ default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
docs-filter: ${{ steps.selective-checks.outputs.docs-filter }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
- sourceHeadRepo: ${{ steps.source-run-info.outputs.sourceHeadRepo }}
- pullRequestNumber: ${{ steps.source-run-info.outputs.pullRequestNumber }}
- pullRequestLabels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
- runsOn: ${{ steps.set-runs-on.outputs.runsOn }}
- runCoverage: ${{ steps.set-run-coverage.outputs.runCoverage }}
- inWorkflowBuild: ${{ steps.set-in-workflow-build.outputs.inWorkflowBuild }}
- buildJobDescription: ${{ steps.set-in-workflow-build.outputs.buildJobDescription }}
- mergeRun: ${{ steps.set-merge-run.outputs.merge-run }}
+ source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
+ pull-request-labels: ${{ steps.source-run-info.outputs.pullRequestLabels }}
+ runs-on: ${{ steps.set-runs-on.outputs.runs-on }}
+ run-coverage: ${{ steps.set-run-coverage.outputs.run-coverage }}
+ in-workflow-build: ${{ steps.set-in-workflow-build.outputs.in-workflow-build }}
+ build-job-description: ${{ steps.set-in-workflow-build.outputs.build-job-description }}
+ merge-run: ${{ steps.set-merge-run.outputs.merge-run }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -202,27 +194,42 @@ jobs:
python-version: 3.7
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- - run: ./scripts/ci/install_breeze.sh
- - name: "Retrieve DEFAULTS from the _initialization.sh"
- # We cannot "source" the script here because that would be a security problem (we cannot run
- # any code that comes from the sources coming from the PR. Therefore, we extract the
- # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands
+ - name: "Retrieve defaults from branch_defaults.py"
id: defaults
+ # We could retrieve it differently here - by just importing the variables and
+ # printing them from python code, however we want to have the same code as used in
+ # the build-images.yml (there we cannot import python code coming from the PR - we need to
+ # treat the python code as text and extract the variables from there.
run: |
- DEFAULT_BRANCH=$(grep "export DEFAULT_BRANCH" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_BRANCH=${DEFAULT_BRANCH}" >> $GITHUB_ENV
- DEFAULT_CONSTRAINTS_BRANCH=$(grep "export DEFAULT_CONSTRAINTS_BRANCH" \
- scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH}" >> $GITHUB_ENV
- DEBIAN_VERSION=$(grep "export DEBIAN_VERSION" scripts/ci/libraries/_initialization.sh | \
- awk 'BEGIN{FS="="} {print $3}' | sed s'/["}]//g')
- echo "DEBIAN_VERSION=${DEBIAN_VERSION}" >> $GITHUB_ENV
+ python - <<EOF >>$GITHUB_ENV
+ from pathlib import Path
+ import re
+ import sys
+
+ DEFAULTS_CONTENT = Path('dev/breeze/src/airflow_breeze/branch_defaults.py').read_text()
+ BRANCH_PATTERN = r'^AIRFLOW_BRANCH = "(.*)"$'
+ CONSTRAINTS_BRANCH_PATTERN = r'^DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "(.*)"$'
+ DEBIAN_VERSION_PATTERN = r'^DEBIAN_VERSION = "(.*)"$'
+
+ branch = re.search(BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ constraints_branch = re.search(CONSTRAINTS_BRANCH_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+ debian_version = re.search(DEBIAN_VERSION_PATTERN, DEFAULTS_CONTENT, re.MULTILINE).group(1)
+
+ output = f"""
+ DEFAULT_BRANCH={branch}
+ DEFAULT_CONSTRAINTS_BRANCH={constraints_branch}
+ DEBIAN_VERSION={debian_version}
+ """.strip()
+
+ print(output)
+ # Stdout is redirected to GITHUB_ENV but we also print it to stderr to see it in ci log
+ print(output, file=sys.stderr)
+ EOF
+ - run: ./scripts/ci/install_breeze.sh
- name: Selective checks
id: selective-checks
env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ steps.source-run-info.outputs.pull-request-labels }}"
COMMIT_REF: "${{ github.sha }}"
run: breeze selective-check
# Avoid having to specify the runs-on logic every time. We use the custom
@@ -231,22 +238,22 @@ jobs:
- name: Set runs-on
id: set-runs-on
env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ steps.source-run-info.outputs.pull-request-labels }}"
run: |
if [[ ${PR_LABELS=} == *"use public runners"* ]]; then
echo "Forcing running on Public Runners via `use public runners` label"
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
+ echo "::set-output name=runs-on::\"ubuntu-20.04\""
elif [[ ${AIRFLOW_SELF_HOSTED_RUNNER} == "" ]]; then
echo "Regular PR running with Public Runner"
- echo "::set-output name=runsOn::\"ubuntu-20.04\""
+ echo "::set-output name=runs-on::\"ubuntu-20.04\""
else
echo "Maintainer or main run running with self-hosted runner"
- echo "::set-output name=runsOn::\"self-hosted\""
+ echo "::set-output name=runs-on::\"self-hosted\""
fi
# Avoid having to specify the coverage logic every time.
- name: Set run coverage
id: set-run-coverage
- run: echo "::set-output name=runCoverage::true"
+ run: echo "::set-output name=run-coverage::true"
if: >
github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
github.event_name == 'push' &&
@@ -262,12 +269,12 @@ jobs:
if [[ ${GITHUB_EVENT_NAME} == "push" || ${GITHUB_EVENT_NAME} == "push" || \
${{steps.source-run-info.outputs.sourceHeadRepo}} == "apache/airflow" ]]; then
echo "Images will be built in current workflow"
- echo "::set-output name=inWorkflowBuild::true"
- echo "::set-output name=buildJobDescription::Build"
+ echo "::set-output name=in-workflow-build::true"
+ echo "::set-output name=build-job-description::Build"
else
echo "Images will be built in pull_request_target workflow"
- echo "::set-output name=inWorkflowBuild::false"
- echo "::set-output name=buildJobDescription::Skip Build (pull_request_target)"
+ echo "::set-output name=in-workflow-build::false"
+ echo "::set-output name=build-job-description::Skip Build (pull_request_target)"
fi
- name: Determine if this is merge run
id: set-merge-run
@@ -279,64 +286,46 @@ jobs:
github.ref_name == 'main' ||
startsWith(github.ref_name, 'v2') && endsWith(github.ref_name, 'test')
)
- - name: Compute dynamic outputs
- id: dynamic-outputs
- run: |
- set -x
- if [[ "${{ github.event_name }}" == 'schedule' ]]; then
- echo "::set-output name=cacheDirective::disabled"
- else
- echo "::set-output name=cacheDirective::registry"
- fi
-
- if [[ "$SELECTIVE_CHECKS_IMAGE_BUILD" == "true" ]]; then
- echo "::set-output name=image-build::true"
- else
- echo "::set-output name=image-build::false"
- fi
- env:
- SELECTIVE_CHECKS_IMAGE_BUILD: ${{ steps.selective-checks.outputs.image-build }}
- name: env
run: printenv
env:
- dynamicOutputs: ${{ toJSON(steps.dynamic-outputs.outputs) }}
- PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pullRequestLabels }}
+ PR_LABELS: ${{ steps.get-latest-pr-labels.outputs.pull-request-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
build-ci-images:
permissions:
packages: write
timeout-minutes: 80
- name: >
- ${{needs.build-info.outputs.buildJobDescription}} CI images
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ name: >-
+ ${{needs.build-info.outputs.build-job-description}} CI images
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v3
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Free space"
run: breeze free-space
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
@@ -344,67 +333,72 @@ jobs:
key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ if: >
+ needs.build-info.outputs.in-workflow-build == 'true' &&
+ needs.build-info.outputs.default-branch == 'main'
- name: "Regenerate dependencies in case they was modified manually so that we can build an image"
run: >
breeze static-checks --type update-providers-dependencies --all-files
--show-diff-on-failure --color always || true
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: >
+ needs.build-info.outputs.in-workflow-build == 'true' &&
+ needs.build-info.outputs.default-branch == 'main'
- name: >
Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: breeze build-image --push-image --tag-as-latest --run-in-parallel
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Candidates for pip resolver backtrack triggers"
if: failure() || cancelled()
run: >
breeze find-newer-dependencies --max-age 1
- --python "${{ needs.build-info.outputs.defaultPythonVersion }}"
+ --python "${{ needs.build-info.outputs.default-python-version }}"
- name: "Fix ownership"
run: breeze fix-ownership
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
build-prod-images:
permissions:
packages: write
timeout-minutes: 80
name: >
- ${{needs.build-info.outputs.buildJobDescription}} PROD images
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.build-job-description}} PROD images
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-ci-images]
env:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
BACKEND: sqlite
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v3
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Free space"
run: breeze free-space
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: Cache pre-commit envs
uses: actions/cache@v3
with:
@@ -412,38 +406,43 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\
${{ hashFiles('.pre-commit-config.yaml') }}"
restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}}
+ if: >
+ needs.build-info.outputs.in-workflow-build == 'true' &&
+ needs.build-info.outputs.default-branch == 'main'
- name: "Regenerate dependencies in case they was modified manually so that we can build an image"
run: >
breeze static-checks --type update-providers-dependencies --all-files
--show-diff-on-failure --color always || true
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: >
+ needs.build-info.outputs.in-workflow-build == 'true' &&
+ needs.build-info.outputs.default-branch == 'main'
- name: >
Pull CI image for PROD build:
- ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
+ ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}"
run: breeze pull-image --tag-as-latest
env:
# Always use default Python version of CI image for preparing packages
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Prepare providers packages"
run: >
breeze prepare-provider-packages
--package-list-file ./scripts/ci/installed_providers.txt
--package-format wheel --version-suffix-for-pypi dev0
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Prepare airflow package"
run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Move dist packages to docker-context files"
run: mv -v ./dist/*.whl ./docker-context-files
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-prod-image
--tag-as-latest
@@ -453,19 +452,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
--disable-airflow-repo-cache
--airflow-is-in-context
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Fix ownership"
run: breeze fix-ownership
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
run-new-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
steps:
- name: Cleanup repo
@@ -475,7 +474,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
persist-credentials: false
- uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: python -m pip install --editable ./dev/breeze/
@@ -485,7 +484,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-ui:
timeout-minutes: 10
name: React UI tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.run-ui-tests == 'true'
steps:
@@ -512,7 +511,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-www:
timeout-minutes: 10
name: React WWW tests
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.run-www-tests == 'true'
steps:
@@ -540,7 +539,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
test-openapi-client-generation:
timeout-minutes: 10
name: "Test OpenAPI client generation"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.needs-api-codegen == 'true'
steps:
@@ -557,7 +556,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
test-examples-of-prod-image-building:
timeout-minutes: 60
name: "Test examples of production image building"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
if: needs.build-info.outputs.image-build == 'true'
steps:
@@ -571,7 +570,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/requirements.txt
- name: "Test examples of PROD image building"
@@ -582,11 +581,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
wait-for-ci-images:
timeout-minutes: 120
name: "Wait for CI images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, build-ci-images]
if: needs.build-info.outputs.image-build == 'true'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: sqlite
steps:
- name: Cleanup repo
@@ -598,7 +597,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -608,7 +607,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
id: wait-for-images
run: breeze pull-image --run-in-parallel --verify-image --wait-for-image --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Fix ownership"
run: breeze fix-ownership
@@ -617,11 +616,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
static-checks:
timeout-minutes: 30
name: "Static checks"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: Cleanup repo
@@ -633,7 +632,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: Cache pre-commit envs
@@ -671,22 +670,18 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
if: needs.build-info.outputs.basic-checks-only == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v3
- with:
- persist-credentials: false
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: Cache pre-commit envs
@@ -725,12 +720,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
docs:
timeout-minutes: 45
name: "Build docs"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
if: needs.build-info.outputs.docs-build == 'true'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -741,7 +736,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
submodules: recursive
- uses: actions/setup-python@v4
with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
+ python-version: ${{needs.build-info.outputs.default-python-version}}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -782,11 +777,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
prepare-test-provider-packages-wheel:
timeout-minutes: 40
name: "Build and test provider packages wheel"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
@@ -799,7 +794,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -823,7 +818,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze verify-provider-packages --use-airflow-version wheel --use-packages-from-dist
--package-format wheel
env:
- SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgradeToNewerDependencies }}"
+ SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Remove airflow package and replace providers with 2.2-compliant versions"
run: |
rm -vf dist/apache_airflow-*.whl \
@@ -857,11 +852,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
prepare-test-provider-packages-sdist:
timeout-minutes: 80
name: "Build and test provider packages sdist"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
@@ -874,7 +869,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -902,7 +897,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze verify-provider-packages --use-airflow-version sdist --use-packages-from-dist
--package-format sdist
env:
- SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgradeToNewerDependencies }}"
+ SKIP_CONSTRAINTS: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- name: "Fix ownership"
run: breeze fix-ownership
if: always()
@@ -910,15 +905,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-helm:
timeout-minutes: 80
name: "Python unit tests for helm chart"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
MOUNT_SELECTED_LOCAL_SOURCES: "true"
TEST_TYPES: "Helm"
BACKEND: ""
DB_RESET: "false"
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.default-python-version}}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule') &&
@@ -933,7 +928,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -947,7 +942,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Tests: Helm"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -965,7 +960,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-helm
path: "./files/coverage*.xml"
@@ -978,20 +973,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
timeout-minutes: 130
name: >
Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ postgres-version: ${{ fromJson(needs.build-info.outputs.postgres-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.postgres-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: postgres
POSTGRES_VERSION: ${{ matrix.postgres-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1004,7 +999,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1018,10 +1013,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/testing/run_downgrade_test.sh
- name: "Test Offline SQL generation"
run: ./scripts/ci/testing/run_offline_sql_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1039,7 +1034,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}}
path: "./files/coverage*.xml"
@@ -1051,20 +1046,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-mysql:
timeout-minutes: 130
name: >
- MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ mysql-version: ${{ fromJson(needs.build-info.outputs.mysql-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.mysql-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: mysql
MYSQL_VERSION: ${{ matrix.mysql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1077,7 +1072,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1091,10 +1086,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/testing/run_downgrade_test.sh
- name: "Test Offline SQL generation"
run: ./scripts/ci/testing/run_offline_sql_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1112,7 +1107,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}}
path: "./files/coverage*.xml"
@@ -1124,20 +1119,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-mssql:
timeout-minutes: 130
name: >
- MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- mssql-version: ${{ fromJson(needs.build-info.outputs.mssqlVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.mssqlExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ mssql-version: ${{ fromJson(needs.build-info.outputs.mssql-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.mssql-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: mssql
MSSQL_VERSION: ${{ matrix.mssql-version }}
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1150,7 +1145,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1162,10 +1157,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Test downgrade"
run: ./scripts/ci/testing/run_downgrade_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1183,7 +1178,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}}
path: "./files/coverage*.xml"
@@ -1195,18 +1190,18 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-sqlite:
timeout-minutes: 130
name: >
- Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images]
strategy:
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
- exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
+ exclude: ${{ fromJson(needs.build-info.outputs.sqlite-exclude) }}
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: sqlite
- TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
+ TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
@@ -1219,7 +1214,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1231,10 +1226,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Test downgrade"
run: ./scripts/ci/testing/run_downgrade_test.sh
- - name: "Tests: ${{needs.build-info.outputs.testTypes}}"
+ - name: "Tests: ${{needs.build-info.outputs.test-types}}"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Upload airflow logs"
uses: actions/upload-artifact@v3
@@ -1252,7 +1247,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-sqlite-${{matrix.python-version}}
path: ./files/coverage*.xml
@@ -1264,15 +1259,15 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-quarantined:
timeout-minutes: 60
name: "Quarantined tests"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
continue-on-error: true
needs: [build-info, wait-for-ci-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}}
- POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
+ POSTGRES_VERSION: ${{needs.build-info.outputs.default-postgres-version}}
TEST_TYPES: "Quarantined"
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: Cleanup repo
@@ -1284,7 +1279,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- name: "Set issue id for main"
@@ -1309,7 +1304,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Tests: Quarantined"
run: ./scripts/ci/testing/ci_run_quarantined_tests.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
- name: "Upload Quarantine test results"
uses: actions/upload-artifact@v3
if: always()
@@ -1333,7 +1328,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
retention-days: 7
- name: "Upload artifact for coverage"
uses: actions/upload-artifact@v3
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
with:
name: coverage-quarantined-${{ matrix.backend }}
path: "./files/coverage*.xml"
@@ -1345,7 +1340,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
upload-coverage:
timeout-minutes: 15
name: "Upload coverage"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
continue-on-error: true
needs:
- build-info
@@ -1355,9 +1350,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- tests-mssql
- tests-quarantined
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
# Only upload coverage on merges to main
- if: needs.build-info.outputs.runCoverage == 'true'
+ if: needs.build-info.outputs.run-coverage == 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1380,13 +1375,13 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
wait-for-prod-images:
timeout-minutes: 120
name: "Wait for PROD images"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-ci-images, build-prod-images]
if: needs.build-info.outputs.image-build == 'true'
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1397,7 +1392,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1415,7 +1410,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
# For the images to be available and test them.
run: breeze pull-prod-image --verify-image --wait-for-image --run-in-parallel
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Fix ownership"
run: breeze fix-ownership
@@ -1424,11 +1419,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
test-docker-compose-quick-start:
timeout-minutes: 60
name: "Test docker-compose quick start"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
if: needs.build-info.outputs.image-build == 'true'
env:
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1440,7 +1435,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1459,25 +1454,25 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-kubernetes:
timeout-minutes: 240
name: Helm Chart; ${{matrix.executor}}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
strategy:
matrix:
executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor]
fail-fast: false
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
KUBERNETES_MODE: "image"
EXECUTOR: ${{matrix.executor}}
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
+ KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
+ HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
+ ${{needs.build-info.outputs.python-versions-list-as-string}}
CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
+ ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
if: >
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.needs-helm-tests == 'true' ) &&
@@ -1492,7 +1487,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1501,20 +1496,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --run-in-parallel --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/kubernetes-bin"
key: "kubernetes-binaries
--${{ needs.build-info.outputs.defaultKindVersion }}\
--${{ needs.build-info.outputs.defaultHelmVersion }}"
+-${{ needs.build-info.outputs.default-kind-version }}\
+-${{ needs.build-info.outputs.default-helm-version }}"
restore-keys: "kubernetes-binaries"
- name: "Kubernetes Tests"
run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
- name: "Upload KinD logs"
uses: actions/upload-artifact@v3
if: failure() || cancelled()
@@ -1529,21 +1524,21 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-helm-executor-upgrade:
timeout-minutes: 150
name: Helm Chart Executor Upgrade
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
BACKEND: postgres
RUN_TESTS: "true"
RUNTIME: "kubernetes"
KUBERNETES_MODE: "image"
EXECUTOR: "KubernetesExecutor"
- KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
- HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
+ KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
+ HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
+ ${{needs.build-info.outputs.python-versions-list-as-string}}
CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetesVersionsListAsString}}
+ ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
if: >
needs.build-info.outputs.run-kubernetes-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -1557,7 +1552,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1566,31 +1561,31 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-prod-image --run-in-parallel --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/.kubernetes_venv"
- key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}
+ key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\
+ -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ -${{needs.build-info.outputs.python-versions-list-as-string}}
-${{ hashFiles('setup.py','setup.cfg') }}"
- restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\
- -${{needs.build-info.outputs.kubernetesVersionsListAsString}}
- -${{needs.build-info.outputs.pythonVersionsListAsString}}"
+ restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\
+ -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ -${{needs.build-info.outputs.python-versions-list-as-string}}"
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
path: ".build/kubernetes-bin"
key: "kubernetes-binaries
- -${{ needs.build-info.outputs.defaultKindVersion }}\
- -${{ needs.build-info.outputs.defaultHelmVersion }}"
+ -${{ needs.build-info.outputs.default-kind-version }}\
+ -${{ needs.build-info.outputs.default-helm-version }}"
restore-keys: "kubernetes-binaries"
- name: "Kubernetes Helm Chart Executor Upgrade Tests"
run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh
env:
- PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}"
+ PR_LABELS: "${{ needs.build-info.outputs.pull-request-labels }}"
- name: "Upload KinD logs"
uses: actions/upload-artifact@v3
if: failure() || cancelled()
@@ -1607,7 +1602,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
contents: write
timeout-minutes: 40
name: "Constraints"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs:
- build-info
- wait-for-ci-images
@@ -1618,8 +1613,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- tests-mssql
- tests-postgres
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
- if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
+ if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
@@ -1631,7 +1626,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1640,7 +1635,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }}
run: breeze pull-image --run-in-parallel --tag-as-latest
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Generate constraints"
run: |
@@ -1649,24 +1644,24 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints-no-providers
breeze generate-constraints --run-in-parallel --airflow-constraints-mode constraints
env:
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.pythonVersionsListAsString }}
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
uses: actions/checkout@v3
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
with:
path: "repo"
ref: ${{ steps.constraints-branch.outputs.branch }}
persist-credentials: false
- - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}"
+ - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
- name: "Push changes"
uses: ./.github/actions/github-push-action
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.constraints-branch.outputs.branch }}
@@ -1684,19 +1679,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
packages: write
timeout-minutes: 120
name: "Push Image Cache"
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs:
- build-info
- constraints
- docs
- if: needs.build-info.outputs.mergeRun == 'true'
+ if: needs.build-info.outputs.merge-run == 'true'
strategy:
fail-fast: false
matrix:
- python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
+ python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
platform: ["linux/amd64", "linux/arm64"]
env:
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on) }}
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
steps:
- name: Cleanup repo
@@ -1708,7 +1703,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Setup python"
uses: actions/setup-python@v4
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: ./scripts/ci/install_breeze.sh
@@ -1720,7 +1715,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: breeze pull-image --tag-as-latest
env:
# Always use default Python version of CI image for preparing packages
- PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.default-python-version }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
@@ -1770,9 +1765,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
build-ci-arm-images:
timeout-minutes: 120
name: >
- ${{needs.build-info.outputs.buildJobDescription}} CI ARM images
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+ ${{needs.build-info.outputs.build-job-description}} CI ARM images
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs:
- build-info
- wait-for-ci-images
@@ -1786,46 +1781,46 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
DEBIAN_VERSION: ${{ needs.build-info.outputs.debian-version }}
- RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }}
- if: needs.build-info.outputs.upgradeToNewerDependencies != 'false'
+ RUNS_ON: ${{ fromJson(needs.build-info.outputs.runs-on)[0] }}
+ if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- uses: actions/checkout@v2
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
submodules: recursive
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Setup python"
uses: actions/setup-python@v2
with:
- python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ python-version: ${{ needs.build-info.outputs.default-python-version }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Free space"
run: breeze free-space
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: >
Build CI ARM images ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
+ ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
run: >
breeze build-image --run-in-parallel --parallelism 1
--builder airflow_cache --platform "linux/arm64"
env:
- UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
+ UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }}
- PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }}
- if: needs.build-info.outputs.inWorkflowBuild == 'true'
+ PYTHON_VERSIONS: ${{ needs.build-info.outputs.all-python-versions-list-as-string }}
+ if: needs.build-info.outputs.in-workflow-build == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
- name: "Fix ownership"
run: breeze fix-ownership
- if: always() && needs.build-info.outputs.inWorkflowBuild == 'true'
+ if: always() && needs.build-info.outputs.in-workflow-build == 'true'
diff --git a/dev/breeze/src/airflow_breeze/branch_defaults.py b/dev/breeze/src/airflow_breeze/branch_defaults.py
index f4903cd077..7afc40d7f9 100644
--- a/dev/breeze/src/airflow_breeze/branch_defaults.py
+++ b/dev/breeze/src/airflow_breeze/branch_defaults.py
@@ -38,3 +38,4 @@ Examples:
AIRFLOW_BRANCH = "main"
DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH = "constraints-main"
+DEBIAN_VERSION = "bullseye"
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 4e225dd278..0b5c93bc48 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -498,3 +498,7 @@ class SelectiveChecks:
@cached_property
def skip_pre_commits(self) -> str:
return "identity" if self._default_branch == "main" else "identity,check-airflow-2-2-compatibility"
+
+ @cached_property
+ def cache_directive(self) -> str:
+ return "disabled" if self._github_event == GithubEvents.SCHEDULE else "registry"