You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/10/20 04:28:23 UTC
[airflow] branch master updated: Optimizes CI builds heavily with
selective checks (#11656)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new dd1c07b Optimizes CI builds heavily with selective checks (#11656)
dd1c07b is described below
commit dd1c07b20d9037527b2979b72013ecd6b0caf1cd
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Oct 20 06:27:20 2020 +0200
Optimizes CI builds heavily with selective checks (#11656)
* Images are not built if the change is not touching code or docs.
* In case we have no need for CI images we run stripped-down
pre-commit checks which skip the long checks and only run for
changed files
* If none of the CLI/Providers/Kubernetes/WWW files changed
the relevant tests are skipped, unless some of the core files
changed as well.
* The selective checks logic is explained and documented.
This is the second attempt at the problem with better
strategy to get the list of files from the incoming PR.
The strategy works now better in a number of cases:
* when PR comes from the same repo
* when PR comes from the pull_repo
* when PR contains more than one commit
* when PR is based on older master and GitHub creates
merge commit
---
.github/workflows/build-images-workflow-run.yml | 42 ++--
.github/workflows/ci.yml | 161 ++++++++++---
.github/workflows/codeql-analysis.yml | 2 +-
CI.rst | 154 +++++++++++-
.../{selective_tests.sh => selective_ci_checks.sh} | 262 ++++++++++++++-------
.../ci/static_checks/run_basic_static_checks.sh | 49 ++++
6 files changed, 528 insertions(+), 142 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
index 9ca2f8d..853feac 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -185,9 +185,12 @@ jobs:
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
- pythonVersions: ${{ steps.versions.outputs.python-versions }}
- allPythonVersions: ${{ steps.versions.outputs.all-python-versions }}
- defaultPythonVersion: ${{ steps.versions.outputs.default-python-version }}
+ pythonVersions: ${{ steps.selective-checks.python-versions }}
+ allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
+ defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
+ run-tests: ${{ steps.selective-checks.outputs.run-tests }}
+ run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
+ basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
if: >
needs.cancel-workflow-runs.outputs.buildImages == 'true' &&
(github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule')
@@ -204,19 +207,27 @@ jobs:
Source Sha: ${{ needs.cancel-workflow-runs.outputs.sourceHeadSha }}
Merge commit Sha: ${{ needs.cancel-workflow-runs.outputs.mergeCommitSha }}
Target commit Sha: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
+ run: printenv
+ - name: >
+ Fetch incoming commit ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }} with its parent
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
+ fetch-depth: 2
+ if: needs.cancel-workflow-runs.outputs.sourceEvent == 'pull_request'
+ - name: Selective checks
+ id: selective-checks
+ env:
+ EVENT_NAME: ${{ needs.cancel-workflow-runs.outputs.sourceEvent }}
+ INCOMING_COMMIT_SHA: ${{ needs.cancel-workflow-runs.outputs.targetCommitSha }}
run: |
- printenv
- - name: Set versions
- id: versions
- run: |
- . ./scripts/ci/libraries/_script_init.sh
-
- initialization::ga_output python-versions \
- "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
- initialization::ga_output default-python-version "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[0]}"
-
- initialization::ga_output all-python-versions \
- "$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
+ if [[ ${EVENT_NAME} == "pull_request" ]]; then
+ # Run selective checks
+ ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ else
+ # Run all checks
+ ./scripts/ci/selective_ci_checks.sh
+ fi
build-images:
timeout-minutes: 80
@@ -231,6 +242,7 @@ jobs:
image-type: [CI, PROD]
fail-fast: true
if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
needs.cancel-workflow-runs.outputs.buildImages == 'true' &&
(github.repository == 'apache/airflow' || github.event.workflow_run.event != 'schedule')
env:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e0f06c2..06eea5e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -70,27 +70,30 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
waitForImage: ${{ steps.wait-for-image.outputs.wait-for-image }}
- pythonVersions: ${{ steps.selective-tests.outputs.python-versions }}
- defaultPythonVersion: ${{ steps.selective-tests.outputs.default-python-version }}
- kubernetesVersions: ${{ steps.selective-tests.outputs.kubernetes-versions }}
- defaultKubernetesVersion: ${{ steps.selective-tests.outputs.default-kubernetes-version }}
- kubernetesModes: ${{ steps.selective-tests.outputs.kubernetes-modes }}
- defaultKubernetesMode: ${{ steps.selective-tests.outputs.default-kubernetes-mode }}
- postgresVersions: ${{ steps.selective-tests.outputs.postgres-versions }}
- defaultPostgresVersion: ${{ steps.selective-tests.outputs.default-postgres-version }}
- mysqlVersions: ${{ steps.selective-tests.outputs.mysql-versions }}
- defaultMySQLVersion: ${{ steps.selective-tests.outputs.default-mysql-version }}
- helmVersions: ${{ steps.selective-tests.outputs.helm-versions }}
- defaultHelmVersion: ${{ steps.selective-tests.outputs.default-helm-version }}
- kindVersions: ${{ steps.selective-tests.outputs.kind-versions }}
- defaultKindVersion: ${{ steps.selective-tests.outputs.default-kind-version }}
- testTypes: ${{ steps.selective-tests.outputs.test-types }}
- postgresExclude: ${{ steps.selective-tests.outputs.postgres-exclude }}
- mysqlExclude: ${{ steps.selective-tests.outputs.mysql-exclude }}
- sqliteExclude: ${{ steps.selective-tests.outputs.sqlite-exclude }}
- kubernetesExclude: ${{ steps.selective-tests.outputs.kubernetes-exclude }}
- run-tests: ${{ steps.selective-tests.outputs.run-tests }}
- run-kubernetes-tests: ${{ steps.selective-tests.outputs.run-kubernetes-tests }}
+ pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
+ defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
+ kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }}
+ defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
+ kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }}
+ defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }}
+ postgresVersions: ${{ steps.selective-checks.outputs.postgres-versions }}
+ defaultPostgresVersion: ${{ steps.selective-checks.outputs.default-postgres-version }}
+ mysqlVersions: ${{ steps.selective-checks.outputs.mysql-versions }}
+ defaultMySQLVersion: ${{ steps.selective-checks.outputs.default-mysql-version }}
+ helmVersions: ${{ steps.selective-checks.outputs.helm-versions }}
+ defaultHelmVersion: ${{ steps.selective-checks.outputs.default-helm-version }}
+ kindVersions: ${{ steps.selective-checks.outputs.kind-versions }}
+ defaultKindVersion: ${{ steps.selective-checks.outputs.default-kind-version }}
+ testTypes: ${{ steps.selective-checks.outputs.test-types }}
+ postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
+ mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
+ sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
+ kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude }}
+ run-tests: ${{ steps.selective-checks.outputs.run-tests }}
+ run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }}
+ basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }}
+ needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
+ needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -106,20 +109,39 @@ jobs:
- name: Set wait for image
id: wait-for-image
run: |
- if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
+ if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == 'true' ]]; then
echo "::set-output name=wait-for-image::true"
else
echo "::set-output name=wait-for-image::false"
fi
- - name: Selective tests
- id: selective-tests
- run: ./scripts/ci/selective_tests.sh
+ - name: Fetch incoming commit ${{ github.sha }} with its parent
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.sha }}
+ fetch-depth: 2
+ if: github.event_name == 'pull_request'
+ - name: Selective checks
+ id: selective-checks
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ INCOMING_COMMIT_SHA: ${{ github.sha }}
+ run: |
+ if [[ ${EVENT_NAME} == "pull_request" ]]; then
+ # Run selective checks
+ ./scripts/ci/selective_ci_checks.sh "${INCOMING_COMMIT_SHA}"
+ else
+ # Run all checks
+ ./scripts/ci/selective_ci_checks.sh
+ fi
helm-tests:
timeout-minutes: 5
name: "Checks: Helm tests"
runs-on: ubuntu-latest
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ needs: [build-info]
+ if: >
+ needs.build-info.outputs.needs-helm-tests == 'true' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -129,7 +151,9 @@ jobs:
test-openapi-client-generation:
name: "Test OpenAPI client generation"
runs-on: ubuntu-latest
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.needs-api-tests == 'true' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -141,7 +165,9 @@ jobs:
name: "Wait for CI images"
runs-on: ubuntu-latest
needs: [build-info]
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
env:
BACKEND: sqlite
steps:
@@ -176,9 +202,9 @@ jobs:
SKIP: "pylint"
MOUNT_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
- # We want to make sure we have latest sources as only in_container scripts are added
- # to the image but we want to static-check all of them
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -202,12 +228,56 @@ jobs:
env:
VERBOSE: false
+ # Those checks are run if no image needs to be built for checks. This is for simple changes that
+ # Do not touch any of the python code or any of the important files that might require building
+ # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
+ static-checks-basic-checks-only:
+ timeout-minutes: 30
+ name: "Static checks: basic checks only"
+ runs-on: ubuntu-latest
+ needs: [build-info]
+ env:
+ SKIP: "build,mypy,flake8,pylint,bats-in-container-tests"
+ MOUNT_LOCAL_SOURCES: "true"
+ PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'true' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v2
+ - name: "Setup python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
+ - name: Cache pre-commit env
+ uses: actions/cache@v2
+ env:
+ cache-name: cache-pre-commit-master-basic-checks-v1
+ with:
+ path: ~/.cache/pre-commit
+ key: ${{ env.cache-name }}-${{ github.job }}-${{ hashFiles('.pre-commit-config.yaml') }}
+ - name: >
+ Fetch incoming commit ${{ github.sha }} with its parent
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.sha }}
+ fetch-depth: 2
+ - name: "Static checks: basic checks only"
+ run: |
+ ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}"
+ env:
+ VERBOSE: false
+
+
static-checks-pylint:
timeout-minutes: 30
name: "Pylint"
runs-on: ubuntu-latest
needs: [build-info, ci-images]
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
env:
# We want to make sure we have latest sources as only in_container scripts are added
# to the image but we want to static-check all of them
@@ -241,7 +311,9 @@ jobs:
name: "Build docs"
runs-on: ubuntu-latest
needs: [ci-images]
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -263,7 +335,9 @@ jobs:
needs: [build-info, ci-images]
env:
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -282,7 +356,9 @@ jobs:
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
BACKPORT_PACKAGES: "true"
VERSION_SUFFIX_FOR_SVN: "rc1"
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -319,7 +395,9 @@ jobs:
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
VERSION_SUFFIX_FOR_PYPI: "a0"
VERSION_SUFFIX_FOR_SVN: "a0"
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -604,7 +682,9 @@ jobs:
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- if: github.repository == 'apache/airflow' || github.event_name != 'schedule'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ (github.repository == 'apache/airflow' || github.event_name != 'schedule')
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -764,6 +844,7 @@ jobs:
- docs
- docs-spell-check
if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
github.event_name != 'schedule'
strategy:
@@ -805,6 +886,7 @@ jobs:
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
github.event_name != 'pull' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -833,6 +915,7 @@ jobs:
runs-on: ubuntu-latest
needs: [build-info, constraints]
if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
(github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ) &&
github.event_name != 'pull' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -876,7 +959,9 @@ jobs:
- tests-kubernetes
- constraints-push
- prepare-provider-packages
- if: github.event_name == 'schedule' && github.repository == 'apache/airflow'
+ if: >
+ needs.build-info.outputs.basic-checks-only == 'false' &&
+ github.event_name == 'schedule' && github.repository == 'apache/airflow'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 01bac7e..570c118 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -33,7 +33,7 @@ jobs:
# If this run was triggered by a pull request event, then checkout
# the head of the pull request instead of the merge commit.
- run: git checkout HEAD^2
- if: ${{ github.event_name == 'pull_request' }}
+ if: github.event_name == 'pull_request'
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
diff --git a/CI.rst b/CI.rst
index c42c9d4..b936c7d 100644
--- a/CI.rst
+++ b/CI.rst
@@ -32,6 +32,69 @@ However part of the philosophy we have is that we are not tightly coupled with a
environments we use. Most of our CI jobs are written as bash scripts which are executed as steps in
the CI jobs. And we have a number of variables determine build behaviour.
+
+
+
+Github Actions runs
+-------------------
+
+Our builds on CI are highly optimized. They utilise some of the latest features provided by GitHub Actions
+environment that make it possible to reuse parts of the build process across different Jobs.
+
+Big part of our CI runs use Container Images. Airflow has a lot of dependencies and in order to make
+sure that we are running tests in a well configured and repeatable environment, most of the tests,
+documentation building, and some more sophisticated static checks are run inside a docker container
+environment. This environment consist of two types of images: CI images and PROD images. CI Images
+are used for most of the tests and checks where PROD images are used in the Kubernetes tests.
+
+In order to run the tests, we need to make sure tha the images are built using latest sources and that it
+is done quickly (full rebuild of such image from scratch might take ~15 minutes). Therefore optimisation
+techniques have been implemented that use efficiently cache from the GitHub Docker registry - in most cases
+this brings down the time needed to rebuild the image to ~4 minutes. In some cases (when dependencies change)
+it can be ~6-7 minutes and in case base image of Python releases new patch-level, it can be ~12 minutes.
+
+Currently in master version of Airflow we run tests in 3 different versions of Python (3.6, 3.7, 3.8)
+which means that we have to build 6 images (3 CI ones and 3 PROD ones). Yet we run around 12 jobs
+with each of the CI images. That is a lot of time to just build the environment to run. Therefore
+we are utilising ``workflow_run`` feature of GitHub Actions. This feature allows to run a separate,
+independent workflow, when the main workflow is run - this separate workflow is different than the main
+one, because by default it runs using ``master`` version of the sources but also - and most of all - that
+it has WRITE access to the repository. This is especially important in our case where Pull Requests to
+Airflow might come from any repository, and it would be a huge security issue if anyone from outside could
+utilise the WRITE access to Apache Airflow repository via an external Pull Request.
+
+Thanks to the WRITE access and fact that the 'workflow_run' by default uses the 'master' version of the
+sources, we can safely run some logic there will checkout the incoming Pull Request, build the container
+image from the sources from the incoming PR and push such image to an Github Docker Registry - so that
+this image can be built only once and used by all the jobs running tests. The image is tagged with unique
+``RUN_ID`` of the incoming Pull Request and the tests run in the Pull Request can simply pull such image
+rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving
+a lot of precious time for jobs.
+
+
+Local runs
+----------
+
+The main goal of the CI philosophy we have that no matter how complex the test and integration
+infrastructure, as a developer you should be able to reproduce and re-run any of the failed checks
+locally. One part of it are pre-commit checks, that allow you to run the same static checks in CI
+and locally, but another part is the CI environment which is replicated locally with Breeze.
+
+You can read more about Breeze in `BREEZE.rst <BREEZE.rst>`_ but in essence it is a script that allows
+you to re-create CI environment in your local development instance and interact with it. In its basic
+form, when you do development you can run all the same tests that will be run in CI - but locally,
+before you submit them as PR. Another use case where Breeze is useful is when tests fail on CI. You can
+take the ``RUN_ID`` of failed build pass it as ``--github-image-id`` parameter of Breeze and it will
+download the very same version of image that was used in CI and run it locally. This way, you can very
+easily reproduce any failed test that happens in CI - even if you do not check out the sources
+connected with the run.
+
+You can read more about it in `BREEZE.rst <BREEZE.rst>`_ and `TESTING.rst <TESTING.rst>`_
+
+
+Difference between local runs and Github Action workflows
+---------------------------------------------------------
+
Depending whether the scripts are run locally (most often via `Breeze <BREEZE.rst>`_) or whether they
are run in "CI Build" or "Build Image" workflows they can take different values.
@@ -363,14 +426,13 @@ that to your own repository by setting those environment variables:
CI Architecture
===============
-.. image:: images/ci/CI.png
- :align: center
- :alt: CI architecture of Apache Airflow
-
.. This image is an export from the 'draw.io' graph available in
https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-23+Migrate+out+of+Travis+CI
You can edit it there and re-export.
+.. image:: images/ci/CI.png
+ :align: center
+ :alt: CI architecture of Apache Airflow
The following components are part of the CI infrastructure
@@ -629,6 +691,90 @@ delete old artifacts that are > 7 days old. It only runs for the 'apache/airflow
We also have a script that can help to clean-up the old artifacts:
`remove_artifacts.sh <dev/remove_artifacts.sh>`_
+Selective CI Checks
+===================
+
+In order to optimise our CI builds, we've implemented optimisations to only run selected checks for some
+kind of changes. The logic implemented reflects the internal architecture of Airflow 2.0 packages
+and it helps to keep down both the usage of jobs in GitHub Actions as well as CI feedback time to
+contributors in case of simpler changes.
+
+We have the following test types (separated by packages in which they are):
+
+* Core - for the core Airflow functionality (core folder)
+* API - Tests for the Airflow API (api and api_connexion folders)
+* CLI - Tests for the Airflow CLI (cli folder)
+* WWW - Tests for the Airflow webserver (www and www_rbac in 1.10 folders)
+* Providers - Tests for all Providers of Airflow (providers folder)
+* Other - all other tests (all other folders that are not part of any of the above)
+
+We also have several special kinds of tests that are not separated by packages but they are marked with
+pytest markers. They can be found in any of those packages and they can be selected by the appropriate
+pylint custom command line options. See `TESTING.rst <TESTING.rst>`_ for details but those are:
+
+* Integration - tests that require external integration images running in docker-compose
+* Heisentests - tests that are vulnerable to some side effects and are better to be run on their own
+* Quarantined - tests that are flaky and need to be fixed
+* Postgres - tests that require Postgres database. They are only run when backend is Postgres
+* MySQL - tests that require MySQL database. They are only run when backend is MySQL
+
+Even if the types are separated, In case they share the same backend version/python version, they are
+run sequentially in the same job, on the same CI machine. Each of them in a separate ``docker run`` command
+and with additional docker cleaning between the steps to not fall into the trap of exceeding resource
+usage in one big test run, but also not to increase the number of jobs per each Pull Request.
+
+The logic implemented for the changes works as follows:
+
+1) In case of direct push (so when PR gets merged) or scheduled run, we always run all tests and checks.
+ This is in order to make sure that the merge did not miss anything important. The remainder of the logic
+ is executed only in case of Pull Requests.
+
+2) We retrieve which files have changed in the incoming Merge Commit (github.sha is a merge commit
+ automatically prepared by GitHub in case of Pull Request, so we can retrieve the list of changed
+ files from that commit directly).
+
+3) If any of the important, environment files changed (Dockerfile, ci scripts, setup.py, GitHub workflow
+ files), then we again run all tests and checks. Those are cases where the logic of the checks changed
+ or the environment for the checks changed so we want to make sure to check everything.
+
+4) If any of docs changed: we need to have CI image so we enable image building
+
+5) If any of chart files changed, we need to run helm tests so we enable helm unit tests
+
+6) If any of API files changed, we need to run API tests so we enable them
+
+7) If any of the relevant source files that trigger the tests have changed at all. Those are airflow
+ sources, chart, tests and kubernetes_tests. If any of those files changed, we enable tests and we
+ enable image building, because the CI images are needed to run tests.
+
+8) Then we determine which types of the tests should be run. We count all the changed files in the
+ relevant airflow sources (airflow, chart, tests, kubernetes_tests) first and then we count how many
+ files changed in different packages:
+
+ a) if any of the Airflow API files changed we enable ``API`` test type
+ b) if any of the Airflow CLI files changed we enable ``CLI`` test type
+ c) if any of the Provider files changed we enable ``Providers`` test type
+ d) if any of the WWW files changed we enable ``WWW`` test type
+ e) if any of the Kubernetes files changed we enable ``Kubernetes`` test type
+ f) Then we subtract count of all the ``specific`` above per-type changed files from the count of
+ all changed files. In case there are any files changed, then we assume that some unknown files
+ changed (likely from the core of airflow) and in this case we enable all test types above and the
+ Core test types - simply because we do not want to risk to miss anything.
+ g) In all cases where tests are enabled we also add Heisentests, Integration and - depending on
+ the backend used = Postgres or MySQL types of tests.
+
+9) Quarantined tests are always run when tests are run - we need to run them often to observe how
+ often they fail so that we can decide to move them out of quarantine. Details about the
+ Quarantined tests are described in `TESTING.rst <TESTING.rst>`_
+
+10) There is a special case of static checks. In case the above logic determines that the CI image
+ needs to be build, we run long and more comprehensive version of static checks - including Pylint,
+ MyPy, Flake8. And those tests are run on all files, no matter how many files changed.
+ In case the image is not built, we run only simpler set of changes - the longer static checks
+ that require CI image are skipped, and we only run the tests on the files that changed in the incoming
+ commit - unlike pylint/flake8/mypy, those static checks are per-file based and they should not miss any
+ important change.
+
Naming conventions for stored images
====================================
diff --git a/scripts/ci/selective_tests.sh b/scripts/ci/selective_ci_checks.sh
similarity index 62%
rename from scripts/ci/selective_tests.sh
rename to scripts/ci/selective_ci_checks.sh
index 66cf709..6ceb73a 100755
--- a/scripts/ci/selective_tests.sh
+++ b/scripts/ci/selective_ci_checks.sh
@@ -18,12 +18,24 @@
# shellcheck source=scripts/ci/libraries/_script_init.sh
. ./scripts/ci/libraries/_script_init.sh
+# Parameter:
+#
+# $1 - COMMIT SHA of the incoming commit. If this parameter is missing, this script does not check anything,
+# it simply sets all the version outputs that determine that all tests should be run.
+# This happens in case the even triggering the workflow is 'schedule' or 'push'.
+#
+# The logic of retrieving changes works by comparing the incoming commit with the target branch
+# The commit addresses.
+#
+#
declare -a pattern_array
function output_all_basic_variables() {
initialization::ga_output python-versions \
"$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
+ initialization::ga_output all-python-versions \
+ "$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")"
initialization::ga_output kubernetes-versions \
"$(initialization::parameters_to_json "${CURRENT_KUBERNETES_VERSIONS[@]}")"
@@ -58,28 +70,24 @@ function output_all_basic_variables() {
initialization::ga_output kubernetes-exclude '[]'
}
-function set_outputs_run_all_tests() {
- initialization::ga_output run-tests "true"
- initialization::ga_output run-kubernetes-tests "true"
- initialization::ga_output test-types "Core Other API CLI Providers WWW Integration Heisentests"
-}
-
-function set_output_skip_all_tests() {
- initialization::ga_output run-tests "false"
- initialization::ga_output run-kubernetes-tests "false"
- initialization::ga_output test-types ""
-}
+function get_changed_files() {
+ INCOMING_COMMIT_SHA="${1}"
+ readonly INCOMING_COMMIT_SHA
-function initialize_git_repo() {
- git remote add target "https://github.com/${CI_TARGET_REPO}"
- git fetch target "${CI_TARGET_BRANCH}:${CI_TARGET_BRANCH}" --depth=1
echo
- echo "My commit SHA: ${COMMIT_SHA}"
+ echo "Incoming commit SHA: ${INCOMING_COMMIT_SHA}"
echo
+ echo "Changed files from ${INCOMING_COMMIT_SHA} vs it's first parent"
echo
- echo "Retrieved changed files from ${COMMIT_SHA} comparing to ${CI_TARGET_BRANCH} in ${CI_TARGET_REPO}"
- echo
- CHANGED_FILES=$(git diff-tree --no-commit-id --name-only -r "${COMMIT_SHA}" "${CI_TARGET_BRANCH}" || true)
+ CHANGED_FILES=$(git diff-tree --no-commit-id --name-only \
+ -r "${INCOMING_COMMIT_SHA}^" "${INCOMING_COMMIT_SHA}" || true)
+ if [[ ${CHANGED_FILES} == "" ]]; then
+ >&2 echo
+ >&2 echo Warning! Could not find any changed files
+ >&2 echo Assuming that we should run all tests in this case
+ >&2 echo
+ set_outputs_run_everything_and_exit
+ fi
echo
echo "Changed files:"
echo
@@ -88,6 +96,59 @@ function initialize_git_repo() {
readonly CHANGED_FILES
}
+
+function run_tests() {
+ initialization::ga_output run-tests "${@}"
+}
+
+function run_kubernetes_tests() {
+ initialization::ga_output run-kubernetes-tests "${@}"
+}
+
+function needs_helm_tests() {
+ initialization::ga_output needs-helm-tests "${@}"
+}
+
+function needs_api_tests() {
+ initialization::ga_output needs-api-tests "${@}"
+}
+
+function set_test_types() {
+ initialization::ga_output test-types "${@}"
+}
+
+function set_basic_checks_only() {
+ initialization::ga_output basic-checks-only "${@}"
+}
+
+ALL_TESTS="Core Other API CLI Providers WWW Integration Heisentests"
+readonly ALL_TESTS
+
+function set_outputs_run_everything_and_exit() {
+ needs_api_tests "true"
+ needs_helm_tests "true"
+ run_tests "true"
+ run_kubernetes_tests "true"
+ set_test_types "${ALL_TESTS}"
+ set_basic_checks_only "false"
+ exit
+}
+
+function set_outputs_run_all_tests() {
+ run_tests "true"
+ run_kubernetes_tests "true"
+ set_test_types "${ALL_TESTS}"
+ set_basic_checks_only "false"
+}
+
+function set_output_skip_all_tests_and_exit() {
+ run_tests "false"
+ run_kubernetes_tests "false"
+ set_test_types ""
+ set_basic_checks_only "true"
+ exit
+}
+
# Converts array of patterns into single | pattern string
# pattern_array - array storing regexp patterns
# Outputs - pattern string
@@ -121,76 +182,91 @@ function show_changed_files() {
# Output:
# Count of changed files matching the patterns
function count_changed_files() {
- count_changed_files=$(echo "${CHANGED_FILES}" | grep -c -E "$(get_regexp_from_patterns)" || true)
- echo "${count_changed_files}"
+ echo "${CHANGED_FILES}" | grep -c -E "$(get_regexp_from_patterns)" || true
}
-function run_all_tests_when_push_or_schedule() {
- if [[ ${GITHUB_EVENT_NAME} == "push" || ${GITHUB_EVENT_NAME} == "schedule" ]]; then
- echo
- echo "Always run all tests in case of push/schedule events"
- echo
- set_outputs_run_all_tests
- exit
+function check_if_api_tests_should_be_run() {
+ local pattern_array=(
+ "^airflow/api"
+ )
+ show_changed_files
+
+ if [[ $(count_changed_files) == "0" ]]; then
+ needs_api_tests "false"
+ else
+ needs_api_tests "true"
+ fi
+}
+
+function check_if_helm_tests_should_be_run() {
+ local pattern_array=(
+ "^chart"
+ )
+ show_changed_files
+
+ if [[ $(count_changed_files) == "0" ]]; then
+ needs_helm_tests "false"
+ else
+ needs_helm_tests "true"
+ fi
+}
+
+function check_if_docs_should_be_generated() {
+ local pattern_array=(
+ "^docs$"
+ "^airflow/.*\.py$"
+ "^CHANGELOG\.txt"
+ )
+ show_changed_files
+
+ if [[ $(count_changed_files) == "0" ]]; then
+ echo "None of the docs changed"
+ else
+ image_build_needed="true"
fi
}
-function check_if_tests_should_be_run_at_all() {
- TEST_TRIGGERING_PATTERNS=(
+AIRFLOW_SOURCES_TRIGGERING_TESTS=(
"^airflow"
- "^.github/workflows/"
- "^Dockerfile"
- "^scripts"
"^chart"
- "^setup.py"
"^tests"
"^kubernetes_tests"
- )
- readonly TEST_TRIGGERING_PATTERNS
+)
+readonly AIRFLOW_SOURCES_TRIGGERING_TESTS
- pattern_array=("${TEST_TRIGGERING_PATTERNS[@]}")
- show_changed_files "$(get_regexp_from_patterns)"
+function check_if_tests_are_needed_at_all() {
+ local pattern_array=("${AIRFLOW_SOURCES_TRIGGERING_TESTS[@]}")
+ show_changed_files
if [[ $(count_changed_files) == "0" ]]; then
echo "None of the important files changed, Skipping tests"
- set_output_skip_all_tests
- exit
+ set_output_skip_all_tests_and_exit
else
- initialization::ga_output run-tests "true"
+ image_build_needed="true"
+ tests_needed="true"
fi
}
-function check_if_environment_files_changed() {
- ENVIRONMENT_TRIGGERING_PATTERNS=(
+function run_all_tests_if_environment_files_changed() {
+ local pattern_array=(
"^.github/workflows/"
"^Dockerfile"
"^scripts"
"^setup.py"
)
- readonly ENVIRONMENT_TRIGGERING_PATTERNS
-
- pattern_array=("${ENVIRONMENT_TRIGGERING_PATTERNS[@]}")
- show_changed_files "$(get_regexp_from_patterns)"
+ show_changed_files
if [[ $(count_changed_files) != "0" ]]; then
- echo "Important environment files changed. Running all tests"
- set_outputs_run_all_tests
- exit
+ echo "Important environment files changed. Running everything"
+ set_outputs_run_everything_and_exit
fi
}
function get_count_all_files() {
echo
- echo "Count All files"
+ echo "Count All airflow source files"
echo
- ALL_TRIGGERING_PATTERNS=(
- "^airflow"
- "^chart"
- "^tests"
- "^kubernetes_tests"
- )
- readonly ALL_TRIGGERING_PATTERNS
- pattern_array=("${ALL_TRIGGERING_PATTERNS[@]}")
+ local pattern_array=("${AIRFLOW_SOURCES_TRIGGERING_TESTS[@]}")
show_changed_files
COUNT_ALL_CHANGED_FILES=$(count_changed_files)
echo "Files count: ${COUNT_ALL_CHANGED_FILES}"
@@ -201,14 +277,12 @@ function get_count_api_files() {
echo
echo "Count API files"
echo
- API_TRIGGERING_PATTERNS=(
+ local pattern_array=(
"^airflow/api"
"^airflow/api_connexion"
"^tests/api"
"^tests/api_connexion"
)
- readonly API_TRIGGERING_PATTERNS
- pattern_array=("${API_TRIGGERING_PATTERNS[@]}")
show_changed_files
COUNT_API_CHANGED_FILES=$(count_changed_files)
echo "Files count: ${COUNT_API_CHANGED_FILES}"
@@ -219,12 +293,10 @@ function get_count_cli_files() {
echo
echo "Count CLI files"
echo
- CLI_TRIGGERING_PATTERNS=(
+ local pattern_array=(
"^airflow/cli"
"^tests/cli"
)
- readonly CLI_TRIGGERING_PATTERNS
- pattern_array=("${CLI_TRIGGERING_PATTERNS[@]}")
show_changed_files
COUNT_CLI_CHANGED_FILES=$(count_changed_files)
echo "Files count: ${COUNT_CLI_CHANGED_FILES}"
@@ -235,12 +307,10 @@ function get_count_providers_files() {
echo
echo "Count Providers files"
echo
- PROVIDERS_TRIGGERING_PATTERNS=(
+ local pattern_array=(
"^airflow/providers"
"^tests/providers"
)
- readonly PROVIDERS_TRIGGERING_PATTERNS
- pattern_array=("${PROVIDERS_TRIGGERING_PATTERNS[@]}")
show_changed_files
COUNT_PROVIDERS_CHANGED_FILES=$(count_changed_files)
echo "Files count: ${COUNT_PROVIDERS_CHANGED_FILES}"
@@ -251,12 +321,10 @@ function get_count_www_files() {
echo
echo "Count WWW files"
echo
- WWW_TRIGGERING_PATTERNS=(
+ local pattern_array=(
"^airflow/www"
"^tests/www"
)
- readonly WWW_TRIGGERING_PATTERNS
- pattern_array=("${WWW_TRIGGERING_PATTERNS[@]}")
show_changed_files
COUNT_WWW_CHANGED_FILES=$(count_changed_files)
echo "Files count: ${COUNT_WWW_CHANGED_FILES}"
@@ -267,13 +335,11 @@ function get_count_kubernetes_files() {
echo
echo "Count Kubernetes files"
echo
- KUBERNETES_TRIGGERING_PATTERNS=(
+ local pattern_array=(
"^airflow/kubernetes"
"^chart"
"^tests/kubernetes_tests"
)
- readonly KUBERNETES_TRIGGERING_PATTERNS
- pattern_array=("${KUBERNETES_TRIGGERING_PATTERNS[@]}")
show_changed_files
COUNT_KUBERNETES_CHANGED_FILES=$(count_changed_files)
echo "Files count: ${COUNT_KUBERNETES_CHANGED_FILES}"
@@ -284,9 +350,7 @@ function calculate_test_types_to_run() {
echo
echo "Count Core/Other files"
echo
- COUNT_CORE_OTHER_CHANGED_FILES=$((COUNT_ALL_CHANGED_FILES - COUNT_WWW_CHANGED_FILES - \
- COUNT_PROVIDERS_CHANGED_FILES - COUNT_CLI_CHANGED_FILES - COUNT_API_CHANGED_FILES - \
- COUNT_KUBERNETES_CHANGED_FILES))
+ COUNT_CORE_OTHER_CHANGED_FILES=$((COUNT_ALL_CHANGED_FILES - COUNT_WWW_CHANGED_FILES - COUNT_PROVIDERS_CHANGED_FILES - COUNT_CLI_CHANGED_FILES - COUNT_API_CHANGED_FILES - COUNT_KUBERNETES_CHANGED_FILES))
readonly COUNT_CORE_OTHER_CHANGED_FILES
echo
@@ -301,11 +365,9 @@ function calculate_test_types_to_run() {
set_outputs_run_all_tests
else
if [[ ${COUNT_KUBERNETES_CHANGED_FILES} != "0" ]]; then
- initialization::ga_output run-kubernetes-tests "true"
- else
- initialization::ga_output run-kubernetes-tests "false"
+ kubernetes_tests_needed="true"
fi
- initialization::ga_output run-tests "true"
+ tests_needed="true"
SELECTED_TESTS=""
if [[ ${COUNT_API_CHANGED_FILES} != "0" ]]; then
echo
@@ -336,10 +398,24 @@ function calculate_test_types_to_run() {
}
output_all_basic_variables
-run_all_tests_when_push_or_schedule
-initialize_git_repo
-check_if_tests_should_be_run_at_all
-check_if_environment_files_changed
+
+if (($# < 1)); then
+ echo
+ echo "No Commit SHA - running all tests!"
+ echo
+ set_outputs_run_everything_and_exit
+fi
+
+image_build_needed="false"
+tests_needed="false"
+kubernetes_tests_needed="false"
+
+get_changed_files "${1}"
+run_all_tests_if_environment_files_changed
+check_if_docs_should_be_generated
+check_if_helm_tests_should_be_run
+check_if_api_tests_should_be_run
+check_if_tests_are_needed_at_all
get_count_all_files
get_count_api_files
get_count_cli_files
@@ -347,3 +423,21 @@ get_count_providers_files
get_count_www_files
get_count_kubernetes_files
calculate_test_types_to_run
+
+if [[ ${image_build_needed} == "true" ]]; then
+ set_basic_checks_only "false"
+else
+ set_basic_checks_only "true"
+fi
+
+if [[ ${tests_needed} == "true" ]]; then
+ run_tests
+else
+ skip_running_tests
+fi
+
+if [[ ${kubernetes_tests_needed} == "true" ]]; then
+ run_kubernetes_tests
+else
+ skip_running_kubernetes_tests
+fi
diff --git a/scripts/ci/static_checks/run_basic_static_checks.sh b/scripts/ci/static_checks/run_basic_static_checks.sh
new file mode 100755
index 0000000..e017bec
--- /dev/null
+++ b/scripts/ci/static_checks/run_basic_static_checks.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# shellcheck source=scripts/ci/libraries/_script_init.sh
+. "$(dirname "${BASH_SOURCE[0]}")/../libraries/_script_init.sh"
+
+if [[ -f "${BUILD_CACHE_DIR}/.skip_tests" ]]; then
+ echo
+ echo "Skip tests"
+ echo
+ exit
+fi
+
+if (($# < 1)); then
+ echo
+ echo "Missing SHAs of incoming commit!"
+ echo
+ exit 1
+fi
+
+COMMIT_SHA="${1}"
+shift
+
+python -m pip install pre-commit \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
+
+if [[ $# == "0" ]]; then
+ pre-commit run --all-files --show-diff-on-failure --color always \
+ --from-ref "${COMMIT_SHA}^" --to-ref "${COMMIT_SHA}"
+else
+ for pre_commit_check in "${@}"; do
+ pre-commit run "${pre_commit_check}" --all-files --show-diff-on-failure --color always \
+ --from-ref "${COMMIT_SHA}^" --to-ref "${COMMIT_SHA}"
+ done
+fi