You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/10/18 15:08:37 UTC
[airflow] 02/13: Update CI documentation, renaming runs to "Canary" (#26151)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 56ba1eb715f8e870dc6d2e7a92f0940e3fe6ab8b
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Sep 9 17:56:50 2022 +0200
Update CI documentation, renaming runs to "Canary" (#26151)
For quite some time we did not have the name of the main builds
However more and more the "main" builds are used to provide early
warnigns for some problems:
* 3rd-party dependencies breaking our builds
* our own dependencies breaking the constraints
* building ARM images
* building breeze images quickly
* running complete matrix of tests
* finding flaky tests
So effectively, those main builds are really "Canary" builds - when
those builds are failing, they give us a chance to react quickly,
without affecting the regular PR builds.
This PR clarifies the meaning and reasoning for those builds
and introduces "Canary" name for them.
During related documentation review, it also turned out that a number
of old environment variables are not used any more (after the breeze
changing to Python) and this PR also removes them from documentation
and removes the variables from all the scripts (including removal of
some unused scripts)
The new documentation also mentions somethign that we've learned
recently - that in case you use Breeze in non-airflow workflows in
GitHub Actions, you need to override the variables through command
line parameters rather than through environment variables, because
GitHub actions treats GITHUB_* variables as immutable for security.
(cherry picked from commit eb03959e437e11891b8c3696b76f664a991a37a4)
---
.github/workflows/build-images.yml | 2 -
.github/workflows/ci.yml | 52 ++--
CI.rst | 308 ++++++++-------------
CI_DIAGRAMS.md | 8 +-
.../src/airflow_breeze/commands/ci_commands.py | 4 +-
dev/breeze/src/airflow_breeze/global_constants.py | 6 -
.../airflow_breeze/utils/docker_command_utils.py | 1 -
dev/breeze/tests/test_pr_info.py | 16 +-
scripts/ci/docker-compose/_docker.env | 1 -
scripts/ci/docker-compose/base.yml | 1 -
scripts/ci/docker-compose/devcontainer.env | 1 -
scripts/ci/libraries/_all_libs.sh | 2 -
scripts/ci/libraries/_initialization.sh | 14 +-
scripts/ci/libraries/_md5sum.sh | 154 -----------
14 files changed, 164 insertions(+), 406 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 5c3be9270f..f8433ab92e 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -26,8 +26,6 @@ permissions:
env:
MOUNT_SELECTED_LOCAL_SOURCES: "false"
ANSWER: "yes"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 51fd6ac9b0..ece19a4c8f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -31,8 +31,6 @@ permissions:
env:
MOUNT_SELECTED_LOCAL_SOURCES: "false"
ANSWER: "yes"
- CHECK_IMAGE_FOR_REBUILD: "true"
- SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
VERBOSE: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
@@ -168,7 +166,7 @@ jobs:
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
build-job-description: ${{ steps.source-run-info.outputs.build-job-description }}
runs-on: ${{ steps.source-run-info.outputs.runs-on }}
- merge-run: ${{ steps.source-run-info.outputs.merge-run }}
+ canary-run: ${{ steps.source-run-info.outputs.canary-run }}
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
steps:
- name: Cleanup repo
@@ -261,27 +259,27 @@ jobs:
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Setup python"
uses: actions/setup-python@v4
with:
python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Free space"
run: breeze ci free-space
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: matrix.platform == 'linux/arm64' && needs.build-info.outputs.merge-run == 'true'
+ if: matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
- name: "Push CI cache ${{ matrix.platform }}"
run: >
breeze ci-image build
@@ -290,23 +288,23 @@ jobs:
--run-in-parallel
--force-build
--platform ${{ matrix.platform }}
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Push CI latest image ${{ matrix.platform }}"
run: >
breeze ci-image build
--tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }}
# We only push "amd" image as it is really only needed for any kind of automated builds in CI
# and currently there is not an easy way to make multi-platform image from two separate builds
- if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.merge-run == 'true'
+ if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.merge-run == 'true'
+ if: always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true'
- name: "Clean docker cache for ${{ matrix.platform }}"
run: docker system prune --all --force
- if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.merge-run == 'true'
+ if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
- name: "Fix ownership"
run: breeze ci fix-ownership
- if: always() && needs.build-info.outputs.merge-run == 'true'
+ if: always() && needs.build-info.outputs.canary-run == 'true'
# Check that after earlier cache push, breeze command will build quickly
chcek-that-image-builds-quicklly:
@@ -323,30 +321,30 @@ jobs:
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v3
with:
persist-credentials: false
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Setup python"
uses: actions/setup-python@v4
with:
python-version: ${{ needs.build-info.outputs.default-python-version }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- run: ./scripts/ci/install_breeze.sh
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Free space"
run: breeze ci free-space
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Check that image builds quickly"
run: breeze shell --max-time 120
- if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.merge-run == 'true'
+ if: matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true'
- name: "Fix ownership"
run: breeze ci fix-ownership
- if: always() && needs.build-info.outputs.merge-run == 'true'
+ if: always() && needs.build-info.outputs.canary-run == 'true'
build-ci-images:
permissions:
@@ -1636,20 +1634,20 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- name: "Set constraints branch name"
id: constraints-branch
run: ./scripts/ci/constraints/ci_branch_constraints.sh
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: Checkout ${{ steps.constraints-branch.outputs.branch }}
uses: actions/checkout@v3
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
with:
path: "repo"
ref: ${{ steps.constraints-branch.outputs.branch }}
persist-credentials: false
- name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}"
run: ./scripts/ci/constraints/ci_commit_constraints.sh
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
- name: "Push changes"
uses: ./.github/actions/github-push-action
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.constraints-branch.outputs.branch }}
@@ -1672,7 +1670,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
- build-info
- constraints
- docs
- if: needs.build-info.outputs.merge-run == 'true'
+ if: needs.build-info.outputs.canary-run == 'true'
strategy:
fail-fast: false
matrix:
@@ -1776,7 +1774,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
if: >
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' &&
needs.build-info.outputs.in-workflow-build == 'true' &&
- needs.build-info.outputs.merge-run != 'true'
+ needs.build-info.outputs.canary-run != 'true'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
diff --git a/CI.rst b/CI.rst
index 1b86a43b41..ccabd58463 100644
--- a/CI.rst
+++ b/CI.rst
@@ -21,7 +21,7 @@ CI Environment
==============
Continuous Integration is important component of making Apache Airflow robust and stable. We are running
-a lot of tests for every pull request, for main and v2-*-test branches and regularly as CRON jobs.
+a lot of tests for every pull request, for main and v2-*-test branches and regularly as scheduled jobs.
Our execution environment for CI is `GitHub Actions <https://github.com/features/actions>`_. GitHub Actions
(GA) are very well integrated with GitHub code and Workflow and it has evolved fast in 2019/202 to become
@@ -84,6 +84,13 @@ We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/ab
``GITHUB_TOKEN`` is needed to push to the registry and we configured scopes of the tokens in our jobs
to be able to write to the registry.
+The latest cache is kept as ``:cache-amd64`` and ``:cache-arm64`` tagged cache (suitable for
+``--cache-from`` directive of buildx - it contains metadata and cache for all segments in the image,
+and cache is separately kept for different platform.
+
+The ``latest`` images of CI and PROD are ``amd64`` only images for CI, because there is no very easy way
+to push multiplatform images without merging the manifests and it is not really needed nor used for cache.
+
Locally replicating CI failures
-------------------------------
@@ -151,13 +158,6 @@ You can use those variables when you try to reproduce the build locally.
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| Force variables |
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
-| ``FORCE_BUILD_IMAGES`` | false | false | false | Forces building images. This is generally not |
-| | | | | very useful in CI as in CI environment image |
-| | | | | is built or pulled only once, so there is no |
-| | | | | need to set the variable to true. For local |
-| | | | | builds it forces rebuild, regardless if it |
-| | | | | is determined to be needed. |
-+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| ``ANSWER`` | | yes | yes | This variable determines if answer to questions |
| | | | | during the build process should be |
| | | | | automatically given. For local development, |
@@ -167,17 +167,6 @@ You can use those variables when you try to reproduce the build locally.
| | | | | the user has to answer but in the CI |
| | | | | environment, we force "yes" answer. |
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
-| ``SKIP_CHECK_REMOTE_IMAGE`` | false | true | true | Determines whether we check if remote image |
-| | | | | is "fresher" than the current image. |
-| | | | | When doing local breeze runs we try to |
-| | | | | determine if it will be faster to rebuild |
-| | | | | the image or whether the image should be |
-| | | | | pulled first from the cache because it has |
-| | | | | been rebuilt. This is slightly experimental |
-| | | | | feature and will be improved in the future |
-| | | | | as the current mechanism does not always |
-| | | | | work properly. |
-+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| Host variables |
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
| ``HOST_USER_ID`` | | | | User id of the host user. |
@@ -265,107 +254,38 @@ You can use those variables when you try to reproduce the build locally.
| | | | | \* true in case of direct pushes and |
| | | | | scheduled builds |
+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
-| ``CHECK_IMAGE_FOR_REBUILD`` | true | true | true\* | Determines whether attempt should be |
-| | | | | made to rebuild the CI image with latest |
-| | | | | sources. It is true by default for |
-| | | | | local builds, however it is set to |
-| | | | | true in case we know that the image |
-| | | | | we pulled or built already contains |
-| | | | | the right sources. In such case we |
-| | | | | should set it to false, especially |
-| | | | | in case our local sources are not the |
-| | | | | ones we intend to use (for example |
-| | | | | when ``--image-tag`` is used |
-| | | | | in Breeze. |
-| | | | | |
-| | | | | In CI jobs it is set to true |
-| | | | | in case of the ``Build Images`` |
-| | | | | workflow or when |
-| | | | | waiting for images is disabled |
-| | | | | in the "Tests" workflow. |
-| | | | | |
-| | | | | \* if waiting for images the variable is set |
-| | | | | to false automatically. |
-+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
-| ``SKIP_BUILDING_PROD_IMAGE`` | false | false | false\* | Determines whether we should skip building |
-| | | | | the PROD image with latest sources. |
-| | | | | It is set to false, but in deploy app for |
-| | | | | kubernetes step it is set to "true", because at |
-| | | | | this stage we know we have good image build or |
-| | | | | pulled. |
-| | | | | |
-| | | | | \* set to true in "Deploy App to Kubernetes" |
-| | | | | to false automatically. |
-+-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+
Running CI Jobs locally
=======================
-The scripts and configuration files for CI jobs are all in ``scripts/ci`` - so that in the
-``pull_request_target`` target workflow, we can copy those scripts from the ``main`` branch and use them
-regardless of the changes done in the PR. This way we are kept safe from PRs injecting code into the builds.
-
-* ``build_airflow`` - builds airflow packages
-* ``constraints`` - scripts to build and publish latest set of valid constraints
-* ``docs`` - scripts to build documentation
-* ``images`` - scripts to build and push CI and PROD images
-* ``kubernetes`` - scripts to setup kubernetes cluster, deploy airflow and run kubernetes tests with it
-* ``openapi`` - scripts to run openapi generation
-* ``pre_commit`` - scripts to run pre-commit checks
-* ``provider_packages`` - scripts to build and test provider packages
-* ``static_checks`` - scripts to run static checks manually
-* ``testing`` - scripts that run unit and integration tests
-* ``tools`` - scripts that can be used for various clean-up and preparation tasks
-
-Common libraries of functions for all the scripts can be found in ``libraries`` folder. The ``dockerfiles``,
-``mysql.d``, ``openldap``, ``spectral_rules`` folders contains DockerFiles and configuration of integrations
-needed to run tests.
-
-For detailed use of those scripts you can refer to ``.github/workflows/`` - those scripts are used
-by the CI workflows of ours. There are some variables that you can set to change the behaviour of the
-scripts.
-
-The default values are "sane" you can change them to interact with your own repositories or registries.
-Note that you need to set "CI" variable to true in order to get the same results as in CI.
-
-+------------------------------+----------------------+-----------------------------------------------------+
-| Variable | Default | Comment |
-+==============================+======================+=====================================================+
-| CI | ``false`` | If set to "true", we simulate behaviour of |
-| | | all scripts as if they are in CI environment |
-+------------------------------+----------------------+-----------------------------------------------------+
-| CI_TARGET_REPO | ``apache/airflow`` | Target repository for the CI job. Used to |
-| | | compare incoming changes from PR with the target. |
-+------------------------------+----------------------+-----------------------------------------------------+
-| CI_TARGET_BRANCH | ``main`` | Target branch where the PR should land. Used to |
-| | | compare incoming changes from PR with the target. |
-+------------------------------+----------------------+-----------------------------------------------------+
-| CI_BUILD_ID | ``0`` | Unique id of the build that is kept across re runs |
-| | | (for GitHub actions it is ``GITHUB_RUN_ID``) |
-+------------------------------+----------------------+-----------------------------------------------------+
-| CI_JOB_ID | ``0`` | Unique id of the job - used to produce unique |
-| | | artifact names. |
-+------------------------------+----------------------+-----------------------------------------------------+
-| CI_EVENT_TYPE | ``pull_request`` | Type of the event. It can be one of |
-| | | [``pull_request``, ``pull_request_target``, |
-| | | ``schedule``, ``push``] |
-+------------------------------+----------------------+-----------------------------------------------------+
-| CI_REF | ``refs/head/main`` | Branch in the source repository that is used to |
-| | | make the pull request. |
-+------------------------------+----------------------+-----------------------------------------------------+
+All our CI jobs are executed via ``breeze`` commands. You can replicate exactly what our CI is doing
+by running the sequence of corresponding ``breeze`` command. Make sure however that you look at both:
+* flags passed to ``breeze`` commands
+* environment variables used when ``breeze`` command is run - this is useful when we want
+ to set a common flag for all ``breeze`` commands in the same job or even the whole workflow. For
+ example ``VERBOSE`` variable is set to ``true`` for all our workflows so that more detailed information
+ about internal commands executed in CI is printed.
+
+In the output of the CI jobs, you will find both - the flags passed and environment variables set.
GitHub Registry Variables
=========================
-Our CI uses GitHub Registry to pull and push images to/from by default. You can use your own repo by changing
-``GITHUB_REPOSITORY`` and providing your own GitHub Username and Token.
+Our CI uses GitHub Registry to pull and push images to/from by default. Those variables are set automatically
+by GitHub Actions when you run Airflow workflows in your fork, so they should automatically use your
+own repository as GitHub Registry to build and keep the images as build image cache.
+
+The variables are automatically set in GitHub actions
+--------------------------------+---------------------------+----------------------------------------------+
| Variable | Default | Comment |
+================================+===========================+==============================================+
| GITHUB_REPOSITORY | ``apache/airflow`` | Prefix of the image. It indicates which. |
-| | | registry from GitHub to use |
+| | | registry from GitHub to use for image cache |
+| | | and to determine the name of the image. |
++--------------------------------+---------------------------+----------------------------------------------+
+| CONSTRAINTS_GITHUB_REPOSITORY | ``apache/airflow`` | Repository where constraints are stored |
+--------------------------------+---------------------------+----------------------------------------------+
| GITHUB_USERNAME | | Username to use to login to GitHub |
| | | |
@@ -373,37 +293,37 @@ Our CI uses GitHub Registry to pull and push images to/from by default. You can
| GITHUB_TOKEN | | Token to use to login to GitHub. |
| | | Only used when pushing images on CI. |
+--------------------------------+---------------------------+----------------------------------------------+
-| GITHUB_REGISTRY_PULL_IMAGE_TAG | ``latest`` | Pull this image tag. This is "latest" by |
-| | | default, can also be full-length commit SHA. |
-+--------------------------------+---------------------------+----------------------------------------------+
-| GITHUB_REGISTRY_PUSH_IMAGE_TAG | ``latest`` | Push this image tag. This is "latest" by |
-| | | default, can also be full-length commit SHA. |
-+--------------------------------+---------------------------+----------------------------------------------+
+
+The Variables beginning with ``GITHUB_`` cannot be overridden in GitHub Actions by the workflow.
+Those variables are set by GitHub Actions automatically and they are immutable. Therefore if
+you want to override them in your own CI workflow and use ``breeze``, you need to pass the
+values by corresponding ``breeze`` flags ``--github-repository``, ``--github-username``,
+``--github-token`` rather than by setting them as environment variables in your workflow.
+Unless you want to keep your own copy of constraints in orphaned ``constraints-*``
+branches, the ``CONSTRAINTS_GITHUB_REPOSITORY`` should remain ``apache/airflow``, regardless in which
+repository the CI job is run.
+
+One of the variables you might want to override in your own GitHub Actions workflow when using ``breeze`` is
+``--github-repository`` - you might want to force it to ``apache/airflow``, because then the cache from
+``apache/airflow`` repository will be used and your builds will be much faster.
+
+Example command to build your CI image efficiently in your own CI workflow:
+
+.. code-block:: bash
+
+ # GITHUB_REPOSITORY is set automatically in Github Actions so we need to override it with flag
+ #
+ breeze ci-image build --github-repository apache/airflow --python 3.10
+ docker tag ghcr.io/apache/airflow/main/ci/python3.10 your-image-name:tag
+
Authentication in GitHub Registry
=================================
We are using GitHub Container Registry as cache for our images. Authentication uses GITHUB_TOKEN mechanism.
Authentication is needed for pushing the images (WRITE) only in "push", "pull_request_target" workflows.
+When you are running the CI jobs in GitHub Actions, GITHUB_TOKEN is set automatically by the actions.
-CI Architecture
-===============
-
-The following components are part of the CI infrastructure
-
-* **Apache Airflow Code Repository** - our code repository at https://github.com/apache/airflow
-* **Apache Airflow Forks** - forks of the Apache Airflow Code Repository from which contributors make
- Pull Requests
-* **GitHub Actions** - (GA) UI + execution engine for our jobs
-* **GA CRON trigger** - GitHub Actions CRON triggering our jobs
-* **GA Workers** - virtual machines running our jobs at GitHub Actions (max 20 in parallel)
-* **GitHub Image Registry** - image registry used as build cache for CI jobs.
- It is at https://ghcr.io/apache/airflow
-* **DockerHub Image Registry** - image registry used to pull base Python images and (manually) publish
- the released Production Airflow images. It is at https://dockerhub.com/apache/airflow
-* **Official Images** (future) - these are official images that are prominently visible in DockerHub.
- We aim our images to become official images so that you will be able to pull them
- with ``docker pull apache-airflow``
CI run types
============
@@ -411,6 +331,12 @@ CI run types
The following CI Job run types are currently run for Apache Airflow (run by ci.yaml workflow)
and each of the run types has different purpose and context.
+Besides the regular "PR" runs we also have "Canary" runs that are able to detect most of the
+problems that might impact regular PRs early, without necessarily failing all PRs when those
+problems happen. This allows to provide much more stable environment for contributors, who
+contribute their PR, while giving a chance to maintainers to react early on problems that
+need reaction, when the "canary" builds fail.
+
Pull request run
----------------
@@ -426,27 +352,37 @@ CI, Production Images as well as base Python images that are also cached in the
Also for those builds we only execute Python tests if important files changed (so for example if it is
"no-code" change, no tests will be executed.
-Direct Push/Merge Run
----------------------
+Regular PR builds run in a "stable" environment:
+
+* fixed set of constraints (constraints that passed the tests) - except the PRs that change dependencies
+* limited matrix and set of tests (determined by selective checks based on what changed in the PR)
+* no ARM image builds are build in the regular PRs
+* lower probability of flaky tests for non-committer PRs (public runners and less parallelism)
-Those runs are results of direct pushes done by the committers or as result of merge of a Pull Request
+Canary run
+----------
+
+Those runs are results of direct pushes done by the committers - basically merging of a Pull Request
by the committers. Those runs execute in the context of the Apache Airflow Code Repository and have also
write permission for GitHub resources (container registry, code repository).
+
The main purpose for the run is to check if the code after merge still holds all the assertions - like
-whether it still builds, all tests are green.
+whether it still builds, all tests are green. This is a "Canary" build that helps us to detect early
+problems with dependencies, image building, full matrix of tests in case they passed through selective checks.
This is needed because some of the conflicting changes from multiple PRs might cause build and test failures
after merge even if they do not fail in isolation. Also those runs are already reviewed and confirmed by the
committers so they can be used to do some housekeeping:
-- pushing most recent image build in the PR to the GitHub Container Registry (for caching)
+
+- pushing most recent image build in the PR to the GitHub Container Registry (for caching) including recent
+ Dockerfile changes and setup.py/setup.cfg changes (Early Cache)
+- test that image in ``breeze`` command builds quickly
+- run full matrix of tests to detect any tests that will be mistakenly missed in ``selective checks``
- upgrading to latest constraints and pushing those constraints if all tests succeed
- refresh latest Python base images in case new patch-level is released
The housekeeping is important - Python base images are refreshed with varying frequency (once every few months
usually but sometimes several times per week) with the latest security and bug fixes.
-Those patch level images releases can occasionally break Airflow builds (specifically Docker image builds
-based on those images) therefore in PRs we only use latest "good" Python image that we store in the
-GitHub Container Registry and those push requests will refresh the latest images if they changed.
Scheduled runs
--------------
@@ -522,53 +458,55 @@ Tests Workflow
This workflow is a regular workflow that performs all checks of Airflow code.
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Job | Description | PR | Push | CRON |
-| | | | Merge | (1) |
-+===========================+==============================================+=======+=======+======+
-| Build info | Prints detailed information about the build | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Push early cache & images | Pushes early cache/images to GitHub Registry | - | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Test OpenAPI client gen | Tests if OpenAPIClient continues to generate | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| UI tests | React UI tests for new Airflow UI | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| WWW tests | React tests for current Airflow UI | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Test image building | Tests if PROD image build examples work | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| CI Images | Waits for and verify CI Images (3) | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| (Basic) Static checks | Performs static checks (full or basic) | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Build docs | Builds documentation | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Tests | Run all the Pytest tests for Python code | Yes(2)| Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Tests provider packages | Tests if provider packages work | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Upload coverage | Uploads test coverage from all the tests | - | Yes | - |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| PROD Images | Waits for and verify PROD Images (3) | Yes | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Tests Kubernetes | Run Kubernetes test | Yes(2)| Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Constraints | Upgrade constraints to latest ones (4) | - | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-| Push cache & images | Pushes cache/images to GitHub Registry (4) | - | Yes | Yes |
-+---------------------------+----------------------------------------------+-------+-------+------+
-
-
-Comments:
-
- (1) CRON jobs builds images from scratch - to test if everything works properly for clean builds
- (2) The tests are run when the Trigger Tests job determine that important files change (this allows
- for example "no-code" changes to build much faster)
- (3) The jobs wait for CI images to be available.
- (4) PROD and CI cache & images are pushed as "latest" to GitHub Container registry and constraints are
- upgrade only if all tests are successful. The images are rebuilt in this step using constraints pushed
- in the previous step.
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Job | Description | PR | Canary | Scheduled |
++=============================+==========================================================+=========+==========+===========+
+| Build info | Prints detailed information about the build | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Build CI/PROD images | Builds images in-workflow (not in the build images one) | - | Yes | Yes (1) |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Push early cache & images | Pushes early cache/images to GitHub Registry and test | - | Yes | - |
+| | speed of building breeze images from scratch | | | |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Test OpenAPI client gen | Tests if OpenAPIClient continues to generate | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| UI tests | React UI tests for new Airflow UI | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| WWW tests | React tests for current Airflow UI | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Test image building | Tests if PROD image build examples work | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| CI Images | Waits for and verify CI Images (3) | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| (Basic) Static checks | Performs static checks (full or basic) | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Build docs | Builds documentation | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Tests | Run all the Pytest tests for Python code | Yes(2) | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Tests provider packages | Tests if provider packages work | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Upload coverage | Uploads test coverage from all the tests | - | Yes | - |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| PROD Images | Waits for and verify PROD Images (3) | Yes | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Tests Kubernetes | Run Kubernetes test | Yes(2) | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Constraints | Upgrade constraints to latest ones (4) | - | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+| Push cache & images | Pushes cache/images to GitHub Registry (4) | - | Yes | Yes |
++-----------------------------+----------------------------------------------------------+---------+----------+-----------+
+
+``(1)`` Scheduled jobs builds images from scratch - to test if everything works properly for clean builds
+
+``(2)`` The tests are run when the Trigger Tests job determine that important files change (this allows
+for example "no-code" changes to build much faster)
+
+``(3)`` The jobs wait for CI images to be available.
+
+``(4)`` PROD and CI cache & images are pushed as "latest" to GitHub Container registry and constraints are
+upgraded only if all tests are successful. The images are rebuilt in this step using constraints pushed
+in the previous step.
CodeQL scan
-----------
diff --git a/CI_DIAGRAMS.md b/CI_DIAGRAMS.md
index 878990645f..e19228a024 100644
--- a/CI_DIAGRAMS.md
+++ b/CI_DIAGRAMS.md
@@ -178,11 +178,11 @@ sequenceDiagram
deactivate Tests
```
-## Direct Push/Merge flow
+## Merge "Canary" run
```mermaid
sequenceDiagram
- Note over Airflow Repo: pull request
+ Note over Airflow Repo: push/merge
Note over Tests: push<br>[Write Token]
activate Airflow Repo
Airflow Repo -->> Tests: Trigger 'push'
@@ -199,6 +199,7 @@ sequenceDiagram
and
Note over Tests: Build CI Images<br>Use original constraints
Tests ->> GitHub Registry: Push CI Image Early cache + latest
+ Note over Tests: Test 'breeze' image build quickly
end
Tests ->> GitHub Registry: Push CI Images<br>[COMMIT_SHA]
GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
@@ -257,7 +258,7 @@ sequenceDiagram
deactivate Tests
```
-## Scheduled build flow
+## Scheduled run
```mermaid
sequenceDiagram
@@ -279,6 +280,7 @@ sequenceDiagram
and
Note over Tests: Build CI Images<br>Use original constraints
Tests ->> GitHub Registry: Push CI Image Early cache + latest
+ Note over Tests: Test 'breeze' image build quickly
end
Tests ->> GitHub Registry: Push CI Images<br>[COMMIT_SHA]
GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
index bddb96b36e..c5cd05c8b4 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
@@ -299,7 +299,7 @@ class WorkflowInfo(NamedTuple):
print(get_ga_output(name="runs-on", value=self.get_runs_on()))
print(get_ga_output(name='in-workflow-build', value=self.in_workflow_build()))
print(get_ga_output(name="build-job-description", value=self.get_build_job_description()))
- print(get_ga_output(name="merge-run", value=self.is_merge_run()))
+ print(get_ga_output(name="canary-run", value=self.is_canary_run()))
print(get_ga_output(name="run-coverage", value=self.run_coverage()))
def get_runs_on(self) -> str:
@@ -321,7 +321,7 @@ class WorkflowInfo(NamedTuple):
return "Build"
return "Skip Build (look in pull_request_target)"
- def is_merge_run(self) -> str:
+ def is_canary_run(self) -> str:
if (
self.event_name == 'push'
and self.head_repo == "apache/airflow"
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
index 07676e6d2c..a5a7f9126a 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -29,13 +29,7 @@ from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
RUNS_ON_PUBLIC_RUNNER = "ubuntu-20.04"
RUNS_ON_SELF_HOSTED_RUNNER = "self-hosted"
-# Commented this out as we are using buildkit and this vars became irrelevant
-# FORCE_PULL_IMAGES = False
-# CHECK_IF_BASE_PYTHON_IMAGE_UPDATED = False
-FORCE_BUILD_IMAGES = False
ANSWER = ""
-SKIP_CHECK_REMOTE_IMAGE = False
-# PUSH_PYTHON_BASE_IMAGE = False
APACHE_AIRFLOW_GITHUB_REPOSITORY = "apache/airflow"
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index e3aa376c77..f276051008 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -579,7 +579,6 @@ def update_expected_environment_variables(env: dict[str, str]) -> None:
set_value_to_default_if_not_set(env, 'DEFAULT_BRANCH', AIRFLOW_BRANCH)
set_value_to_default_if_not_set(env, 'ENABLED_SYSTEMS', "")
set_value_to_default_if_not_set(env, 'ENABLE_TEST_COVERAGE', "false")
- set_value_to_default_if_not_set(env, 'GITHUB_REGISTRY_PULL_IMAGE_TAG', "latest")
set_value_to_default_if_not_set(env, 'HOST_GROUP_ID', get_host_group_id())
set_value_to_default_if_not_set(env, 'HOST_OS', get_host_os())
set_value_to_default_if_not_set(env, 'HOST_USER_ID', get_host_user_id())
diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py
index 031ae752cd..9cdc1b1bb7 100644
--- a/dev/breeze/tests/test_pr_info.py
+++ b/dev/breeze/tests/test_pr_info.py
@@ -40,7 +40,7 @@ def test_pr_info():
assert wi.event_name == 'pull_request'
assert wi.pr_number == 26004
assert wi.get_runs_on() == "ubuntu-20.04"
- assert wi.is_merge_run() == "false"
+ assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -54,7 +54,7 @@ def test_push_info():
assert wi.event_name == 'push'
assert wi.pr_number is None
assert wi.get_runs_on() == "ubuntu-20.04"
- assert wi.is_merge_run() == "true"
+ assert wi.is_canary_run() == "true"
assert wi.run_coverage() == "true"
@@ -68,7 +68,7 @@ def test_schedule():
assert wi.event_name == 'schedule'
assert wi.pr_number is None
assert wi.get_runs_on() == "ubuntu-20.04"
- assert wi.is_merge_run() == "false"
+ assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -82,7 +82,7 @@ def test_runs_on_self_hosted():
assert wi.event_name == 'pull_request'
assert wi.pr_number == 1234
assert wi.get_runs_on() == "self-hosted"
- assert wi.is_merge_run() == "false"
+ assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -96,7 +96,7 @@ def test_runs_on_forced_public_runner():
assert wi.event_name == 'pull_request'
assert wi.pr_number == 1234
assert wi.get_runs_on() == "ubuntu-20.04"
- assert wi.is_merge_run() == "false"
+ assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -110,7 +110,7 @@ def test_runs_on_simple_pr_other_repo():
assert wi.event_name == 'pull_request'
assert wi.pr_number == 1234
assert wi.get_runs_on() == "ubuntu-20.04"
- assert wi.is_merge_run() == "false"
+ assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -124,7 +124,7 @@ def test_runs_on_push_other_branch():
assert wi.event_name == 'push'
assert wi.pr_number is None
assert wi.get_runs_on() == "self-hosted"
- assert wi.is_merge_run() == "false"
+ assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -138,5 +138,5 @@ def test_runs_on_push_v_test_branch():
assert wi.event_name == 'push'
assert wi.pr_number is None
assert wi.get_runs_on() == "self-hosted"
- assert wi.is_merge_run() == "true"
+ assert wi.is_canary_run() == "true"
assert wi.run_coverage() == "false"
diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env
index 8b90912eaa..3dafb73f68 100644
--- a/scripts/ci/docker-compose/_docker.env
+++ b/scripts/ci/docker-compose/_docker.env
@@ -35,7 +35,6 @@ ENABLED_INTEGRATIONS
ENABLED_SYSTEMS
ENABLE_TEST_COVERAGE
GITHUB_ACTIONS
-GITHUB_REGISTRY_PULL_IMAGE_TAG
HOST_USER_ID
HOST_GROUP_ID
HOST_OS
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 7904bc74fb..b5d0f895b5 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -48,7 +48,6 @@ services:
- ENABLED_SYSTEMS=${ENABLED_SYSTEMS}
- ENABLE_TEST_COVERAGE=${ENABLE_TEST_COVERAGE}
- GITHUB_ACTIONS=${GITHUB_ACTIONS}
- - GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG}
- HOST_USER_ID=${HOST_USER_ID}
- HOST_GROUP_ID=${HOST_GROUP_ID}
- HOST_OS=${HOST_OS}
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index 3f40808d98..4f10d5beb6 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -35,7 +35,6 @@ ENABLED_INTEGRATIONS=
ENABLED_SYSTEMS=
ENABLE_TEST_COVERAGE="false"
GITHUB_ACTIONS="false"
-GITHUB_REGISTRY_PULL_IMAGE_TAG=""
HOST_USER_ID=
HOST_GROUP_ID=
HOST_OS="linux"
diff --git a/scripts/ci/libraries/_all_libs.sh b/scripts/ci/libraries/_all_libs.sh
index 87de95bfa0..da893d5dcb 100755
--- a/scripts/ci/libraries/_all_libs.sh
+++ b/scripts/ci/libraries/_all_libs.sh
@@ -36,8 +36,6 @@ readonly SCRIPTS_CI_DIR
. "${LIBRARIES_DIR}"/_sanity_checks.sh
# shellcheck source=scripts/ci/libraries/_local_mounts.sh
. "${LIBRARIES_DIR}"/_local_mounts.sh
-# shellcheck source=scripts/ci/libraries/_md5sum.sh
-. "${LIBRARIES_DIR}"/_md5sum.sh
# shellcheck source=scripts/ci/libraries/_start_end.sh
. "${LIBRARIES_DIR}"/_start_end.sh
# shellcheck source=scripts/ci/libraries/_testing.sh
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index d16f817290..82edac3491 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -274,16 +274,10 @@ function initialization::initialize_mount_variables() {
# Determine values of force settings
function initialization::initialize_force_variables() {
- # Determines whether to force build without checking if it is needed
- # Can be overridden by '--force-build-images' flag.
- export FORCE_BUILD_IMAGES=${FORCE_BUILD_IMAGES:="false"}
# Can be set to "yes/no/quit" in order to force specified answer to all questions asked to the user.
export ANSWER=${ANSWER:=""}
- # Can be set to true to skip if the image is newer in registry
- export SKIP_CHECK_REMOTE_IMAGE=${SKIP_CHECK_REMOTE_IMAGE:="false"}
-
# integrations are disabled by default
export ENABLED_INTEGRATIONS=${ENABLED_INTEGRATIONS:=""}
@@ -461,8 +455,6 @@ function initialization::initialize_git_variables() {
}
function initialization::initialize_github_variables() {
- export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
- export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
export GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
# Allows to override the repository which is used as source of constraints during the build
@@ -557,7 +549,7 @@ function initialization::get_docker_cache_image_names() {
# Example:
# ghcr.io/apache/airflow/main/ci/python3.8:latest
# ghcr.io/apache/airflow/main/ci/python3.8:<COMMIT_SHA>
- export AIRFLOW_CI_IMAGE_WITH_TAG="${image_name}/${BRANCH_NAME}/ci/python${PYTHON_MAJOR_MINOR_VERSION}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ export AIRFLOW_CI_IMAGE_WITH_TAG="${image_name}/${BRANCH_NAME}/ci/python${PYTHON_MAJOR_MINOR_VERSION}:latest"
# File that is touched when the CI image is built for the first time locally
export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
@@ -585,9 +577,7 @@ Mount variables:
Force variables:
- FORCE_BUILD_IMAGES: ${FORCE_BUILD_IMAGES}
ANSWER: ${ANSWER}
- SKIP_CHECK_REMOTE_IMAGE: ${SKIP_CHECK_REMOTE_IMAGE}
Host variables:
@@ -643,7 +633,6 @@ Detected GitHub environment:
GITHUB_REPOSITORY: '${GITHUB_REPOSITORY}'
GITHUB_USERNAME: '${GITHUB_USERNAME}'
GITHUB_REGISTRY_PULL_IMAGE_TAG: '${GITHUB_REGISTRY_PULL_IMAGE_TAG}'
- GITHUB_REGISTRY_PUSH_IMAGE_TAG: '${GITHUB_REGISTRY_PUSH_IMAGE_TAG}'
GITHUB_ACTIONS: '${GITHUB_ACTIONS=}'
Initialization variables:
@@ -772,7 +761,6 @@ function initialization::make_constants_read_only() {
readonly ADDITIONAL_RUNTIME_APT_ENV
readonly GITHUB_REGISTRY_PULL_IMAGE_TAG
- readonly GITHUB_REGISTRY_PUSH_IMAGE_TAG
readonly GITHUB_REPOSITORY
readonly GITHUB_TOKEN
diff --git a/scripts/ci/libraries/_md5sum.sh b/scripts/ci/libraries/_md5sum.sh
deleted file mode 100644
index 1838936fc9..0000000000
--- a/scripts/ci/libraries/_md5sum.sh
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-declare -a MODIFIED_FILES
-#
-# Verifies if stored md5sum of the file changed since the last time it was checked
-# The md5sum files are stored in .build directory - you can delete this directory
-# If you want to rebuild everything from the scratch
-#
-function md5sum::calculate_file_md5sum {
- local file="${1}"
- local md5sum
- local md5sum_cache_dir="${BUILD_CACHE_DIR}/${BRANCH_NAME}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}"
- mkdir -pv "${md5sum_cache_dir}"
- md5sum=$(md5sum "${file}")
- local md5sum_file
- md5sum_file="${md5sum_cache_dir}"/$(basename "$(dirname "${file}")")-$(basename "${file}").md5sum
- local md5sum_file_new
- md5sum_file_new=${CACHE_TMP_FILE_DIR}/$(basename "$(dirname "${file}")")-$(basename "${file}").md5sum.new
- echo "${md5sum}" > "${md5sum_file_new}"
- local ret_code=0
- if [[ ! -f "${md5sum_file}" ]]; then
- verbosity::print_info "Missing md5sum for ${file#"${AIRFLOW_SOURCES}"} (${md5sum_file#"${AIRFLOW_SOURCES}"})"
- ret_code=1
- else
- diff "${md5sum_file_new}" "${md5sum_file}" >/dev/null
- local res=$?
- if [[ "${res}" != "0" ]]; then
- verbosity::print_info "The md5sum changed for ${file}: was $(cat "${md5sum_file}") now it is $(cat "${md5sum_file_new}")"
- if [[ ${CI} == "true" ]]; then
- echo "${COLOR_RED}The file has changed: ${file}${COLOR_RESET}"
- echo "${COLOR_BLUE}==============================${COLOR_RESET}"
- cat "${file}"
- echo "${COLOR_BLUE}==============================${COLOR_RESET}"
- fi
- ret_code=1
- fi
- fi
- return ${ret_code}
-}
-
-#
-# Moves md5sum file from it's temporary location in CACHE_TMP_FILE_DIR to
-# BUILD_CACHE_DIR - thus updating stored MD5 sum for the file
-#
-function md5sum::move_file_md5sum {
- local file="${1}"
- local md5sum_file
- local md5sum_cache_dir="${BUILD_CACHE_DIR}/${BRANCH_NAME}/${PYTHON_MAJOR_MINOR_VERSION}/${THE_IMAGE_TYPE}"
- mkdir -pv "${md5sum_cache_dir}"
- md5sum_file="${md5sum_cache_dir}"/$(basename "$(dirname "${file}")")-$(basename "${file}").md5sum
- local md5sum_file_new
- md5sum_file_new=${CACHE_TMP_FILE_DIR}/$(basename "$(dirname "${file}")")-$(basename "${file}").md5sum.new
- if [[ -f "${md5sum_file_new}" ]]; then
- mv "${md5sum_file_new}" "${md5sum_file}"
- verbosity::print_info "Updated md5sum file ${md5sum_file} for ${file}: $(cat "${md5sum_file}")"
- fi
-}
-
-#
-# Stores md5sum files for all important files and
-# records that we built the images locally so that next time we use
-# it from the local docker cache rather than pull (unless forced)
-#
-function md5sum::update_all_md5() {
- verbosity::print_info
- verbosity::print_info "Updating md5sum files"
- verbosity::print_info
- for file in "${FILES_FOR_REBUILD_CHECK[@]}"
- do
- md5sum::move_file_md5sum "${AIRFLOW_SOURCES}/${file}"
- done
- mkdir -pv "${BUILD_CACHE_DIR}/${BRANCH_NAME}"
- touch "${BUILT_CI_IMAGE_FLAG_FILE}"
-}
-
-function md5sum::update_all_md5_with_group() {
- start_end::group_start "Update MD5 hashes for pulled images"
- md5sum::update_all_md5
- start_end::group_end
-}
-
-function md5sum::calculate_md5sum_for_all_files() {
- FILES_MODIFIED="false"
- set +e
- for file in "${FILES_FOR_REBUILD_CHECK[@]}"
- do
- if ! md5sum::calculate_file_md5sum "${AIRFLOW_SOURCES}/${file}"; then
- FILES_MODIFIED="true"
- MODIFIED_FILES+=( "${file}" )
- fi
- done
- set -e
-}
-
-#
-# Checks md5sum of all important files in order to optimise speed of running various operations
-# That mount sources of Airflow to container and require docker image built with latest dependencies.
-# the Docker image will only be marked for rebuilding only in case any of the important files change:
-# * setup.py
-# * setup.cfg
-# * Dockerfile.ci
-#
-# This is needed because we want to skip rebuilding of the image when only airflow sources change but
-# Trigger rebuild in case we need to change dependencies (setup.py, setup.cfg, change version of Airflow
-# or the Dockerfile.ci itself changes.
-#
-# Another reason to skip rebuilding Docker is thar currently it takes a bit longer time than simple Docker
-# We need to fix group permissions of files in Docker because different linux build services have
-# different default umask and Docker uses group permissions in checking for cache invalidation.
-#
-# As result of this check - most of the static checks will start pretty much immediately.
-#
-function md5sum::check_if_docker_build_is_needed() {
- verbosity::print_info
- verbosity::print_info "Checking if image build is needed for ${THE_IMAGE_TYPE} image."
- verbosity::print_info
- if [[ ${FORCE_BUILD_IMAGES:=""} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "${COLOR_YELLOW}Docker image build is forced for ${THE_IMAGE_TYPE} image${COLOR_RESET}"
- verbosity::print_info
- md5sum::calculate_md5sum_for_all_files
- needs_docker_build="true"
- else
- md5sum::calculate_md5sum_for_all_files
- if [[ ${FILES_MODIFIED} == "true" ]]; then
- needs_docker_build="true"
- fi
- if [[ ${needs_docker_build} == "true" ]]; then
- verbosity::print_info
- verbosity::print_info "${COLOR_YELLOW}The files were modified and likely the ${THE_IMAGE_TYPE} image needs rebuild: ${MODIFIED_FILES[*]}${COLOR_RESET}"
- verbosity::print_info
- else
- verbosity::print_info
- verbosity::print_info "${COLOR_GREEN}Docker image build is not needed for ${THE_IMAGE_TYPE} image!${COLOR_RESET}"
- verbosity::print_info
- fi
- fi
-}