You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/02/23 11:44:47 UTC
[airflow] branch master updated: Easy switching between GitHub
Container Registries (#14120)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 25fa309 Easy switching between GitHub Container Registries (#14120)
25fa309 is described below
commit 25fa3092323e0cd13b11bb8e809b88ff3c043e4e
Author: Jarek Potiuk <po...@apache.org>
AuthorDate: Tue Feb 23 12:44:06 2021 +0100
Easy switching between GitHub Container Registries (#14120)
This change enables easy switching between GitHub Package Registry
and GitHub Container Registry by simply adding GITHUB_REGISTRY
secret to be either `docker.package.github.com` or `ghcr.io`.
This makes it easy to switch only by the Apache Airflow repository
run builds, as it requires preparation of images (to make them
public and to add permissions to manage them) after they got
created for the first time. GitHub Package Registry works
out-of-the-box but it is less stable and considered a legacy,
also it does not allow image retention.
Documentation has been updated to reflect the reasoning of choosing
this solution as well as describing maintenance processes around
images (including adding new Python version)
---
.github/workflows/build-images-workflow-run.yml | 3 +-
.github/workflows/ci.yml | 47 +++++++--
.github/workflows/scheduled_quarantined.yml | 5 +-
CI.rst | 124 +++++++++++++++++++++--
IMAGES.rst | 61 ++++++++++-
scripts/ci/images/ci_wait_for_all_prod_images.sh | 1 +
scripts/ci/libraries/_initialization.sh | 2 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 103 ++++++++++---------
8 files changed, 270 insertions(+), 76 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
index b9873e7..e5f8b41 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -30,8 +30,6 @@ env:
DB_RESET: "true"
VERBOSE: "true"
USE_GITHUB_REGISTRY: "true"
- # Might be either 'ghcr.io' or 'docker.pkg.github.com'
- GITHUB_REGISTRY: "docker.pkg.github.com"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
@@ -47,6 +45,7 @@ env:
GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
BUILD_IMAGES: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
INSTALL_PROVIDERS_FROM_SOURCES: "true"
+ GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }}
jobs:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6f4a472..e958aa4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -67,7 +67,6 @@ env:
#
# You can also switch back to building images locally and disabling the "Build Images" workflow
# by defining AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE secret with value set to "false"
-
GITHUB_REGISTRY_WAIT_FOR_IMAGE: ${{ secrets.AIRFLOW_GITHUB_REGISTRY_WAIT_FOR_IMAGE != 'false' }}
jobs:
@@ -203,6 +202,9 @@ jobs:
env:
BACKEND: sqlite
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
+ WAIT_FOR_IMAGE: ${{ needs.build-info.outputs.waitForImage }}
+ outputs:
+ githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -222,16 +224,20 @@ jobs:
- name: >
Wait for CI images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
+ id: wait-for-images
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
# We wait for the images to be available either from the build-ci-image step or from
- # "build-images-workflow-run.yml' run as pull_request_target (it has the write
- # permissions in case pull_request from fork is run.
+ # "build-images-workflow-run.yml' run as pull_request_target.
# We are utilising single job to wait for all images because this job merely waits
- # For the images to be available. The test jobs wait for it to complete!
+ # for the images to be available.
+ # The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
+ # The job will set the output "githubRegistry" - result of auto-detect which registry has
+ # been used by checking where the image can be downloaded from.
+ #
run: ./scripts/ci/images/ci_wait_for_all_ci_images.sh
- if: needs.build-info.outputs.waitForImage == 'true'
+
verify-ci-images:
timeout-minutes: 20
@@ -243,6 +249,7 @@ jobs:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
env:
BACKEND: sqlite
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -270,6 +277,7 @@ jobs:
SKIP: "pylint,identity"
MOUNT_SELECTED_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.basic-checks-only == 'false'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -360,6 +368,7 @@ jobs:
# to the image but we want to static-check all of them
MOUNT_SELECTED_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -397,6 +406,8 @@ jobs:
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, ci-images]
if: needs.build-info.outputs.docs-build == 'true'
+ env:
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -447,6 +458,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev"
VERSION_SUFFIX_FOR_SVN: "dev"
PACKAGE_FORMAT: ${{ matrix.package-format }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -497,6 +509,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev"
VERSION_SUFFIX_FOR_SVN: "dev"
PACKAGE_FORMAT: ${{ matrix.package-format }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
strategy:
matrix:
package-format: ['wheel', 'sdist']
@@ -543,6 +556,7 @@ jobs:
VERSION_SUFFIX_FOR_PYPI: "dev"
VERSION_SUFFIX_FOR_SVN: "dev"
PACKAGE_FORMAT: ${{ matrix.package-format }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
strategy:
matrix:
package-format: ['wheel', 'sdist']
@@ -579,6 +593,7 @@ jobs:
TEST_TYPES: "Helm"
BACKEND: "sqlite"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: >
needs.build-info.outputs.needs-helm-tests == 'true' &&
(github.repository == 'apache/airflow' || github.event_name != 'schedule')
@@ -640,6 +655,7 @@ jobs:
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -698,6 +714,7 @@ jobs:
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -753,6 +770,7 @@ jobs:
RUN_TESTS: true
TEST_TYPES: "${{needs.build-info.outputs.testTypes}}"
TEST_TYPE: ""
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -813,6 +831,7 @@ jobs:
TEST_TYPE: ""
NUM_RUNS: 10
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -904,11 +923,13 @@ jobs:
name: "Wait for PROD images"
runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
needs: [build-info, ci-images]
+ if: needs.build-info.outputs.image-build == 'true'
env:
BACKEND: sqlite
PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- if: needs.build-info.outputs.image-build == 'true'
+ outputs:
+ githubRegistry: ${{ steps.wait-for-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -928,11 +949,18 @@ jobs:
- name: >
Wait for PROD images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
+ # We wait for the images to be available either from the build-ci-image step or from
+ # "build-images-workflow-run.yml' run as pull_request_target.
+ # We are utilising single job to wait for all images because this job merely waits
+ # For the images to be available. The test jobs wait for it to complete!
+ # The job will set the output "githubRegistry" - result of auto-detect which registry has
+ # been used by checking where the image can be downloaded from.
+ #
+ id: wait-for-images
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
run: ./scripts/ci/images/ci_wait_for_all_prod_images.sh
- if: needs.build-info.outputs.waitForImage == 'true'
verify-prod-images:
timeout-minutes: 20
@@ -944,6 +972,7 @@ jobs:
python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }}
env:
BACKEND: sqlite
+ GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -982,6 +1011,7 @@ jobs:
KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}"
KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}"
HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}"
+ GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
if: needs.build-info.outputs.run-kubernetes-tests == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -1063,6 +1093,7 @@ jobs:
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
+ GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -1111,6 +1142,7 @@ jobs:
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
@@ -1141,6 +1173,7 @@ jobs:
- ci-images
env:
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
+ GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
# Only run it for direct pushes
if: >
github.ref == 'refs/heads/master' || github.ref == 'refs/heads/v1-10-test' ||
diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml
index 889cdb7..00663cb 100644
--- a/.github/workflows/scheduled_quarantined.yml
+++ b/.github/workflows/scheduled_quarantined.yml
@@ -33,8 +33,6 @@ env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
PYTHON_MAJOR_MINOR_VERSION: 3.6
USE_GITHUB_REGISTRY: "true"
- # Might be either 'ghcr.io' or 'docker.pkg.github.com'
- GITHUB_REGISTRY: "docker.pkg.github.com"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# This token is WRITE one - schedule type of events always have the WRITE token
@@ -46,6 +44,7 @@ env:
GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest"
GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest"
GITHUB_REGISTRY_WAIT_FOR_IMAGE: "false"
+ GITHUB_REGISTRY: ${{ secrets.OVERRIDE_GITHUB_REGISTRY }}
jobs:
@@ -88,6 +87,8 @@ jobs:
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Build CI image ${{ matrix.python-version }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+ env:
+ GITHUB_REGISTRY: ${{ steps.determine-github-registry.outputs.githubRegistry }}
- name: "Tests"
run: ./scripts/ci/testing/ci_run_airflow_testing.sh
- uses: actions/upload-artifact@v2
diff --git a/CI.rst b/CI.rst
index 850839b..0c8e03b 100644
--- a/CI.rst
+++ b/CI.rst
@@ -33,8 +33,6 @@ environments we use. Most of our CI jobs are written as bash scripts which are e
the CI jobs. And we have a number of variables determine build behaviour.
-
-
GitHub Actions runs
-------------------
@@ -53,14 +51,23 @@ techniques have been implemented that use efficiently cache from the GitHub Dock
this brings down the time needed to rebuild the image to ~4 minutes. In some cases (when dependencies change)
it can be ~6-7 minutes and in case base image of Python releases new patch-level, it can be ~12 minutes.
+Container Registry used as cache
+--------------------------------
+
+For the CI builds of our we are using Container Registry to store results of the "Build Image" workflow
+and pass it to the "CI Build" workflow.
+
Currently in master version of Airflow we run tests in 3 different versions of Python (3.6, 3.7, 3.8)
which means that we have to build 6 images (3 CI ones and 3 PROD ones). Yet we run around 12 jobs
with each of the CI images. That is a lot of time to just build the environment to run. Therefore
-we are utilising ``workflow_run`` feature of GitHub Actions. This feature allows to run a separate,
-independent workflow, when the main workflow is run - this separate workflow is different than the main
-one, because by default it runs using ``master`` version of the sources but also - and most of all - that
-it has WRITE access to the repository. This is especially important in our case where Pull Requests to
-Airflow might come from any repository, and it would be a huge security issue if anyone from outside could
+we are utilising ``workflow_run`` feature of GitHub Actions.
+
+This feature allows to run a separate, independent workflow, when the main workflow is run -
+this separate workflow is different than the main one, because by default it runs using ``master`` version
+of the sources but also - and most of all - that it has WRITE access to the repository.
+
+This is especially important in our case where Pull Requests to Airflow might come from any repository,
+and it would be a huge security issue if anyone from outside could
utilise the WRITE access to Apache Airflow repository via an external Pull Request.
Thanks to the WRITE access and fact that the 'workflow_run' by default uses the 'master' version of the
@@ -71,9 +78,56 @@ this image can be built only once and used by all the jobs running tests. The im
rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving
a lot of precious time for jobs.
-
-Local runs
-----------
+We can use either of the two available GitHub Container registries as cache:
+
+* Legacy `GitHub Package Registry <https://github.com/features/packages>`_ which is not very
+ stable, uses old infrastructure of GitHub and it lacks certain features - notably it does not allow
+ us to delete the old image. The benefit of using GitHub Package Registry is that it works
+ out-of-the-box (write authentication is done using ``GITHUB_TOKEN`` and users do not have to do any
+ action to make it work in case they want to run build using their own forks. Also those images
+ do not provide public access, so you need to login to ``docker.pkg.github.com`` docker registry
+ using your username and personal token to be able to pull those images.
+
+* The new `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_
+ which is in Public Beta, has many more features (including permission management, public access and
+ image retention possibility). It has also the drawback (at least as of January 2020) that you need to
+ have separate personal access token created as ``PAT_CR`` secret in your repository with write access
+ to registry in order to make it works. You also have to manually manage permissions of the images,
+ i.e. after creating images for the first time, you need to set their visibility to "Public" and
+ add ``Admin`` permissions to group of people managing the images (in our case ``airflow-committers`` group).
+ This makes it not very suitable to use GitHub container registry if you want to run builds of Airflow
+ in your own forks (note - it does not affect pull requests from forks to Airflow).
+
+Those two images have different naming schemas. See `Images documentation <IMAGES.rst>`_ for details.
+
+You can choose which registry should be used by the repository by setting ``OVERRIDE_GITHUB_REGISTRY`` secret
+to either ``docker.pkg.github.com`` for Github Package Registry or ``ghcr.io`` for GitHub Container Registry.
+Default is the Github Package Registry one. The Pull Request forks have no access to the secret but they
+auto-detect the registry used when they wait for the images.
+
+You can interact with the Github Registry images (pull/push) via `Breeze <BREEZE.rst>`_ - you can
+pass ``--github-registry`` flag wih either ``docker.pkg.github.com`` for Github Package Registry or
+``ghcr.io`` for GitHub Container Registry and pull/push operations will be performed using the chosen
+registry, using appropriate naming convention. This allows building and pushing the images locally by
+committers who have access to push/pull those images.
+
+
+Github Container Registry Token
+-------------------------------
+
+Unlike GitHub Packages, GitHub Registry requires a personal access token added as ``PAT_CR`` secret in order
+to make it works. This token has to have "Registry Write" scope. Ideally you should not use a token
+of a person who has access to many repositories, because this token allows to write packages in
+ANY repository, where the person has write access (including private organisations). Ideally, you need to have
+a separate account with only access to that repository and generate Personal Access Token with Package
+Registry write permission for that Account. Discussion about setting up such account is opened at
+`ASF Jira <https://issues.apache.org/jira/projects/INFRA/issues/INFRA-20959>`_. More info about
+the token for GitHub Container Registry can be found
+`here <https://docs.github.com/en/packages/guides/migrating-to-github-container-registry-for-docker-images#authenticating-with-the-container-registry>`_
+
+
+Locally replicating CI failures
+-------------------------------
The main goal of the CI philosophy we have that no matter how complex the test and integration
infrastructure, as a developer you should be able to reproduce and re-run any of the failed checks
@@ -808,7 +862,7 @@ you need to reproduce a MySQL environment with kerberos integration enabled for
.. code-block:: bash
- ./breeze --github-image-id 210056909 --python 3.8 --integration kerberos
+ ./breeze --github-image-id 210056909 --github-registry docker.pkg.github.com --python 3.8
You will be dropped into a shell with the exact version that was used during the CI run and you will
be able to run pytest tests manually, easily reproducing the environment that was used in CI. Note that in
@@ -842,3 +896,51 @@ Scheduled build flow
.. image:: images/ci/scheduled_ci_flow.png
:align: center
:alt: Scheduled build flow
+
+
+Adding new Python versions to CI
+--------------------------------
+
+In 2.0 line we currently support Python 3.6, 3.7, 3.8.
+
+In order to add a new version the following operations should be done (example uses python 3.9)
+
+* copy the latest constraints in ``constraints-master`` branch from previous versions and name it
+ using the new Python version (``constraints-3.9.txt``). Commit and push
+
+* add the new python version to `breeze-complete <breeze-complete>`_ and
+ `_initialization.sh <scripts/ci/libraries/_initialization.sh>`_ - tests will fail if they are not
+ in sync.
+
+* build image locally for both prod and CI locally using Breeze:
+
+.. code-block:: bash
+
+ ./breeze build-image --python 3.9
+
+* push image as cache to DockerHub and both registries:
+
+.. code-block:: bash
+
+ ./breeze push-image --python 3.9
+ ./breeze push-image --python 3.9 --github-registry ghcr.io
+ ./breeze push-image --python 3.9 --github-registry docker.pkg.github.com
+
+* Find the 3 new images (main, ci, build) created in
+ `GitHub Container registry<https://github.com/orgs/apache/packages?tab=packages&ecosystem=container&q=airflow>`_
+ go to Package Settings and turn on ``Public Visibility`` and add ``airflow-committers``
+ group as ``Admin Role`` to all of them.
+
+* In `DockerHub <https://hub.docker.com/repository/docker/apache/airflow/builds/edit>`_ create three entries
+ for automatically built nightly-tag and release images:
+
+
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
+| Source type | Source | Docker Tag | Dockerfile location | Build Context | Autobuild | Build caching | Comment |
++=============+================+=======================+=====================+===============+===========+===============+========================================================================+
+| Tag | nightly-master | master-python3.9 | Dockerfile | / | x | - | Nightly CI/PROD images from successful scheduled master nightly builds |
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
+| Branch | v2-0-stable | v2-0-stable-python3.9 | Dockerfile | / | x | | CI/PROD images automatically built pushed stable branch |
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
+| Tag | /^([1-2].*)$/ | {\1}-python3.9 | Dockerfile | / | x | | CI/PROD images automatically built from pushed release tags |
++-------------+----------------+-----------------------+---------------------+---------------+-----------+---------------+------------------------------------------------------------------------+
diff --git a/IMAGES.rst b/IMAGES.rst
index 102c120..c6d09b0 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -228,7 +228,12 @@ Choosing image registry
=======================
By default images are pulled and pushed from and to DockerHub registry when you use Breeze's push-image
-or build commands.
+or build commands. But as described in `CI Documentaton <CI.rst>`_, you can choose different image
+registry by setting ``GITHUB_REGISTRY`` to ``docker.pkg.github.com`` for Github Package Registry or
+``ghcr.io`` for GitHub Container Registry.
+
+Default is the Github Package Registry one. The Pull Request forks have no access to the secret but they
+auto-detect the registry used when they wait for the images.
Our images are named like that:
@@ -349,6 +354,60 @@ GitHub Container Registry
docker login ghcr.io
+Interacting with container registries
+=====================================
+
+Since there are different naming conventions used for Airflow images and there are multiple images used,
+`Breeze <BREEZE.rst>`_ provides easy to use management interface for the images. The
+`CI system of ours <CI.rst>`_ is designed in the way that it should automatically refresh caches, rebuild
+the images periodically and update them whenever new version of base python is released.
+However, occasionally, you might need to rebuild images locally and push them directly to the registries
+to refresh them.
+
+This can be done with ``Breeze`` command line which has easy-to-use tool to manage those images. For
+example:
+
+
+Force building Python 3.6 CI image using local cache and pushing it container registry:
+
+.. code-block:: bash
+
+ ./breeze build-image --python 3.6 --force-build-images --build-cache-local
+ ./breeze push-image --python 3.6 --github-registry ghcr.io
+
+
+Building Python 3.7 PROD images (both build and final image) using cache pulled
+from ``docker.pkg.github.com`` and pushing it back:
+
+.. code-block:: bash
+
+ ./breeze build-image --production-image --python 3.7 --github-registry docker.pkg.github.com
+ ./breeze push-image --production-image --python 3.7 --github-registry docker.pkg.github.com
+
+
+Building Python 3.8 CI image using cache pulled from DockerHub and pushing it back:
+
+.. code-block:: bash
+
+ ./breeze build-image --python 3.8
+ ./breeze push-image --python 3.8
+
+You can also pull and run images being result of a specific CI run in GitHub Actions. This is a powerful
+tool that allows to reproduce CI failures locally, enter the images and fix them much faster. It is enough
+to pass ``--github-image-id`` and the registry and Breeze will download and execute commands using
+the same image that was used during the CI build.
+
+For example this command will run the same Python 3.8 image as was used in 210056909
+run with enabled Kerberos integration (assuming docker.pkg.github.com was used as build cache).
+
+.. code-block:: bash
+
+ ./breeze --github-image-id 210056909 \
+ --github-registry docker.pkg.github.com \
+ --python 3.8 --integration kerberos
+
+You can see more details and examples in `Breeze <BREEZE.rst>`_
+
Technical details of Airflow images
===================================
diff --git a/scripts/ci/images/ci_wait_for_all_prod_images.sh b/scripts/ci/images/ci_wait_for_all_prod_images.sh
index 25bfd7c..626c3ae 100755
--- a/scripts/ci/images/ci_wait_for_all_prod_images.sh
+++ b/scripts/ci/images/ci_wait_for_all_prod_images.sh
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
echo
echo "Waiting for all PROD images to appear: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}"
echo
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 832cb17..c6631d0 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -509,7 +509,7 @@ function initialization::initialize_github_variables() {
# Defaults for interacting with GitHub
export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
export GITHUB_REGISTRY_IMAGE_SUFFIX=${GITHUB_REGISTRY_IMAGE_SUFFIX:="-v2"}
- export GITHUB_REGISTRY=${GITHUB_REGISTRY:="ghcr.io"}
+ export GITHUB_REGISTRY=${GITHUB_REGISTRY:="docker.pkg.github.com"}
export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
export GITHUB_REGISTRY_PUSH_IMAGE_TAG=${GITHUB_REGISTRY_PUSH_IMAGE_TAG:="latest"}
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index 1609e00..10c1503 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -285,74 +285,73 @@ function push_pull_remove_images::push_prod_images() {
fi
}
-# waits for an image to be available in GitHub Packages
-function push_pull_remove_images::wait_for_image_in_github_packages() {
+# waits for an image to be available in GitHub Packages. Should be run with `set +e`
+# the buid automatically determines which registry to use based one the images available
+function push_pull_remove_images::check_for_image_in_github_packages() {
local github_repository_lowercase
github_repository_lowercase="$(echo "${GITHUB_REPOSITORY}" |tr '[:upper:]' '[:lower:]')"
local github_api_endpoint
- github_api_endpoint="https://${GITHUB_REGISTRY}/v2/${github_repository_lowercase}"
+ github_api_endpoint="https://docker.pkg.github.com/v2/${github_repository_lowercase}"
local image_name_in_github_registry="${1}"
local image_tag_in_github_registry=${2}
-
- echo
- echo "Waiting for ${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry} image"
- echo
-
- GITHUB_API_CALL="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
- while true; do
- http_status=$(curl --silent --output "${OUTPUT_LOG}" --write-out "%{http_code}" \
- --connect-timeout 60 --max-time 60 \
- -X GET "${GITHUB_API_CALL}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}")
- if [[ ${http_status} == "200" ]]; then
- echo "${COLOR_GREEN}OK. ${COLOR_RESET}"
- break
- else
- echo "${COLOR_YELLOW}Still waiting - status code ${http_status}!${COLOR_RESET}"
- cat "${OUTPUT_LOG}"
- fi
- sleep 60
- done
- verbosity::print_info "Found ${image_name_in_github_registry}:${image_tag_in_github_registry} image"
+ local image_to_wait_for=${GITHUB_REPOSITORY}/${image_name_in_github_registry}:${image_tag_in_github_registry}
+ local github_api_call
+ github_api_call="${github_api_endpoint}/${image_name_in_github_registry}/manifests/${image_tag_in_github_registry}"
+ echo "Github Packages: checking for ${image_to_wait_for} via ${github_api_call}!"
+ http_status=$(curl --silent --output "${OUTPUT_LOG}" --write-out "%{http_code}" \
+ --connect-timeout 60 --max-time 60 \
+ -X GET "${github_api_call}" -u "${GITHUB_USERNAME}:${GITHUB_TOKEN}")
+ if [[ ${http_status} == "200" ]]; then
+ echo "Image: ${image_to_wait_for} found in GitHub Packages: ${COLOR_GREEN}OK. ${COLOR_RESET}"
+ echo "::set-output name=githubRegistry::docker.pkg.github.com"
+ echo
+ echo "Setting githubRegistry output to docker.pkg.github.com"
+ echo
+ return 0
+ else
+ cat "${OUTPUT_LOG}"
+ echo "${COLOR_YELLOW}Still waiting. Status code ${http_status}!${COLOR_RESET}"
+ return 1
+ fi
}
-
-# waits for an image to be available in GitHub Container Registry
-function push_pull_remove_images::wait_for_image_in_github_container_registry() {
+# waits for an image to be available in GitHub Container Registry. Should be run with `set +e`
+function push_pull_remove_images::check_for_image_in_github_container_registry() {
local image_name_in_github_registry="${1}"
local image_tag_in_github_registry=${2}
- local image_to_wait_for="${GITHUB_REGISTRY}/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}"
- echo
- echo "Waiting for ${GITHUB_REGISTRY}/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry} image"
- echo
+ local image_to_wait_for="ghcr.io/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}"
+ echo "Github Container Registry: checking for ${image_to_wait_for} via docker manifest inspect!"
+ docker manifest inspect "${image_to_wait_for}"
+ local res=$?
+ if [[ ${res} == "0" ]]; then
+ echo "Image: ${image_to_wait_for} found in Container Registry: ${COLOR_GREEN}OK.${COLOR_RESET}"
+ echo
+ echo "Setting githubRegistry output to ghcr.io"
+ echo
+ echo "::set-output name=githubRegistry::ghcr.io"
+ return 0
+ else
+ echo "${COLOR_YELLOW}Still waiting. Not found!${COLOR_RESET}"
+ return 1
+ fi
+}
+
+# waits for an image to be available in the GitHub registry
+function push_pull_remove_images::wait_for_github_registry_image() {
set +e
- while true; do
- docker manifest inspect "${image_to_wait_for}"
- local res=$?
- if [[ ${res} == "0" ]]; then
- echo "${COLOR_GREEN}OK.${COLOR_RESET}"
+ echo " Waiting for github registry image: " "${@}"
+ while true
+ do
+ if push_pull_remove_images::check_for_image_in_github_container_registry "${@}"; then
+ break
+ fi
+ if push_pull_remove_images::check_for_image_in_github_packages "${@}"; then
break
- else
- echo "${COLOR_YELLOW}Still waiting for ${image_to_wait_for}!${COLOR_RESET}"
fi
sleep 30
done
set -e
- verbosity::print_info "Found ${image_name_in_github_registry}:${image_tag_in_github_registry} image"
-}
-
-# waits for an image to be available in the GitHub registry
-function push_pull_remove_images::wait_for_github_registry_image() {
- if [[ ${GITHUB_REGISTRY} == "ghcr.io" ]]; then
- push_pull_remove_images::wait_for_image_in_github_container_registry "${@}"
- elif [[ ${GITHUB_REGISTRY} == "docker.pkg.github.com" ]]; then
- push_pull_remove_images::wait_for_image_in_github_packages "${@}"
- else
- echo
- echo "${COLOR_RED}ERROR: Bad value of '${GITHUB_REGISTRY}'. Should be either 'ghcr.io' or 'docker.pkg.github.com'!${COLOR_RESET}"
- echo
- exit 1
- fi
}
function push_pull_remove_images::check_if_github_registry_wait_for_image_enabled() {