You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/07/18 11:20:38 UTC
[airflow] 21/29: Move CI-integration images to ghcr.io (#16797)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e10acaa38854b1b4896059e314cd0ce63f932159
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Jul 14 07:39:00 2021 +0200
Move CI-integration images to ghcr.io (#16797)
This is the final step of moving the images used for CI integration
to `ghcr.io` from DockerHub. With Publicly available images
with self-management provided by GitHub, we can finally move to
keep the images "properly" - i.e. each image is separate and
tag is only image version.
Part of #16555
(cherry picked from commit 3143f1af44f96088eb417f34cc111d7361b4c6d8)
---
BREEZE.rst | 2 +-
CI.rst | 6 +++---
breeze | 2 +-
kubernetes_tests/test_kubernetes_pod_operator.py | 2 +-
scripts/ci/docker-compose/integration-kerberos.yml | 2 +-
scripts/ci/docker-compose/integration-openldap.yml | 2 +-
scripts/ci/docker-compose/integration-trino.yml | 2 +-
scripts/ci/dockerfiles/apache-rat/build_and_push.sh | 11 +++++------
scripts/ci/dockerfiles/bats/build_and_push.sh | 11 +++++------
scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh | 11 +++++------
scripts/ci/dockerfiles/stress/build_and_push.sh | 12 ++++++------
scripts/ci/dockerfiles/trino/Dockerfile | 2 +-
scripts/ci/dockerfiles/trino/build_and_push.sh | 13 ++++++-------
scripts/ci/static_checks/bats_tests.sh | 7 +++----
scripts/ci/static_checks/check_license.sh | 2 +-
tests/kubernetes/pod.yaml | 2 +-
tests/kubernetes/test_pod_generator.py | 4 ++--
17 files changed, 44 insertions(+), 49 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 7e6bedd..90d3f0b 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1199,7 +1199,7 @@ This is the current syntax for `./breeze <./breeze>`_:
you want to pull the image with specific COMMIT_SHA tag.
'breeze shell \
- -- github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
+ --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
'breeze \
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
diff --git a/CI.rst b/CI.rst
index d0eec1b..fedd300 100644
--- a/CI.rst
+++ b/CI.rst
@@ -78,7 +78,7 @@ this image can be built only once and used by all the jobs running tests. The im
rather than build it from the scratch. Pulling such image takes ~ 1 minute, thanks to that we are saving
a lot of precious time for jobs.
-We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_
+We use `GitHub Container Registry <https://docs.github.com/en/packages/guides/about-github-container-registry>`_.
``GITHUB_TOKEN`` is needed to push to the registry and we configured scopes of the tokens in our jobs
to be able to write to the registry.
@@ -411,9 +411,9 @@ The following components are part of the CI infrastructure
* **GitHub Actions** - (GA) UI + execution engine for our jobs
* **GA CRON trigger** - GitHub Actions CRON triggering our jobs
* **GA Workers** - virtual machines running our jobs at GitHub Actions (max 20 in parallel)
-* **GitHub Image Registry**- image registry used as build cache for CI jobs.
+* **GitHub Image Registry** - image registry used as build cache for CI jobs.
It is at https://ghcr.io/apache/airflow
-* **DockerHub Image Registry**- image registry used to pull base Python images and (manually) publish
+* **DockerHub Image Registry** - image registry used to pull base Python images and (manually) publish
the released Production Airflow images. It is at https://dockerhub.com/apache/airflow
* **Official Images** (future) - these are official images that are prominently visible in DockerHub.
We aim our images to become official images so that you will be able to pull them
diff --git a/breeze b/breeze
index 3b09208..a6454c5 100755
--- a/breeze
+++ b/breeze
@@ -1665,7 +1665,7 @@ ${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]
you want to pull the image with specific COMMIT_SHA tag.
'${CMDNAME} shell \\
- -- github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
+ --github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
'${CMDNAME} \\
--github-image-id 9a621eaa394c0a0a336f8e1b31b35eff4e4ee86e' - pull/use image with SHA
diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py
index 9d32782..63b3fb6 100644
--- a/kubernetes_tests/test_kubernetes_pod_operator.py
+++ b/kubernetes_tests/test_kubernetes_pod_operator.py
@@ -869,7 +869,7 @@ class TestKubernetesPodOperatorSystem(unittest.TestCase):
'command': ['stress'],
'env': [],
'envFrom': [],
- 'image': 'apache/airflow-ci:stress-2021.04.28-1.0.4',
+ 'image': 'ghcr.io/apache/airflow-stress:1.0.4-2021.07.04',
'imagePullPolicy': 'IfNotPresent',
'name': 'base',
'ports': [],
diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml
index b9ca4cc..0e79c6a 100644
--- a/scripts/ci/docker-compose/integration-kerberos.yml
+++ b/scripts/ci/docker-compose/integration-kerberos.yml
@@ -18,7 +18,7 @@
version: "2.2"
services:
kdc-server-example-com:
- image: apache/airflow-ci:krb5-kdc-server-2021.04.28
+ image: ghcr.io/apache/airflow-krb5-kdc-server:2021.07.04
hostname: krb5-kdc-server-example-com
domainname: example.com
networks:
diff --git a/scripts/ci/docker-compose/integration-openldap.yml b/scripts/ci/docker-compose/integration-openldap.yml
index 60ff58a..eea3246 100644
--- a/scripts/ci/docker-compose/integration-openldap.yml
+++ b/scripts/ci/docker-compose/integration-openldap.yml
@@ -18,7 +18,7 @@
version: "2.2"
services:
openldap:
- image: apache/airflow-ci:openldap-2020.07.10-2.4.50
+ image: ghcr.io/apache/airflow-openldap:2.4.50-2021.07.04
command: "--copy-service"
environment:
- LDAP_DOMAIN=example.com
diff --git a/scripts/ci/docker-compose/integration-trino.yml b/scripts/ci/docker-compose/integration-trino.yml
index 822a2ff..e381ced 100644
--- a/scripts/ci/docker-compose/integration-trino.yml
+++ b/scripts/ci/docker-compose/integration-trino.yml
@@ -18,7 +18,7 @@
version: "2.2"
services:
trino:
- image: apache/airflow-ci:trino-2021.04.28
+ image: ghcr.io/apache/airflow-trino:359-2021.07.04
container_name: trino
hostname: trino
domainname: example.com
diff --git a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
index ceed534..228b301 100755
--- a/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
+++ b/scripts/ci/dockerfiles/apache-rat/build_and_push.sh
@@ -16,15 +16,13 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
APACHERAT_VERSION="0.13"
readonly APACHERAT_VERSION
-AIRFLOW_APACHERAT_VERSION="2021.04.28"
+AIRFLOW_APACHERAT_VERSION="2021.07.04"
readonly AIRFLOW_APACHERAT_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
@@ -32,7 +30,7 @@ readonly COMMIT_SHA
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:apache-rat-${AIRFLOW_APACHERAT_VERSION}-${APACHERAT_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-apache-rat:${APACHERAT_VERSION}-${AIRFLOW_APACHERAT_VERSION}"
readonly TAG
docker build . \
@@ -40,6 +38,7 @@ docker build . \
--build-arg "APACHERAT_VERSION=${APACHERAT_VERSION}" \
--build-arg "AIRFLOW_APACHERAT_VERSION=${AIRFLOW_APACHERAT_VERSION}" \
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
--tag "${TAG}"
docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/bats/build_and_push.sh b/scripts/ci/dockerfiles/bats/build_and_push.sh
index 1f35c9a..20b6c02 100755
--- a/scripts/ci/dockerfiles/bats/build_and_push.sh
+++ b/scripts/ci/dockerfiles/bats/build_and_push.sh
@@ -16,10 +16,8 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
BATS_VERSION="1.2.1"
BATS_ASSERT_VERSION="2.0.0"
@@ -30,7 +28,7 @@ readonly BATS_ASSERT_VERSION
readonly BATS_SUPPORT_VERSION
readonly BATS_FILE_VERSION
-AIRFLOW_BATS_VERSION="2021.04.28"
+AIRFLOW_BATS_VERSION="2021.07.04"
readonly AIRFLOW_BATS_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
@@ -38,7 +36,7 @@ readonly COMMIT_SHA
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:bats-${AIRFLOW_BATS_VERSION}-${BATS_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-bats:${BATS_VERSION}-${AIRFLOW_BATS_VERSION}"
readonly TAG
docker build . \
@@ -48,6 +46,7 @@ docker build . \
--build-arg "BATS_FILE_VERSION=${BATS_FILE_VERSION}" \
--build-arg "BATS_ASSERT_VERSION=${BATS_ASSERT_VERSION}" \
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
--tag "${TAG}"
docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh b/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh
index e1a242a..6ad4354 100755
--- a/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh
+++ b/scripts/ci/dockerfiles/krb5-kdc-server/build_and_push.sh
@@ -16,12 +16,10 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
-AIRFLOW_KRB5KDCSERVER_VERSION="2021.04.28"
+AIRFLOW_KRB5KDCSERVER_VERSION="2021.07.04"
readonly AIRFLOW_KRB5KDCSERVER_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
@@ -29,13 +27,14 @@ readonly COMMIT_SHA
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:krb5-kdc-server-${AIRFLOW_KRB5KDCSERVER_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-krb5-kdc-server:${AIRFLOW_KRB5KDCSERVER_VERSION}"
readonly TAG
docker build . \
--pull \
--build-arg "AIRFLOW_KRB5KDCSERVER_VERSION=${AIRFLOW_KRB5KDCSERVER_VERSION}" \
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
--tag "${TAG}"
docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/stress/build_and_push.sh b/scripts/ci/dockerfiles/stress/build_and_push.sh
index c158456..ea3144d 100755
--- a/scripts/ci/dockerfiles/stress/build_and_push.sh
+++ b/scripts/ci/dockerfiles/stress/build_and_push.sh
@@ -16,14 +16,13 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
+
STRESS_VERSION="1.0.4"
readonly STRESS_VERSION
-AIRFLOW_STRESS_VERSION="2021.04.28"
+AIRFLOW_STRESS_VERSION="2021.07.04"
readonly AIRFLOW_STRESS_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
@@ -31,7 +30,7 @@ readonly COMMIT_SHA
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:stress-${AIRFLOW_STRESS_VERSION}-${STRESS_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-stress:${STRESS_VERSION}-${AIRFLOW_STRESS_VERSION}"
readonly TAG
docker build . \
@@ -39,6 +38,7 @@ docker build . \
--build-arg "STRESS_VERSION=${STRESS_VERSION}" \
--build-arg "AIRFLOW_STRESS_VERSION=${AIRFLOW_STRESS_VERSION}" \
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
--tag "${TAG}"
docker push "${TAG}"
diff --git a/scripts/ci/dockerfiles/trino/Dockerfile b/scripts/ci/dockerfiles/trino/Dockerfile
index 32ef0fd..a36a8cc 100644
--- a/scripts/ci/dockerfiles/trino/Dockerfile
+++ b/scripts/ci/dockerfiles/trino/Dockerfile
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-ARG TRINO_VERSION="354"
+ARG TRINO_VERSION="359"
FROM trinodb/trino:${TRINO_VERSION}
# Obtain root privileges
diff --git a/scripts/ci/dockerfiles/trino/build_and_push.sh b/scripts/ci/dockerfiles/trino/build_and_push.sh
index c5ebbc7c4..3f876f7 100755
--- a/scripts/ci/dockerfiles/trino/build_and_push.sh
+++ b/scripts/ci/dockerfiles/trino/build_and_push.sh
@@ -16,15 +16,13 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
-DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
-DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow-ci"}
-readonly DOCKERHUB_USER
-readonly DOCKERHUB_REPO
+GITHUB_REPOSITORY=${GITHUB_REPOSITORY:="apache/airflow"}
+readonly GITHUB_REPOSITORY
-TRINO_VERSION="354"
+TRINO_VERSION="359"
readonly TRINO_VERSION
-AIRFLOW_TRINO_VERSION="2021.04.28"
+AIRFLOW_TRINO_VERSION="2021.07.04"
readonly AIRFLOW_TRINO_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
@@ -32,7 +30,7 @@ readonly COMMIT_SHA
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
-TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:trino-${AIRFLOW_TRINO_VERSION}"
+TAG="ghcr.io/${GITHUB_REPOSITORY}-trino:${TRINO_VERSION}-${AIRFLOW_TRINO_VERSION}"
readonly TAG
docker build . \
@@ -40,6 +38,7 @@ docker build . \
--build-arg "TRINO_VERSION=${TRINO_VERSION}" \
--build-arg "AIRFLOW_TRINO_VERSION=${AIRFLOW_TRINO_VERSION}" \
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
+ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
--tag "${TAG}"
docker push "${TAG}"
diff --git a/scripts/ci/static_checks/bats_tests.sh b/scripts/ci/static_checks/bats_tests.sh
index 6130ad5..55a62f3 100755
--- a/scripts/ci/static_checks/bats_tests.sh
+++ b/scripts/ci/static_checks/bats_tests.sh
@@ -50,19 +50,18 @@ function run_bats_tests() {
fi
done
+ local airflow_bats_image="ghcr.io/apache/airflow-bats:1.2.1-2021.07.04"
# deduplicate
FS=" " read -r -a bats_arguments <<< "$(tr ' ' '\n' <<< "${bats_arguments[@]}" | sort -u | tr '\n' ' ' )"
if [[ ${#@} == "0" ]]; then
# Run al tests
- docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow-ci:bats-2021.04.28-1.2.1 --tap /airflow/tests/bats/
+ docker run --workdir /airflow -v "$(pwd):/airflow" --rm "${airflow_bats_image}" --tap /airflow/tests/bats/
elif [[ ${#bats_arguments} == "0" ]]; then
# Skip running anything if all filtered out
true
else
# Run selected tests
- docker run --workdir /airflow -v "$(pwd):/airflow" --rm \
- apache/airflow-ci:bats-2021.04.28-1.2.1 --tap "${bats_arguments[@]}"
+ docker run --workdir /airflow -v "$(pwd):/airflow" --rm "${airflow_bats_image}" --tap "${bats_arguments[@]}"
fi
}
diff --git a/scripts/ci/static_checks/check_license.sh b/scripts/ci/static_checks/check_license.sh
index ce6e6fa..ef365de 100755
--- a/scripts/ci/static_checks/check_license.sh
+++ b/scripts/ci/static_checks/check_license.sh
@@ -34,7 +34,7 @@ function run_check_license() {
if ! docker_v run -v "${AIRFLOW_SOURCES}:/opt/airflow" -t \
--user "$(id -ur):$(id -gr)" \
--rm --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
- apache/airflow-ci:apache-rat-2021.04.28-0.13 \
+ ghcr.io/apache/airflow-apache-rat:0.13-2021.07.04 \
--exclude-file /opt/airflow/.rat-excludes \
--d /opt/airflow | tee "${AIRFLOW_SOURCES}/logs/rat-results.txt" ; then
echo
diff --git a/tests/kubernetes/pod.yaml b/tests/kubernetes/pod.yaml
index 5e02880..b698444 100644
--- a/tests/kubernetes/pod.yaml
+++ b/tests/kubernetes/pod.yaml
@@ -23,7 +23,7 @@ metadata:
spec:
containers:
- name: memory-demo-ctr
- image: apache/airflow-ci:stress-2021.04.28-1.0.4
+ image: ghcr.io/apache/airflow-stress:1.0.4-2021.07.04
resources:
limits:
memory: "200Mi"
diff --git a/tests/kubernetes/test_pod_generator.py b/tests/kubernetes/test_pod_generator.py
index 3ca60c1..b288075 100644
--- a/tests/kubernetes/test_pod_generator.py
+++ b/tests/kubernetes/test_pod_generator.py
@@ -50,7 +50,7 @@ class TestPodGenerator(unittest.TestCase):
{
'args': ['--vm', '1', '--vm-bytes', '150M', '--vm-hang', '1'],
'command': ['stress'],
- 'image': 'apache/airflow-ci:stress-2021.04.28-1.0.4',
+ 'image': 'ghcr.io/apache/airflow-stress:1.0.4-2021.07.04',
'name': 'memory-demo-ctr',
'resources': {'limits': {'memory': '200Mi'}, 'requests': {'memory': '100Mi'}},
}
@@ -688,7 +688,7 @@ metadata:
spec:
containers:
- name: memory-demo-ctr
- image: apache/airflow-ci:stress-2021.04.28-1.0.4
+ image: ghcr.io/apache/airflow-stress:1.0.4-2021.07.04
resources:
limits:
memory: "200Mi"