You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/07/02 19:27:28 UTC
[airflow] 01/01: Switch Breeze to use Github Container Registry
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to tag providers-elasticsearch/2.0.2
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 15a7d4aa73bd7dddcf23997dab2b0b0137e30bc9
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Jul 2 18:28:48 2021 +0200
Switch Breeze to use Github Container Registry
Breeze used traditionally DockerHub to pull images, because
they were public and GitHub Packages were not. With GitHub Container
Regisry however, we can switch fully to using GitHub Container
Registry also for Breeze.
The only remaining GitHub Images now will be:
* images used during CI for integrations (airflow-ci)
* released/versioned Airflow images (airflow)
The integration images will be moved to GitHub Container Registry
in a subsequent PR.
---
.github/workflows/build-images.yml | 5 +-
.github/workflows/ci.yml | 41 +++---
BREEZE.rst | 2 +-
CI.rst | 18 ---
breeze | 53 +-------
scripts/ci/docker-compose/base.yml | 2 +-
scripts/ci/docker-compose/local-prod.yml | 37 ------
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 12 +-
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 15 +--
scripts/ci/images/ci_push_ci_images.sh | 2 +-
scripts/ci/images/ci_push_production_images.sh | 2 +-
.../ci/images/ci_wait_for_and_verify_ci_image.sh | 15 +--
.../ci/images/ci_wait_for_and_verify_prod_image.sh | 15 +--
scripts/ci/libraries/_build_images.sh | 141 +++++++-------------
scripts/ci/libraries/_initialization.sh | 10 --
scripts/ci/libraries/_push_pull_remove_images.sh | 142 +++++----------------
.../build_dockerhub.sh} | 19 ++-
scripts/ci/tools/ci_clear_tmp.sh | 34 -----
.../{ci_fix_ownership.sh => fix_ownership.sh} | 0
.../{ci_free_space_on_ci.sh => free_space.sh} | 0
scripts/ci/tools/prepare_prod_docker_images.sh | 2 +-
scripts/in_container/run_clear_tmp.sh | 21 ---
22 files changed, 137 insertions(+), 451 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 9cbae1d..4afee08 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -34,7 +34,6 @@ env:
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
VERBOSE: "true"
- USE_GITHUB_REGISTRY: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
@@ -199,7 +198,7 @@ jobs:
rm -rf "scripts/ci"
mv "main-airflow/scripts/ci" "scripts"
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Build CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Push CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}"
@@ -274,7 +273,7 @@ jobs:
rm -rf "scripts/ci"
mv "main-airflow/scripts/ci" "scripts"
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Build CI images ${{ matrix.python-version }}:${{ env.TARGET_COMMIT_SHA }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
# Pull images built in the previous step
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 52ce958..52e139e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -36,7 +36,6 @@ env:
DB_RESET: "true"
VERBOSE: "true"
DOCKER_CACHE: "pulled"
- USE_GITHUB_REGISTRY: "true"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.actor }}
# You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the
@@ -246,7 +245,7 @@ jobs:
fetch-depth: 2
persist-credentials: false
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
if: |
needs.build-info.outputs.waitForImage == 'true'
- name: "Setup python"
@@ -279,7 +278,7 @@ jobs:
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
if: needs.build-info.outputs.waitForImage == 'true'
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
if: |
needs.build-info.outputs.waitForImage == 'true'
- name: >
@@ -318,7 +317,7 @@ jobs:
with:
python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Get Python version"
@@ -419,7 +418,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
persist-credentials: false
submodules: recursive
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Fetch inventory versions"
@@ -473,7 +472,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare provider documentation"
@@ -521,7 +520,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare provider packages: sdist"
@@ -563,7 +562,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: Helm"
@@ -620,7 +619,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
@@ -676,7 +675,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
@@ -731,7 +730,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
@@ -784,7 +783,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: ${{needs.build-info.outputs.testTypes}}"
@@ -847,7 +846,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: |
echo "ISSUE_ID=10128" >> $GITHUB_ENV
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Tests: Quarantined"
@@ -935,7 +934,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
if: needs.build-info.outputs.waitForImage == 'true'
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
if: |
needs.build-info.outputs.waitForImage == 'true'
- name: >
@@ -986,7 +985,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Get all PROD images"
run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- name: "Cache virtualenv for kubernetes testing"
@@ -1056,7 +1055,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Get all PROD images"
run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
- name: "Cache virtualenv for kubernetes testing"
@@ -1133,7 +1132,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: Set push-python-image
id: push-python-image
run: |
@@ -1194,7 +1193,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Push CI image ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
@@ -1235,7 +1234,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: >
Wait for CI images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
@@ -1296,7 +1295,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
persist-credentials: false
submodules: recursive
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Tag commit"
run: |
BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///')
@@ -1326,7 +1325,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
node-version: 14
- name: "Free space"
- run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ run: ./scripts/ci/tools/free_space.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Cache eslint"
diff --git a/BREEZE.rst b/BREEZE.rst
index 22f7e0c..de98078 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1007,7 +1007,7 @@ by the root user, you can fix the ownership of those files by running this scrip
.. code-block::
- ./scripts/ci/tools/ci_fix_ownership.sh
+ ./scripts/ci/tools/fix_ownership.sh
Mounting Local Sources to Breeze
--------------------------------
diff --git a/CI.rst b/CI.rst
index 29abae5..f275b8a 100644
--- a/CI.rst
+++ b/CI.rst
@@ -400,9 +400,6 @@ the model of permission management is not the same for Container Registry as it
+--------------------------------+---------------------------+----------------------------------------------+
| Variable | Default | Comment |
+================================+===========================+==============================================+
-| USE_GITHUB_REGISTRY | true | If set to "true", we interact with GitHub |
-| | | Registry registry not the DockerHub one. |
-+--------------------------------+---------------------------+----------------------------------------------+
| GITHUB_REGISTRY | ``ghcr.io`` | Name of the GitHub registry to use. Can be |
| | | ``docker.pkg.github.com`` or ``ghcr.io`` |
+--------------------------------+---------------------------+----------------------------------------------+
@@ -438,21 +435,6 @@ is needed for both pushing the images (WRITE) and pulling them (READ) - which me
is used in "main" build (WRITE) and in fork builds (READ). For container registry, our images are
Publicly Visible and we do not need any authentication to pull them.
-Dockerhub Variables
-===================
-
-If ``USE_GITHUB_REGISTRY`` is set to "false" you can interact directly with DockerHub. By default
-you pull from/push to "apache/airflow" DockerHub repository, but you can change
-that to your own repository by setting those environment variables:
-
-+----------------+----------------+-----------------------------------+
-| Variable | Default | Comment |
-+================+================+===================================+
-| DOCKERHUB_USER | ``apache`` | Name of the DockerHub user to use |
-+----------------+----------------+-----------------------------------+
-| DOCKERHUB_REPO | ``airflow-ci`` | Name of the DockerHub repo to use |
-+----------------+----------------+-----------------------------------+
-
CI Architecture
===============
diff --git a/breeze b/breeze
index e239960..8807873 100755
--- a/breeze
+++ b/breeze
@@ -483,7 +483,6 @@ EOF
Branch name: ${BRANCH_NAME}
Docker image: ${AIRFLOW_PROD_IMAGE}
- GitHub cache prefix: ${GITHUB_REGISTRY}/$(get_github_container_registry_image_prefix)
Airflow source version: $(build_images::get_airflow_version_from_production_image)
EOF
else
@@ -493,7 +492,6 @@ EOF
Branch name: ${BRANCH_NAME}
Docker image: ${AIRFLOW_CI_IMAGE}
- GitHub cache prefix: ${GITHUB_REGISTRY}/$(get_github_container_registry_image_prefix)
Airflow source version: ${AIRFLOW_VERSION}
EOF
fi
@@ -518,7 +516,6 @@ EOF
Branch name: ${BRANCH_NAME}
Docker image: ${AIRFLOW_PROD_IMAGE}
- GitHub cache prefix: ${GITHUB_REGISTRY}/$(get_github_container_registry_image_prefix)
EOF
else
cat <<EOF
@@ -527,7 +524,6 @@ EOF
Branch name: ${BRANCH_NAME}
Docker image: ${AIRFLOW_CI_IMAGE}
- GitHub cache prefix: ${GITHUB_REGISTRY}/$(get_github_container_registry_image_prefix)
EOF
fi
@@ -600,7 +596,6 @@ function breeze::prepare_command_file() {
local file="${1}"
local command="${2}"
local compose_file="${3}"
- local airflow_image="${4}"
cat <<EOF >"${file}"
#!/usr/bin/env bash
if [[ \${VERBOSE} == "true" ]]; then
@@ -634,7 +629,6 @@ export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}"
export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}"
export AIRFLOW_PROD_IMAGE_KUBERNETES="${AIRFLOW_PROD_IMAGE_KUBERNETES}"
export AIRFLOW_PROD_BASE_TAG="${AIRFLOW_PROD_BASE_TAG}"
-export AIRFLOW_IMAGE="${airflow_image}"
export SQLITE_URL="${SQLITE_URL}"
export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
@@ -646,32 +640,23 @@ EOF
#######################################################################################################
#
-# Prepare all command files that we are using. Depending on the command to execute we use two
-# convenience scripts:
-#
-# dc_ci - to run docker compose command for CI image
-# dc_prod - to run docker compose command for PROD image
+# Prepare all command files that we are using. We use dc_ci - to run docker compose command for CI image
#
# Global constants set:
#
# PYTHON_BASE_IMAGE_VERSION
# PYTHON_BASE_IMAGE
# AIRFLOW_CI_IMAGE
-# AIRFLOW_PROD_BASE_TAG
-# AIRFLOW_PROD_IMAGE
-# AIRFLOW_PROD_IMAGE_KUBERNETES
# BUILT_CI_IMAGE_FLAG_FILE
#
#######################################################################################################
function breeze::prepare_command_files() {
local main_ci_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
- local main_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
local backend_port_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}-port.yml
local local_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local.yml
local local_all_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local-all-sources.yml
local files_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/files.yml
- local local_prod_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local-prod.yml
local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
@@ -692,21 +677,17 @@ function breeze::prepare_command_files() {
local compose_ci_file=${main_ci_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}
- local compose_prod_file=${main_prod_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}
if [[ "${MOUNT_SELECTED_LOCAL_SOURCES}" != "false" ]]; then
compose_ci_file=${compose_ci_file}:${local_docker_compose_file}:${backend_port_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${local_prod_docker_compose_file}:${backend_port_docker_compose_file}
fi
if [[ "${MOUNT_ALL_LOCAL_SOURCES}" != "false" ]]; then
compose_ci_file=${compose_ci_file}:${local_all_sources_docker_compose_file}:${backend_port_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${local_all_sources_docker_compose_file}:${backend_port_docker_compose_file}
fi
if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
compose_ci_file=${compose_ci_file}:${forward_credentials_docker_compose_file}
- compose_prod_file=${compose_prod_file}:${forward_credentials_docker_compose_file}
fi
if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE} || -n ${USE_AIRFLOW_VERSION=} ]]; then
@@ -727,16 +708,8 @@ function breeze::prepare_command_files() {
export DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI="dc_ci"
readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI
- export DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD="dc_prod"
- readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD
-
- # Prepare script for "run docker compose CI command"
breeze::prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" \
- "\"\${@}\"" "${compose_ci_file}" "${AIRFLOW_CI_IMAGE}"
-
- # Prepare script for "run docker compose PROD command"
- breeze::prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}" \
- "\"\${@}\"" "${compose_prod_file}" "${AIRFLOW_PROD_IMAGE}"
+ "\"\${@}\"" "${compose_ci_file}"
}
#######################################################################################################
@@ -1152,20 +1125,12 @@ function breeze::parse_arguments() {
export FORWARD_CREDENTIALS="true"
shift
;;
- -c | --use-github-registry)
- echo
- echo "Use GitHub registry"
- echo
- export USE_GITHUB_REGISTRY="true"
- shift
- ;;
-g | --github-repository)
echo
echo "Using GitHub registry."
echo "GitHub repository: ${2}"
echo
export GITHUB_REPOSITORY="${2}"
- export USE_GITHUB_REGISTRY="true"
shift 2
;;
-s | --github-image-id)
@@ -1178,7 +1143,6 @@ function breeze::parse_arguments() {
echo "behaviour as in the CI environment."
echo
export FORCE_PULL_IMAGES="true"
- export USE_GITHUB_REGISTRY="true"
export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}"
export GITHUB_REGISTRY_PUSH_IMAGE_TAG="${2}"
export CHECK_IMAGE_FOR_REBUILD="false"
@@ -3300,16 +3264,11 @@ function breeze::make_sure_precommit_is_installed() {
function breeze::remove_images() {
# shellcheck disable=SC2086
docker rmi --force ${PYTHON_BASE_IMAGE} \
- ${GITHUB_REGISTRY_PYTHON_BASE_IMAGE} \
${AIRFLOW_PYTHON_BASE_IMAGE} \
- ${AIRFLOW_CI_IMAGE} \
- ${DEFAULT_CI_IMAGE} \
${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE} \
- ${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE} \
+ ${AIRFLOW_CI_IMAGE} \
${AIRFLOW_PROD_IMAGE} \
- ${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE} \
${AIRFLOW_PROD_BUILD_IMAGE} \
- ${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE} \
2>/dev/null >/dev/null && true
echo
echo "###################################################################"
@@ -3514,7 +3473,7 @@ function breeze::run_breeze_command() {
docker_engine_resources::check_all_resources
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
${run_command} "${dc_run_file}" run --service-ports --rm airflow "${@}"
- ${run_command} "${SCRIPTS_CI_DIR}/tools/ci_fix_ownership.sh" || true
+ ${run_command} "${SCRIPTS_CI_DIR}/tools/fix_ownership.sh" || true
else
${run_command} "${dc_run_file}" run --service-ports --rm airflow "${@}"
fi
@@ -3571,9 +3530,9 @@ function breeze::run_breeze_command() {
;;
perform_push_image)
if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
- push_pull_remove_images::push_prod_images
+ push_pull_remove_images::push_prod_images_to_github
else
- push_pull_remove_images::push_ci_images
+ push_pull_remove_images::push_ci_images_to_github
fi
;;
perform_initialize_local_virtualenv)
diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml
index 78e9b7d..255bb74 100644
--- a/scripts/ci/docker-compose/base.yml
+++ b/scripts/ci/docker-compose/base.yml
@@ -18,7 +18,7 @@
version: "2.2"
services:
airflow:
- image: ${AIRFLOW_IMAGE}
+ image: ${AIRFLOW_CI_IMAGE}
environment:
- USER=root
- ADDITIONAL_PATH=~/.local/bin
diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml
deleted file mode 100644
index 79476d0..0000000
--- a/scripts/ci/docker-compose/local-prod.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
----
-version: "2.2"
-services:
- airflow:
- # We need to mount files an directories individually because some files
- # such apache_airflow.egg-info should not be mounted from host
- # we only mount those files that it makes sense to edit while developing
- # or those that might be useful to see in the host as output of the
- # tests (such as logs)
- volumes:
- - ../../../.bash_aliases:/root/.bash_aliases:cached
- - ../../../.bash_history:/root/.bash_history:cached
- - ../../../.github:/opt/airflow/.github:cached
- - ../../../.inputrc:/root/.inputrc:cached
- - ../../../.kube:/root/.kube:cached
- - ../../../scripts/in_container/prod/entrypoint_prod.sh:/entrypoint:cached
- - ../../../setup.cfg:/opt/airflow/setup.cfg:cached
- - ../../../setup.py:/opt/airflow/setup.py:cached
- - ../../../tests:/opt/airflow/tests:cached
- - ../../../tmp:/tmp:cached
- - ../../../metastore_browser:/opt/airflow/metastore_browser:cached
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index 415f7a5..a550038 100755
--- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -19,7 +19,7 @@
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
# Builds or waits for the CI image in the CI environment
-# Depending on "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
+# Depending on "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_ci_image_on_ci() {
build_images::prepare_ci_build
start_end::group_start "Prepare CI image ${AIRFLOW_CI_IMAGE}"
@@ -27,7 +27,7 @@ function build_ci_image_on_ci() {
rm -rf "${BUILD_CACHE_DIR}"
mkdir -pv "${BUILD_CACHE_DIR}"
- if [[ ${USE_GITHUB_REGISTRY} == "true" && ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
+ if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
# Pretend that the image was build. We already have image with the right sources baked in!
md5sum::calculate_md5sum_for_all_files
@@ -40,12 +40,12 @@ function build_ci_image_on_ci() {
fi
# first we pull base python image. We will need it to re-push it after main build
# Becoming the new "latest" image for other builds
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
- "${python_tag_suffix}" "${AIRFLOW_PYTHON_BASE_IMAGE}"
+ build_images::wait_for_image_tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \
+ "${python_tag_suffix}"
# And then the actual image
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}" \
- ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_CI_IMAGE}"
+ build_images::wait_for_image_tag "${AIRFLOW_CI_IMAGE}" \
+ ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
md5sum::update_all_md5_with_group
else
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index fe8b489..dbcb07d 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -28,11 +28,10 @@ export VERBOSE="true"
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
# Builds or waits for the PROD image in the CI environment
-# Depending on the "USE_GITHUB_REGISTRY" and "GITHUB_REGISTRY_WAIT_FOR_IMAGE" setting
function build_prod_images_on_ci() {
build_images::prepare_prod_build
- if [[ ${USE_GITHUB_REGISTRY} == "true" && ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
+ if [[ ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
# Tries to wait for the images indefinitely
# skips further image checks - since we already have the target image
@@ -44,19 +43,19 @@ function build_prod_images_on_ci() {
if [[ "${WAIT_FOR_PYTHON_BASE_IMAGE=}" == "true" ]]; then
# first we pull base python image. We will need it to re-push it after main build
# Becoming the new "latest" image for other builds
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
- "${python_tag_suffix}" "${AIRFLOW_PYTHON_BASE_IMAGE}"
+ build_images::wait_for_image_tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \
+ "${python_tag_suffix}"
fi
# And then the actual image
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
- ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_IMAGE}"
+ build_images::wait_for_image_tag "${AIRFLOW_PROD_IMAGE}" \
+ ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
# And the prod build image
if [[ "${WAIT_FOR_PROD_BUILD_IMAGE=}" == "true" ]]; then
# If specified in variable - also waits for the build image
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \
- ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_BUILD_IMAGE}"
+ build_images::wait_for_image_tag "${AIRFLOW_PROD_BUILD_IMAGE}" \
+ ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
else
diff --git a/scripts/ci/images/ci_push_ci_images.sh b/scripts/ci/images/ci_push_ci_images.sh
index df2ae17..5bd5eb3 100755
--- a/scripts/ci/images/ci_push_ci_images.sh
+++ b/scripts/ci/images/ci_push_ci_images.sh
@@ -20,4 +20,4 @@
build_images::prepare_ci_build
-push_pull_remove_images::push_ci_images
+push_pull_remove_images::push_ci_images_to_github
diff --git a/scripts/ci/images/ci_push_production_images.sh b/scripts/ci/images/ci_push_production_images.sh
index 31216c3..43e14d3 100755
--- a/scripts/ci/images/ci_push_production_images.sh
+++ b/scripts/ci/images/ci_push_production_images.sh
@@ -20,4 +20,4 @@
build_images::prepare_prod_build
-push_pull_remove_images::push_prod_images
+push_pull_remove_images::push_prod_images_to_github
diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
index b86a8f0..f66411e 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
+++ b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
@@ -29,30 +29,25 @@ shift
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
function pull_ci_image() {
- local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ local image_name_with_tag="${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
start_end::group_start "Pulling ${image_name_with_tag} image"
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}"
+ push_pull_remove_images::pull_image_if_not_present_or_forced "${image_name_with_tag}"
start_end::group_end
}
-push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
-
start_end::group_start "Configure Docker Registry"
build_images::configure_docker_registry
start_end::group_end
-export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
-
-start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear"
+start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE}"
-push_pull_remove_images::wait_for_github_registry_image \
- "${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+push_pull_remove_images::wait_for_image "${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
build_images::prepare_ci_build
pull_ci_image
-verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
+verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
start_end::group_end
diff --git a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
index 6215147..5142bda 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
+++ b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
@@ -28,25 +28,20 @@ shift
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-push_pull_remove_images::check_if_github_registry_wait_for_image_enabled
-
start_end::group_start "Configure Docker Registry"
build_images::configure_docker_registry
start_end::group_end
-export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
-
-start_end::group_start "Waiting for ${AIRFLOW_PROD_IMAGE_NAME} image to appear"
+start_end::group_start "Waiting for ${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
-push_pull_remove_images::wait_for_github_registry_image \
- "${AIRFLOW_PROD_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+push_pull_remove_images::wait_for_image "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
start_end::group_end
start_end::group_start "Pulling the PROD Image"
build_images::prepare_prod_build
-image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+image_name_with_tag="${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
verbosity::print_info "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}"
-push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
+push_pull_remove_images::pull_image_if_not_present_or_forced "${image_name_with_tag}"
start_end::group_end
-verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
+verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 0a7f6bf..ba61699 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -348,6 +348,17 @@ function build_images::print_build_info() {
verbosity::print_info
}
+# Retrieves GitHub Container Registry image prefix from repository name
+# GitHub Container Registry stores all images at the organization level, they are just
+# linked to the repository via docker label - however we assume a convention where we will
+# add repository name to organisation separated by '-' and convert everything to lowercase
+# this is because in order for it to work for internal PR for users or other organisation's
+# repositories, the other organisations and repositories can be uppercase
+# container registry image name has to be lowercase
+function build_images::get_github_container_registry_image_prefix() {
+ echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]'
+}
+
function build_images::get_docker_image_names() {
# python image version to use
export PYTHON_BASE_IMAGE_VERSION=${PYTHON_BASE_IMAGE_VERSION:=${PYTHON_MAJOR_MINOR_VERSION}}
@@ -355,95 +366,60 @@ function build_images::get_docker_image_names() {
# Python base image to use
export PYTHON_BASE_IMAGE="python:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
+ local image_name
+ image_name="${GITHUB_REGISTRY}/$(build_images::get_github_container_registry_image_prefix)"
+
# CI image base tag
export AIRFLOW_CI_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
- # CI image to build
- export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
- # Default CI image
- export AIRFLOW_PYTHON_BASE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:python${PYTHON_MAJOR_MINOR_VERSION}-${BRANCH_NAME}"
- # CI image to build
- export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_CI_BASE_TAG}"
-
- # Base production image tag - used to build kubernetes tag as well
- if [[ -z "${FORCE_AIRFLOW_PROD_BASE_TAG=}" ]]; then
- export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
- else
- export AIRFLOW_PROD_BASE_TAG="${FORCE_AIRFLOW_PROD_BASE_TAG}"
- fi
- # PROD image to build
- export AIRFLOW_PROD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}"
-
- # PROD build segment
- export AIRFLOW_PROD_BUILD_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-build"
-
- # PROD Kubernetes image to build
- export AIRFLOW_PROD_IMAGE_KUBERNETES="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_PROD_BASE_TAG}-kubernetes"
-
- # PROD default image
- export AIRFLOW_PROD_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${BRANCH_NAME}"
+ # Example:
+ # ghcr.io/apache/airflow-main-python3.8-ci-v2
+ export AIRFLOW_CI_IMAGE="${image_name}-${AIRFLOW_CI_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
# File that is touched when the CI image is built for the first time locally
export BUILT_CI_IMAGE_FLAG_FILE="${BUILD_CACHE_DIR}/${BRANCH_NAME}/.built_${PYTHON_MAJOR_MINOR_VERSION}"
- local image_name
- image_name="${GITHUB_REGISTRY}/$(get_github_container_registry_image_prefix)"
- local image_separator
- if [[ ${GITHUB_REGISTRY} == "ghcr.io" ]]; then
- image_separator="-"
- elif [[ ${GITHUB_REGISTRY} == "docker.pkg.github.com" ]]; then
- image_separator="/"
- else
- echo
- echo "${COLOR_RED}ERROR: Bad value of '${GITHUB_REGISTRY}'. Should be either 'ghcr.io' or 'docker.pkg.github.com'!${COLOR_RESET}"
- echo
- exit 1
- fi
+ # PROD image to build
+ export AIRFLOW_PROD_BASE_TAG="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}"
# Example:
- # docker.pkg.github.com/apache/airflow/main-python3.6-v2
# ghcr.io/apache/airflow-v2-1-test-python-v2:3.6-slim-buster
- # ghcr.io/apache/airflow-python-v2:3.6-slim-buster-<COMMIT_SHA>
- export GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE="${image_name}${image_separator}${AIRFLOW_PROD_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
+ export AIRFLOW_PROD_IMAGE="${image_name}-${AIRFLOW_PROD_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
+
+ # PROD Kubernetes image to build
+ export AIRFLOW_PROD_IMAGE_KUBERNETES="${AIRFLOW_PROD_IMAGE}-kubernetes"
+
# Example:
- # docker.pkg.github.com/apache/airflow/main-python3.6-build-v2
# ghcr.io/apache/airflow-main-python3.6-build-v2
- export GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE="${image_name}${image_separator}${AIRFLOW_PROD_BASE_TAG}-build${GITHUB_REGISTRY_IMAGE_SUFFIX}"
+ export AIRFLOW_PROD_BUILD_IMAGE="${image_name}-${AIRFLOW_PROD_BASE_TAG}-build${GITHUB_REGISTRY_IMAGE_SUFFIX}"
# Example:
- # docker.pkg.github.com/apache/airflow/python-v2:3.6-slim-buster
# ghcr.io/apache/airflow-python-v2:3.6-slim-buster
- # ghcr.io/apache/airflow-python-v2:3.6-slim-buster-<COMMIT_SHA>
- export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${image_name}${image_separator}python${GITHUB_REGISTRY_IMAGE_SUFFIX}:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
+ export AIRFLOW_PYTHON_BASE_IMAGE="${image_name}-python${GITHUB_REGISTRY_IMAGE_SUFFIX}:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
- # Example:
- # docker.pkg.github.com/apache/airflow/main-python3.8-ci-v2
- export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${image_name}${image_separator}${AIRFLOW_CI_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
}
# If GitHub Registry is used, login to the registry using GITHUB_USERNAME and
# GITHUB_TOKEN. In case Personal Access token is not set, skip logging in
# Also enable experimental features of docker (we need `docker manifest` command)
function build_images::configure_docker_registry() {
- if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- local token="${GITHUB_TOKEN}"
- if [[ -z "${token}" ]] ; then
- verbosity::print_info
- verbosity::print_info "Skip logging in to GitHub Registry. No Token available!"
- verbosity::print_info
- fi
- if [[ -n "${token}" ]]; then
- echo "${token}" | docker_v login \
- --username "${GITHUB_USERNAME:-apache}" \
- --password-stdin \
- "${GITHUB_REGISTRY}"
- else
- verbosity::print_info "Skip Login to GitHub Registry ${GITHUB_REGISTRY} as token is missing"
- fi
- local new_config
- new_config=$(jq '.experimental = "enabled"' "${HOME}/.docker/config.json")
- echo "${new_config}" > "${HOME}/.docker/config.json"
+ local token="${GITHUB_TOKEN}"
+ if [[ -z "${token}" ]] ; then
+ verbosity::print_info
+ verbosity::print_info "Skip logging in to GitHub Registry. No Token available!"
+ verbosity::print_info
+ fi
+ if [[ -n "${token}" ]]; then
+ echo "${token}" | docker_v login \
+ --username "${GITHUB_USERNAME:-apache}" \
+ --password-stdin \
+ "${GITHUB_REGISTRY}"
+ else
+ verbosity::print_info "Skip Login to GitHub Registry ${GITHUB_REGISTRY} as token is missing"
fi
+ local new_config
+ new_config=$(jq '.experimental = "enabled"' "${HOME}/.docker/config.json")
+ echo "${new_config}" > "${HOME}/.docker/config.json"
}
@@ -459,9 +435,6 @@ function build_images::prepare_ci_build() {
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}"
readonly AIRFLOW_EXTRAS
- export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
- readonly AIRFLOW_IMAGE
-
build_images::configure_docker_registry
sanity_checks::go_to_airflow_sources
permissions::fix_group_permissions
@@ -525,7 +498,7 @@ function build_images::rebuild_ci_image_if_needed() {
local root_files_count
root_files_count=$(find "airflow" "tests" -user root | wc -l | xargs)
if [[ ${root_files_count} != "0" ]]; then
- ./scripts/ci/tools/ci_fix_ownership.sh || true
+ ./scripts/ci/tools/fix_ownership.sh || true
fi
fi
verbosity::print_info
@@ -595,18 +568,6 @@ function build_images::rebuild_ci_image_if_needed_and_confirmed() {
fi
}
-# Retrieves GitHub Container Registry image prefix from repository name
-# GitHub Container Registry stores all images at the organization level, they are just
-# linked to the repository via docker label - however we assume a convention where we will
-# add repository name to organisation separated by '-' and convert everything to lowercase
-# this is because in order for it to work for internal PR for users or other organisation's
-# repositories, the other organisations and repositories can be uppercase
-# container registry image name has to be lowercase
-function get_github_container_registry_image_prefix() {
- echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]'
-}
-
-
# Builds CI image - depending on the caching strategy (pulled, local, disabled) it
# passes the necessary docker build flags via DOCKER_CACHE_CI_DIRECTIVE array
# it also passes the right Build args depending on the configuration of the build
@@ -716,10 +677,6 @@ Docker building ${AIRFLOW_CI_IMAGE}.
--target "main" \
. -f Dockerfile.ci
set -u
- if [[ -n "${DEFAULT_CI_IMAGE=}" ]]; then
- echo "Tagging additionally image ${AIRFLOW_CI_IMAGE} with ${DEFAULT_CI_IMAGE}"
- docker_v tag "${AIRFLOW_CI_IMAGE}" "${DEFAULT_CI_IMAGE}"
- fi
if [[ -n "${IMAGE_TAG=}" ]]; then
echo "Tagging additionally image ${AIRFLOW_CI_IMAGE} with ${IMAGE_TAG}"
docker_v tag "${AIRFLOW_CI_IMAGE}" "${IMAGE_TAG}"
@@ -769,11 +726,6 @@ function build_images::prepare_prod_build() {
"--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}"
)
fi
- if [[ "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" == "${PYTHON_MAJOR_MINOR_VERSION}" ]]; then
- export DEFAULT_CI_IMAGE="${AIRFLOW_PROD_IMAGE_DEFAULT}"
- else
- export DEFAULT_CI_IMAGE=""
- fi
export THE_IMAGE_TYPE="PROD"
export IMAGE_DESCRIPTION="Airflow production"
@@ -781,9 +733,6 @@ function build_images::prepare_prod_build() {
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_PROD_EXTRAS}"}"
readonly AIRFLOW_EXTRAS
- export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
- readonly AIRFLOW_IMAGE
-
build_images::configure_docker_registry
AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="${BRANCH_NAME}"
sanity_checks::go_to_airflow_sources
@@ -906,10 +855,6 @@ function build_images::build_prod_images() {
--target "main" \
. -f Dockerfile
set -u
- if [[ -n "${DEFAULT_PROD_IMAGE:=}" ]]; then
- echo "Tagging additionally image ${AIRFLOW_PROD_IMAGE} with ${DEFAULT_PROD_IMAGE}"
- docker_v tag "${AIRFLOW_PROD_IMAGE}" "${DEFAULT_PROD_IMAGE}"
- fi
if [[ -n "${IMAGE_TAG=}" ]]; then
echo "Tagging additionally image ${AIRFLOW_PROD_IMAGE} with ${IMAGE_TAG}"
docker_v tag "${AIRFLOW_PROD_IMAGE}" "${IMAGE_TAG}"
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 34e700d..c05258c 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -567,7 +567,6 @@ function initialization::initialize_git_variables() {
function initialization::initialize_github_variables() {
# Defaults for interacting with GitHub
export GITHUB_REGISTRY="ghcr.io"
- export USE_GITHUB_REGISTRY=${USE_GITHUB_REGISTRY:="false"}
export GITHUB_REGISTRY_IMAGE_SUFFIX=${GITHUB_REGISTRY_IMAGE_SUFFIX:="-v2"}
export GITHUB_REGISTRY_WAIT_FOR_IMAGE=${GITHUB_REGISTRY_WAIT_FOR_IMAGE:="false"}
export GITHUB_REGISTRY_PULL_IMAGE_TAG=${GITHUB_REGISTRY_PULL_IMAGE_TAG:="latest"}
@@ -729,7 +728,6 @@ Production image build variables:
Detected GitHub environment:
- USE_GITHUB_REGISTRY: '${USE_GITHUB_REGISTRY}'
GITHUB_REPOSITORY: '${GITHUB_REPOSITORY}'
GITHUB_USERNAME: '${GITHUB_USERNAME}'
GITHUB_TOKEN: '${GITHUB_TOKEN}'
@@ -874,7 +872,6 @@ function initialization::make_constants_read_only() {
readonly DOCKERHUB_REPO
readonly DOCKER_CACHE
- readonly USE_GITHUB_REGISTRY
readonly GITHUB_REGISTRY
readonly GITHUB_REGISTRY_WAIT_FOR_IMAGE
readonly GITHUB_REGISTRY_PULL_IMAGE_TAG
@@ -885,7 +882,6 @@ function initialization::make_constants_read_only() {
readonly GITHUB_USERNAME
readonly FORWARD_CREDENTIALS
- readonly USE_GITHUB_REGISTRY
readonly EXTRA_STATIC_CHECK_OPTIONS
@@ -893,15 +889,9 @@ function initialization::make_constants_read_only() {
readonly PYTHON_BASE_IMAGE_VERSION
readonly PYTHON_BASE_IMAGE
- readonly AIRFLOW_PYTHON_BASE_IMAGE
readonly AIRFLOW_CI_BASE_TAG
- readonly AIRFLOW_CI_IMAGE
- readonly AIRFLOW_CI_IMAGE_DEFAULT
readonly AIRFLOW_PROD_BASE_TAG
- readonly AIRFLOW_PROD_IMAGE
- readonly AIRFLOW_PROD_BUILD_IMAGE
readonly AIRFLOW_PROD_IMAGE_KUBERNETES
- readonly AIRFLOW_PROD_IMAGE_DEFAULT
readonly BUILT_CI_IMAGE_FLAG_FILE
readonly INIT_SCRIPT_FILE
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index f8a1d3e..ca75d68 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -84,26 +84,6 @@ ${COLOR_RESET}
fi
}
-# Pulls image if needed but tries to pull it from GitHub registry before
-# It attempts to pull it from the DockerHub registry. This is used to speed up the builds
-# In GitHub Actions and to pull appropriately tagged builds.
-# Parameters:
-# $1 -> DockerHub image to pull
-# $2 -> GitHub image to try to pull first
-function push_pull_remove_images::pull_image_github_dockerhub() {
- local dockerhub_image="${1}"
- local github_image="${2}"
-
- set +e
- if push_pull_remove_images::pull_image_if_not_present_or_forced "${github_image}"; then
- # Tag the image to be the DockerHub one
- docker_v tag "${github_image}" "${dockerhub_image}"
- else
- push_pull_remove_images::pull_image_if_not_present_or_forced "${dockerhub_image}"
- fi
- set -e
-}
-
# Rebuilds python base image from the latest available Python version
function push_pull_remove_images::rebuild_python_base_image() {
echo
@@ -121,8 +101,8 @@ function push_pull_remove_images::rebuild_python_base_image() {
# * if FORCE_PULL_BASE_PYTHON_IMAGE != false, then it rebuild the image using latest Python image available
# and adds `org.opencontainers.image.source` label to it, so that it is linked to Airflow
# repository when we push it to GHCR registry
-# * Otherwise it pulls the Python base image from either GitHub registry or from DockerHub
-# depending on USE_GITHUB_REGISTRY variable. In case we pull specific build image (via suffix)
+# * Otherwise it pulls the Python base image from either GitHub registry.
+# In case we pull specific build image (via suffix)
# it will pull the right image using the specified suffix
function push_pull_remove_images::pull_base_python_image() {
if [[ ${FORCE_PULL_BASE_PYTHON_IMAGE} == "true" ]] ; then
@@ -136,16 +116,12 @@ function push_pull_remove_images::pull_base_python_image() {
echo -n "Docker pulling base python image. Upgrade to newer deps: ${UPGRADE_TO_NEWER_DEPENDENCIES}
" > "${DETECTED_TERMINAL}"
fi
- if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- local python_tag_suffix=""
- if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then
- python_tag_suffix="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- fi
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PYTHON_BASE_IMAGE}" \
- "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${python_tag_suffix}"
- else
- docker_v pull "${AIRFLOW_PYTHON_BASE_IMAGE}" || true
+ local python_tag_suffix=""
+ if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then
+ python_tag_suffix="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
+ push_pull_remove_images::pull_image_if_not_present_or_forced \
+ "${AIRFLOW_PYTHON_BASE_IMAGE}${python_tag_suffix}"
}
# Pulls CI image in case caching strategy is "pulled" and the image needs to be pulled
@@ -157,12 +133,8 @@ function push_pull_remove_images::pull_ci_images_if_needed() {
push_pull_remove_images::pull_base_python_image
fi
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
- if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" \
- "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- else
- push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_CI_IMAGE}" || true
- fi
+ push_pull_remove_images::pull_image_if_not_present_or_forced \
+ "${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
}
@@ -176,33 +148,15 @@ function push_pull_remove_images::pull_prod_images_if_needed() {
push_pull_remove_images::pull_base_python_image
fi
if [[ "${DOCKER_CACHE}" == "pulled" ]]; then
- if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- # "Build" segment of production image
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_BUILD_IMAGE}" \
- "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- # "Main" segment of production image
- push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" \
- "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
- else
- push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_PROD_BUILD_IMAGE}"
- push_pull_remove_images::pull_image_if_not_present_or_forced "${AIRFLOW_PROD_IMAGE}"
- fi
- fi
-}
-
-# Pushes Ci images and the manifest to the registry in DockerHub.
-function push_pull_remove_images::push_ci_images_to_dockerhub() {
- push_pull_remove_images::push_image_with_retries "${AIRFLOW_PYTHON_BASE_IMAGE}"
- push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_IMAGE}"
- docker_v tag "${AIRFLOW_CI_LOCAL_MANIFEST_IMAGE}" "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
- push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_REMOTE_MANIFEST_IMAGE}"
- if [[ -n ${DEFAULT_CI_IMAGE=} ]]; then
- # Only push default image to DockerHub registry if it is defined
- push_pull_remove_images::push_image_with_retries "${DEFAULT_CI_IMAGE}"
+ # "Build" segment of production image
+ push_pull_remove_images::pull_image_if_not_present_or_forced \
+ "${AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ # "Main" segment of production image
+ push_pull_remove_images::pull_image_if_not_present_or_forced \
+ "${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
}
-
# Push image to GitHub registry with the push tag:
# "${COMMIT_SHA}" - in case of pull-request triggered 'workflow_run' builds
# "latest" - in case of push builds
@@ -215,9 +169,9 @@ function push_pull_remove_images::push_python_image_to_github() {
python_tag_suffix="-${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
fi
docker_v tag "${AIRFLOW_PYTHON_BASE_IMAGE}" \
- "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${python_tag_suffix}"
+ "${AIRFLOW_PYTHON_BASE_IMAGE}${python_tag_suffix}"
push_pull_remove_images::push_image_with_retries \
- "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}${python_tag_suffix}"
+ "${AIRFLOW_PYTHON_BASE_IMAGE}${python_tag_suffix}"
}
# Pushes Ci images and their tags to registry in GitHub
@@ -225,27 +179,17 @@ function push_pull_remove_images::push_ci_images_to_github() {
if [[ "${PUSH_PYTHON_BASE_IMAGE=}" != "false" ]]; then
push_pull_remove_images::push_python_image_to_github
fi
- local airflow_ci_tagged_image="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
+ local airflow_ci_tagged_image="${AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker_v tag "${AIRFLOW_CI_IMAGE}" "${airflow_ci_tagged_image}"
push_pull_remove_images::push_image_with_retries "${airflow_ci_tagged_image}"
if [[ -n ${GITHUB_SHA=} ]]; then
# Also push image to GitHub registry with commit SHA
- local airflow_ci_sha_image="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${COMMIT_SHA}"
+ local airflow_ci_sha_image="${AIRFLOW_CI_IMAGE}:${COMMIT_SHA}"
docker_v tag "${AIRFLOW_CI_IMAGE}" "${airflow_ci_sha_image}"
push_pull_remove_images::push_image_with_retries "${airflow_ci_sha_image}"
fi
}
-
-# Pushes Ci image and it's manifest to the registry.
-function push_pull_remove_images::push_ci_images() {
- if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- push_pull_remove_images::push_ci_images_to_github
- else
- push_pull_remove_images::push_ci_images_to_dockerhub
- fi
-}
-
# Pushes PROD image to registry in DockerHub
function push_pull_remove_images::push_prod_images_to_dockerhub () {
push_pull_remove_images::push_image_with_retries "${AIRFLOW_PYTHON_BASE_IMAGE}"
@@ -259,45 +203,30 @@ function push_pull_remove_images::push_prod_images_to_dockerhub () {
}
-# Pushes PROD image to and their tags to registry in GitHub
+# Pushes PROD image to registry in GitHub
# Push image to GitHub registry with chosen push tag
# the PUSH tag might be:
# "${COMMIT_SHA}" - in case of pull-request triggered 'workflow_run' builds
# "latest" - in case of push builds
function push_pull_remove_images::push_prod_images_to_github () {
- local airflow_prod_tagged_image="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
+ local airflow_prod_tagged_image="${AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker_v tag "${AIRFLOW_PROD_IMAGE}" "${airflow_prod_tagged_image}"
- push_pull_remove_images::push_image_with_retries "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
+ push_pull_remove_images::push_image_with_retries "${airflow_prod_tagged_image}"
if [[ -n ${COMMIT_SHA=} ]]; then
# Also push image to GitHub registry with commit SHA
- local airflow_prod_sha_image="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}"
+ local airflow_prod_sha_image="${AIRFLOW_PROD_IMAGE}:${COMMIT_SHA}"
docker_v tag "${AIRFLOW_PROD_IMAGE}" "${airflow_prod_sha_image}"
push_pull_remove_images::push_image_with_retries "${airflow_prod_sha_image}"
fi
# Also push prod build image
- local airflow_prod_build_tagged_image="${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
+ local airflow_prod_build_tagged_image="${AIRFLOW_PROD_BUILD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker_v tag "${AIRFLOW_PROD_BUILD_IMAGE}" "${airflow_prod_build_tagged_image}"
push_pull_remove_images::push_image_with_retries "${airflow_prod_build_tagged_image}"
}
-
-# Pushes PROD image to the registry. In case the image was taken from cache registry
-# it is also pushed to the cache, not to the main registry
-function push_pull_remove_images::push_prod_images() {
- if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
- push_pull_remove_images::push_prod_images_to_github
- else
- push_pull_remove_images::push_prod_images_to_dockerhub
- fi
-}
-
-
# waits for an image to be available in GitHub Container Registry. Should be run with `set +e`
-function push_pull_remove_images::check_for_image_in_github_container_registry() {
- local image_name_in_github_registry="${1}"
- local image_tag_in_github_registry=${2}
-
- local image_to_wait_for="ghcr.io/${GITHUB_REPOSITORY}-${image_name_in_github_registry}:${image_tag_in_github_registry}"
+function push_pull_remove_images::check_image_manifest() {
+ local image_to_wait_for="${1}"
echo "GitHub Container Registry: checking for ${image_to_wait_for} via docker manifest inspect!"
docker_v manifest inspect "${image_to_wait_for}"
local res=$?
@@ -311,28 +240,15 @@ function push_pull_remove_images::check_for_image_in_github_container_registry()
}
# waits for an image to be available in the GitHub registry
-function push_pull_remove_images::wait_for_github_registry_image() {
+function push_pull_remove_images::wait_for_image() {
set +e
- echo " Waiting for github registry image: " "${@}"
+ echo " Waiting for github registry image: " "$1"
while true
do
- if push_pull_remove_images::check_for_image_in_github_container_registry "${@}"; then
+ if push_pull_remove_images::check_image_manifest "$1"; then
break
fi
sleep 30
done
set -e
}
-
-function push_pull_remove_images::check_if_github_registry_wait_for_image_enabled() {
- if [[ ${USE_GITHUB_REGISTRY} != "true" || ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} != "true" ]]; then
- echo
- echo "This script should not be called"
- echo "It need both USE_GITHUB_REGISTRY and GITHUB_REGISTRY_WAIT_FOR_IMAGE to true!"
- echo
- echo "USE_GITHUB_REGISTRY = ${USE_GITHUB_REGISTRY}"
- echo "GITHUB_REGISTRY_WAIT_FOR_IMAGE =${GITHUB_REGISTRY_WAIT_FOR_IMAGE}"
- echo
- exit 1
- fi
-}
diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/tools/build_dockerhub.sh
similarity index 74%
rename from scripts/ci/images/ci_build_dockerhub.sh
rename to scripts/ci/tools/build_dockerhub.sh
index b464cf2..e1653ae 100755
--- a/scripts/ci/images/ci_build_dockerhub.sh
+++ b/scripts/ci/tools/build_dockerhub.sh
@@ -30,8 +30,6 @@ export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
export DOCKER_CACHE="local"
export FORCE_PULL_BASE_PYTHON_IMAGE="true"
export DOCKER_TAG=${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}
-# Name the image based on the TAG rather than based on the branch name
-export FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
export AIRFLOW_CONSTRAINTS_REFERENCE="constraints-${INSTALL_AIRFLOW_VERSION}"
export AIRFLOW_CONSTRAINTS="constraints"
# shellcheck source=scripts/ci/libraries/_script_init.sh
@@ -44,19 +42,20 @@ rm -rf "${AIRFLOW_SOURCES}/docker-context-files/*"
build_images::prepare_prod_build
build_images::build_prod_images
verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
+
+export RELEASE_IMAGE="apache/airflow:${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}"
echo
-echo "Pushing airflow image as apache/airflow:${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}"
+echo "Pushing airflow PROD image as ${RELEASE_IMAGE}"
echo
# Re-tag the image to be published in "apache/airflow"
-docker tag "apache/airflow-ci:${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}" \
- "apache/airflow:${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}"
-docker push "apache/airflow:${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}"
+docker tag "${AIRFLOW_PROD_IMAGE}" "${RELEASE_IMAGE}"
+docker push "${RELEASE_IMAGE}"
if [[ ${PYTHON_MAJOR_MINOR_VERSION} == "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" ]]; then
+ export DEFAULT_VERSION_IMAGE="apache/airflow:${INSTALL_AIRFLOW_VERSION}"
echo
- echo "Pushing default airflow image as apache/airflow:${INSTALL_AIRFLOW_VERSION}"
+ echo "Pushing default airflow image as ${DEFAULT_VERSION_IMAGE}"
echo
# In case of default Python version we also push ":version" tag
- docker tag "apache/airflow:${INSTALL_AIRFLOW_VERSION}-python${PYTHON_MAJOR_MINOR_VERSION}" \
- "apache/airflow:${INSTALL_AIRFLOW_VERSION}"
- docker push "apache/airflow:${INSTALL_AIRFLOW_VERSION}"
+ docker tag "${RELEASE_IMAGE}" "${DEFAULT_VERSION_IMAGE}"
+ docker push "${DEFAULT_VERSION_IMAGE}"
fi
diff --git a/scripts/ci/tools/ci_clear_tmp.sh b/scripts/ci/tools/ci_clear_tmp.sh
deleted file mode 100755
index bef3fa5..0000000
--- a/scripts/ci/tools/ci_clear_tmp.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Fixes ownership for files created inside container (files owned by root will be owned by host user)
-#
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
-declare -a EXTRA_DOCKER_FLAGS
-
-sanity_checks::sanitize_mounted_files
-
-read -r -a EXTRA_DOCKER_FLAGS <<<"$(local_mounts::convert_local_mounts_to_docker_params)"
-
-docker_v run --entrypoint /bin/bash "${EXTRA_DOCKER_FLAGS[@]}" \
- --rm \
- --env-file "${AIRFLOW_SOURCES}/scripts/ci/docker-compose/_docker.env" \
- "${AIRFLOW_CI_IMAGE}" \
- -c /opt/airflow/scripts/in_container/run_clear_tmp.sh
diff --git a/scripts/ci/tools/ci_fix_ownership.sh b/scripts/ci/tools/fix_ownership.sh
similarity index 100%
rename from scripts/ci/tools/ci_fix_ownership.sh
rename to scripts/ci/tools/fix_ownership.sh
diff --git a/scripts/ci/tools/ci_free_space_on_ci.sh b/scripts/ci/tools/free_space.sh
similarity index 100%
rename from scripts/ci/tools/ci_free_space_on_ci.sh
rename to scripts/ci/tools/free_space.sh
diff --git a/scripts/ci/tools/prepare_prod_docker_images.sh b/scripts/ci/tools/prepare_prod_docker_images.sh
index ad6cc95..928f282 100755
--- a/scripts/ci/tools/prepare_prod_docker_images.sh
+++ b/scripts/ci/tools/prepare_prod_docker_images.sh
@@ -41,5 +41,5 @@ export INSTALL_AIRFLOW_VERSION="${1}"
for python_version in "3.6" "3.7" "3.8" "3.9"
do
export PYTHON_MAJOR_MINOR_VERSION=${python_version}
- "${AIRFLOW_SOURCES_DIR}/scripts/ci/images/ci_build_dockerhub.sh"
+ "${AIRFLOW_SOURCES_DIR}/scripts/ci/tools/build_dockerhub.sh"
done
diff --git a/scripts/in_container/run_clear_tmp.sh b/scripts/in_container/run_clear_tmp.sh
deleted file mode 100755
index a94923c..0000000
--- a/scripts/in_container/run_clear_tmp.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-
-in_container_clear_tmp