You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2021/04/15 21:35:21 UTC
[airflow] branch v2-0-test updated (39593f9 -> 20d4c38)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a change to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from 39593f9 Fix "leaking" log driver in tests
new 2221e71 Avoids error on pushing PROD image as cache (#15321)
new 20d4c38 Test!
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 13 +++++++++++++
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 2 +-
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 17 +++++++++++++++++
scripts/ci/libraries/_build_images.sh | 1 -
scripts/ci/libraries/_push_pull_remove_images.sh | 5 +++--
5 files changed, 34 insertions(+), 4 deletions(-)
[airflow] 02/02: Test!
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 20d4c38a0310798a2f066df2a16c164a57c1e2a6
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Thu Apr 15 22:33:41 2021 +0100
Test!
---
.github/workflows/ci.yml | 13 +++++++++++++
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 10 ++++++----
scripts/ci/libraries/_build_images.sh | 1 -
scripts/ci/libraries/_push_pull_remove_images.sh | 5 +++--
4 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index edd499c..11cf9c1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1053,6 +1053,16 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
python-version: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
+ - name: Set push-python-image
+ id: push-python-image
+ run: |
+ if [[ "${REF}" == 'refs/head/master' || "${REF}" == 'refs/head/main' ]]; then
+ echo "::set-output name=wanted::true"
+ else
+ echo "::set-output name=wanted::false"
+ fi
+ env:
+ REF: ${{ github.ref }}
- name:
"Prepare PROD image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -1060,8 +1070,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
# Since we are going to push both final image and build image segment, we need to pull the
# build image, in case we are pulling from registry rather than building.
WAIT_FOR_PROD_BUILD_IMAGE: "true"
+ WAIT_FOR_PYTHON_BASE_IMAGE: ${{ steps.push-python-image.outputs.wanted}}
- name: "Push PROD images ${{ matrix.python-version }}:${{ env.GITHUB_REGISTRY_PUSH_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_push_production_images.sh
+ env:
+ PUSH_PYTHON_BASE_IMAGE: ${{ steps.push-python-image.outputs.wanted}}
push-ci-images-to-github-registry:
timeout-minutes: 10
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index 52ec2c6..9b088e1 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -41,10 +41,12 @@ function build_prod_images_on_ci() {
python_tag_suffix="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
fi
- # first we pull base python image. We will need it to re-push it after master build
- # Becoming the new "latest" image for other builds
- build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
- "${python_tag_suffix}" "${AIRFLOW_PYTHON_BASE_IMAGE}"
+ if [[ "${WAIT_FOR_PYTHON_BASE_IMAGE=}" == "true" ]]; then
+ # first we pull base python image. We will need it to re-push it after master build
+ # Becoming the new "latest" image for other builds
+ build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
+ "${python_tag_suffix}" "${AIRFLOW_PYTHON_BASE_IMAGE}"
+ fi
# And then the actual image
build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index edf9e29..da015b9 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -453,7 +453,6 @@ function build_images::get_docker_image_names() {
export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${image_name}${image_separator}python${GITHUB_REGISTRY_IMAGE_SUFFIX}:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
export GITHUB_REGISTRY_AIRFLOW_CI_IMAGE="${image_name}${image_separator}${AIRFLOW_CI_BASE_TAG}${GITHUB_REGISTRY_IMAGE_SUFFIX}"
- export GITHUB_REGISTRY_PYTHON_BASE_IMAGE="${image_name}${image_separator}python${GITHUB_REGISTRY_IMAGE_SUFFIX}:${PYTHON_BASE_IMAGE_VERSION}-slim-buster"
}
# If GitHub Registry is used, login to the registry using GITHUB_USERNAME and
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index b0723e4..5b16240 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -222,7 +222,9 @@ function push_pull_remove_images::push_python_image_to_github() {
# Pushes Ci images and their tags to registry in GitHub
function push_pull_remove_images::push_ci_images_to_github() {
- push_pull_remove_images::push_python_image_to_github
+ if [[ "${PUSH_PYTHON_BASE_IMAGE=}" != "false" ]]; then
+ push_pull_remove_images::push_python_image_to_github
+ fi
AIRFLOW_CI_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker_v tag "${AIRFLOW_CI_IMAGE}" "${AIRFLOW_CI_TAGGED_IMAGE}"
push_pull_remove_images::push_image_with_retries "${AIRFLOW_CI_TAGGED_IMAGE}"
@@ -263,7 +265,6 @@ function push_pull_remove_images::push_prod_images_to_dockerhub () {
# "${GITHUB_RUN_ID}" - in case of pull-request triggered 'workflow_run' builds
# "latest" - in case of push builds
function push_pull_remove_images::push_prod_images_to_github () {
- push_pull_remove_images::push_python_image_to_github
AIRFLOW_PROD_TAGGED_IMAGE="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
docker_v tag "${AIRFLOW_PROD_IMAGE}" "${AIRFLOW_PROD_TAGGED_IMAGE}"
push_pull_remove_images::push_image_with_retries "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PUSH_IMAGE_TAG}"
[airflow] 01/02: Avoids error on pushing PROD image as cache
(#15321)
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2221e71e97c59debae6c5c7ca2aab856e307d458
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Apr 11 22:49:48 2021 +0200
Avoids error on pushing PROD image as cache (#15321)
After stabilizing the builds on CI, the master builds started
to finally get green - except pushing to prod image cache which
continuous to fail as it missed python image to push.
This PR fixes it.
(cherry picked from commit 1dfbb8d2031cb8a3e95e4bf91aa478857c5c3a85)
---
scripts/ci/images/ci_prepare_ci_image_on_ci.sh | 2 +-
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 15 +++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
index 8dd015f..ed76b48 100755
--- a/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_ci_image_on_ci.sh
@@ -43,7 +43,7 @@ function build_ci_image_on_ci() {
build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
"${python_tag_suffix}" "${AIRFLOW_PYTHON_BASE_IMAGE}"
- # And then the base image
+ # And then the actual image
build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}" \
":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_CI_IMAGE}"
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index 516cc06..52ec2c6 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -33,9 +33,24 @@ function build_prod_images_on_ci() {
build_images::prepare_prod_build
if [[ ${USE_GITHUB_REGISTRY} == "true" && ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
+ # Tries to wait for the images indefinitely
+ # skips further image checks - since we already have the target image
+
+ local python_tag_suffix=""
+ if [[ ${GITHUB_REGISTRY_PULL_IMAGE_TAG} != "latest" ]]; then
+ python_tag_suffix="-${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
+ fi
+
+ # first we pull base python image. We will need it to re-push it after master build
+ # Becoming the new "latest" image for other builds
+ build_images::wait_for_image_tag "${GITHUB_REGISTRY_PYTHON_BASE_IMAGE}" \
+ "${python_tag_suffix}" "${AIRFLOW_PYTHON_BASE_IMAGE}"
+
+ # And then the actual image
build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_IMAGE}"
+ # And the prod build image
if [[ "${WAIT_FOR_PROD_BUILD_IMAGE=}" == "true" ]]; then
# If specified in variable - also waits for the build image
build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \