You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/03/13 18:39:03 UTC
[airflow] branch master updated: Fixes force-pulling base python
images (#14736)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 61b4482 Fixes force-pulling base python images (#14736)
61b4482 is described below
commit 61b448221d65aae80c89b3a84376b6201319ff33
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 13 19:38:49 2021 +0100
Fixes force-pulling base python images (#14736)
Sometimes base python image patchlevel might case failure of tests.
This happens for example with test_views.py tests fixed in #14719
where CVE fix in all python versions caused our tests to fail.
There was an error in our scripts - when --force-pull-images
were used, the base python version was not updated to the
latest version even if there was a newer one and it caused our
images to bounce few times between two latest patchlevels
when they were manually refreshed.
This change fixes it so that the base python image is always used
when
a) FORCE_PULL_IMAGES is true or
b) UPGRADE_TO_NEWER_DEPENDENCIES is != false
This will cause python upgrade in two cases:
- when images are rebuilt with --force-pull-images locally
- when images are upgraded with newer dependencies on master
---
.github/workflows/build-images-workflow-run.yml | 2 +-
.github/workflows/ci.yml | 2 +-
.github/workflows/scheduled_quarantined.yml | 2 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 27 ++++++++++++------------
4 files changed, 17 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/build-images-workflow-run.yml b/.github/workflows/build-images-workflow-run.yml
index 33c7a79..65ba7aa 100644
--- a/.github/workflows/build-images-workflow-run.yml
+++ b/.github/workflows/build-images-workflow-run.yml
@@ -24,7 +24,7 @@ on: # yamllint disable-line rule:truthy
env:
MOUNT_SELECTED_LOCAL_SOURCES: "false"
FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "true"
+ FORCE_PULL_IMAGES: "false"
CHECK_IMAGE_FOR_REBUILD: "true"
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c5e115b..c0523d6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -28,7 +28,7 @@ on: # yamllint disable-line rule:truthy
env:
MOUNT_SELECTED_LOCAL_SOURCES: "false"
FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "true"
+ FORCE_PULL_IMAGES: "false"
CHECK_IMAGE_FOR_REBUILD: "true"
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
diff --git a/.github/workflows/scheduled_quarantined.yml b/.github/workflows/scheduled_quarantined.yml
index 00663cb..2f7f05f 100644
--- a/.github/workflows/scheduled_quarantined.yml
+++ b/.github/workflows/scheduled_quarantined.yml
@@ -25,7 +25,7 @@ on: # yamllint disable-line rule:truthy
env:
MOUNT_SELECTED_LOCAL_SOURCES: "false"
FORCE_ANSWER_TO_QUESTIONS: "yes"
- FORCE_PULL_IMAGES: "true"
+ FORCE_PULL_IMAGES: "false"
CHECK_IMAGE_FOR_REBUILD: "true"
SKIP_CHECK_REMOTE_IMAGE: "true"
DB_RESET: "true"
diff --git a/scripts/ci/libraries/_push_pull_remove_images.sh b/scripts/ci/libraries/_push_pull_remove_images.sh
index c48491d..5090c5b 100644
--- a/scripts/ci/libraries/_push_pull_remove_images.sh
+++ b/scripts/ci/libraries/_push_pull_remove_images.sh
@@ -104,33 +104,34 @@ function push_pull_remove_images::pull_image_github_dockerhub() {
set -e
}
+# Force pulls the python base image
+function push_pull_remove_images::force_pull_python_base_image() {
+ docker pull "${PYTHON_BASE_IMAGE}"
+ echo "FROM ${PYTHON_BASE_IMAGE}" | \
+ docker build \
+ --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
+ -t "${AIRFLOW_PYTHON_BASE_IMAGE}" -
+}
+
# Pulls the base Python image. This image is used as base for CI and PROD images, depending on the parameters used:
#
-# * if UPGRADE_TO_NEWER_DEPENDENCIES is noy false, then it pulls the latest Python image available first and
+# * if FORCE_PULL_IMAGES is true or UPGRADE_TO_NEWER_DEPENDENCIES != false, then it pulls the latest Python image available first and
# adds `org.opencontainers.image.source` label to it, so that it is linked to Airflow repository when
# we push it to GHCR registry
-# * If UPGRADE_TO_NEWER_DEPENDENCIES it pulls the Python base image from either GitHub registry or from DockerHub
+# * Otherwise it pulls the Python base image from either GitHub registry or from DockerHub
# depending on USE_GITHUB_REGISTRY variable. In case we pull specific build image (via suffix)
# it will pull the right image using the specified suffix
function push_pull_remove_images::pull_base_python_image() {
echo
- echo "Force pull base image ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies: ${UPGRADE_TO_NEWER_DEPENDENCIES}"
+ echo "Force pull python base image ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies: ${UPGRADE_TO_NEWER_DEPENDENCIES}"
echo
if [[ -n ${DETECTED_TERMINAL=} ]]; then
echo -n "
Docker pulling ${AIRFLOW_PYTHON_BASE_IMAGE}. Upgrade to newer dependencies ${UPGRADE_TO_NEWER_DEPENDENCIES}
" > "${DETECTED_TERMINAL}"
fi
- if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
- # Pull latest PYTHON_BASE_IMAGE, so that it is linked to the current project it is build in.
- # This is necessary in case we use Google Container registry - we always use the
- # Airflow version of the python image with the opencontainers label, so that GHCR can link it
- # to the Apache Airflow repository.
- docker pull "${PYTHON_BASE_IMAGE}"
- echo "FROM ${PYTHON_BASE_IMAGE}" | \
- docker build \
- --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \
- -t "${AIRFLOW_PYTHON_BASE_IMAGE}" -
+ if [[ "${FORCE_PULL_IMAGES}" == "true" || ${UPGRADE_TO_NEWER_DEPENDENCIES} != "false" ]]; then
+ push_pull_remove_images::force_pull_python_base_image
else
if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then
PYTHON_TAG_SUFFIX=""