You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/08/22 19:36:34 UTC
[airflow] branch master updated: Optimise production image building
during k8s tests on CI (#10476)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 93ba98c Optimise production image building during k8s tests on CI (#10476)
93ba98c is described below
commit 93ba98ce92a4689f3b5f5f96bac3f20ce5ab4b83
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Aug 22 21:35:55 2020 +0200
Optimise production image building during k8s tests on CI (#10476)
We do not have to rebuild PROD images now because we changed
the strategy of preparing the image for k8s tests instead of
embedding dags during build with EMBEDDED_DAGS build arg we
are now extending the image with FROM: clause and add dags
on top of the PROD base image. We've been rebuilding the
image twice during each k8s run - once in Prepare PROD image
and once in "Deploy airflow to cluster" both are not needed
and both lasted ~ 2m 30s, so we should save around 5m for every
K8S jobi (~30% as the while K8S test job is around 15m).
---
.github/workflows/ci.yml | 3 +++
breeze | 2 ++
scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 3 ++-
scripts/ci/libraries/_build_images.sh | 9 +++++++++
4 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 92d09bb..dc07760 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -585,6 +585,9 @@ jobs:
run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- name: "Deploy airflow to cluster"
run: ./scripts/ci/kubernetes/ci_deploy_app_to_kubernetes.sh
+ env:
+ # We have the right image pulled already by the previous step
+ SKIP_BUILDING_PROD_IMAGE: "true"
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v2
env:
diff --git a/breeze b/breeze
index 10020ed..1e4a939 100755
--- a/breeze
+++ b/breeze
@@ -723,6 +723,7 @@ function parse_arguments() {
echo "Skips checking image for rebuilds"
echo
export CHECK_IMAGE_FOR_REBUILD="false"
+ export SKIP_BUILDING_PROD_IMAGE="true"
shift ;;
-L|--build-cache-local)
echo "Use local cache to build images"
@@ -816,6 +817,7 @@ function parse_arguments() {
export GITHUB_REGISTRY_PULL_IMAGE_TAG="${2}"
export GITHUB_REGISTRY_PUSH_IMAGE_TAG="${2}"
export CHECK_IMAGE_FOR_REBUILD="false"
+ export SKIP_BUILDING_PROD_IMAGE="true"
export MOUNT_LOCAL_SOURCES="false"
export SKIP_CHECK_REMOTE_IMAGE="true"
shift 2;;
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index 1a324ab..ec4b7f5 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -37,9 +37,10 @@ function build_prod_images_on_ci() {
wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \
":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_BUILD_IMAGE}"
+ else
+ build_prod_images
fi
- build_prod_images
# Disable force pulling forced above this is needed for the subsequent scripts so that
# They do not try to pull/build images again
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 66467fc..7dd4ca8 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -637,6 +637,15 @@ function prepare_prod_build() {
# selected by Breeze flags or environment variables.
function build_prod_images() {
print_build_info
+
+ if [[ ${SKIP_BUILDING_PROD_IMAGE:="false"} == "true" ]]; then
+ print_info
+ print_info "Skip building production image. Assume the one we have is good!"
+ print_info
+ return
+ fi
+
+
pull_prod_images_if_needed
if [[ "${DOCKER_CACHE}" == "disabled" ]]; then