You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/07/18 11:20:31 UTC
[airflow] 14/29: Avoid verification of images multiple times
(#16928)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7c1ce3a83c15ea58b4318a9c963f853ffb937683
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Jul 11 13:27:29 2021 +0200
Avoid verification of images multiple times (#16928)
The CI and PROD images are verified in CI build, and it takes
a minute or so, however they were verified multiple times.
Some time ago verification was added to be run then "wait for images"
job was run, but the same script has been run in every test. This is
not needed because this is the exact same image downloaded from
GitHub registry (identified by commit hash), so the extra
verificatoins are not needed. This will speed up the builds by
few percents.
(cherry picked from commit 7354d38cba9fbba3c22981012633d86a48941f92)
---
.github/workflows/ci.yml | 17 ++++++++++++-----
scripts/ci/images/ci_wait_for_and_verify_ci_image.sh | 10 ++++------
scripts/ci/images/ci_wait_for_and_verify_prod_image.sh | 4 +++-
3 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6763886..609b1c4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -284,16 +284,16 @@ jobs:
Wait for CI images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
id: wait-for-images
- env:
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.pythonVersionsListAsString}}
# We wait for the images to be available either from the build-ci-image step or from
# "build-images-workflow-run.yml' run as pull_request_target.
# We are utilising single job to wait for all images because this job merely waits
# for the images to be available.
# The test jobs wait for it to complete if WAIT_FOR_IMAGE is 'true'!
run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
-
+ env:
+ CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
+ ${{needs.build-info.outputs.pythonVersionsListAsString}}
+ VERIFY_IMAGE: "true"
static-checks:
timeout-minutes: 30
@@ -889,10 +889,11 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
# For the images to be available. The test jobs wait for it to complete!
#
id: wait-for-images
+ run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
env:
CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
${{needs.build-info.outputs.pythonVersionsListAsString}}
- run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
+ VERIFY_IMAGE: "true"
tests-kubernetes:
timeout-minutes: 50
@@ -931,6 +932,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Get all PROD images"
run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
+ env:
+ VERIFY_IMAGE: "false"
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v2
with:
@@ -994,6 +997,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Get all PROD images"
run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh
+ env:
+ VERIFY_IMAGE: "false"
- name: "Cache virtualenv for kubernetes testing"
uses: actions/cache@v2
with:
@@ -1165,6 +1170,8 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
Wait for CI images
${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}
run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh
+ env:
+ VERIFY_IMAGE: "false"
- name: "Generate constraints with PyPI providers"
run: ./scripts/ci/constraints/ci_generate_all_constraints.sh
env:
diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
index b86a8f0..9047665 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
+++ b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh
@@ -45,14 +45,12 @@ start_end::group_end
export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci"
start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear"
-
push_pull_remove_images::wait_for_github_registry_image \
"${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}"
-
build_images::prepare_ci_build
-
pull_ci_image
-
-verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
-
start_end::group_end
+
+if [[ ${VERIFY_IMAGE=} != "false" ]]; then
+ verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}"
+fi
diff --git a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
index 6215147..92dbf2b 100755
--- a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
+++ b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh
@@ -49,4 +49,6 @@ verbosity::print_info "Pulling the ${image_name_with_tag} image and tagging with
push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}"
start_end::group_end
-verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
+if [[ ${VERIFY_IMAGE=} != "false" ]]; then
+ verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}"
+fi