You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/19 20:30:52 UTC
[airflow] branch main updated: Speed up Kubernetes upgrade tests at least 2x (#25159)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 907e720b9e Speed up Kubernetes upgrade tests at least 2x (#25159)
907e720b9e is described below
commit 907e720b9e312e9da1e645973a747ddbd3ba69f5
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Jul 19 22:30:42 2022 +0200
Speed up Kubernetes upgrade tests at least 2x (#25159)
We do not need to run upgrade kubernetes tests for all python
versions and all kubernetes versions. It is enough to run them
for the minimum and maximum versions. We already run all k8s tests
in main for all executors, so they "generally" work, and it
is very, very unlikely that if it works for Python 3.7, and 3.10,
it will not work for 3.8 or 3.9 (similarly for K8S version).
This will speed up the build at least 2 times.
---
.github/workflows/ci.yml | 23 +++++++++++++---------
.../src/airflow_breeze/utils/selective_checks.py | 16 +++++++++++++++
2 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ceb0c15c93..14fa44148e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -134,8 +134,13 @@ jobs:
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
+ min-max-python-versions-as-string: >-
+ ${{ steps.selective-checks.outputs.min-max-python-versions-as-string }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
+ default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
+ min-max-kubernetes-versions-as-string: >-
+ ${{ steps.selective-checks.outputs.min-max-kubernetes-versions-as-string }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
@@ -1524,7 +1529,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-helm-executor-upgrade:
timeout-minutes: 150
- name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
env:
@@ -1536,10 +1541,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
EXECUTOR: "KubernetesExecutor"
KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.python-versions-list-as-string}}
- CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >-
+ ${{needs.build-info.outputs.min-max-python-versions-as-string}}
+ CURRENT_KUBERNETES_VERSIONS_AS_STRING: >-
+ ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
if: >
needs.build-info.outputs.run-kubernetes-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -1569,12 +1574,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
path: ".build/.kubernetes_venv"
key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\
- -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
- -${{needs.build-info.outputs.python-versions-list-as-string}}
+ -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
+ -${{needs.build-info.outputs.min-max-python-versions-as-string}}
-${{ hashFiles('setup.py','setup.cfg') }}"
restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\
- -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
- -${{needs.build-info.outputs.python-versions-list-as-string}}"
+ -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} \
+ -${{needs.build-info.outputs.min-max-python-versions-as-string}}"
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index ba0932badc..6d4c9cb351 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -324,6 +324,14 @@ class SelectiveChecks:
def python_versions_list_as_string(self) -> str:
return " ".join(self.python_versions)
+ @cached_property
+ def min_max_python_versions_as_string(self) -> str:
+ return " ".join(
+ [CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[0], CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[-1]]
+ if self._full_tests_needed
+ else [DEFAULT_PYTHON_MAJOR_MINOR_VERSION]
+ )
+
@cached_property
def all_python_versions(self) -> list[str]:
return (
@@ -380,6 +388,14 @@ class SelectiveChecks:
def kubernetes_versions(self) -> list[str]:
return CURRENT_KUBERNETES_VERSIONS if self._full_tests_needed else [DEFAULT_KUBERNETES_VERSION]
+ @cached_property
+ def min_max_kubernetes_versions_as_string(self) -> str:
+ return " ".join(
+ [CURRENT_KUBERNETES_VERSIONS[0], CURRENT_KUBERNETES_VERSIONS[-1]]
+ if self._full_tests_needed
+ else [DEFAULT_KUBERNETES_VERSION]
+ )
+
@cached_property
def kubernetes_versions_list_as_string(self) -> str:
return " ".join(self.kubernetes_versions)