You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/21 14:28:22 UTC
[airflow] 19/22: Speed up Kubernetes upgrade tests at least 2x (#25159)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e57d2fdbd3c6e4335ef2314ee012794b41c237e2
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Jul 19 22:30:42 2022 +0200
Speed up Kubernetes upgrade tests at least 2x (#25159)
We do not need to run upgrade kubernetes tests for all python
versions and all kubernetes versions. It is enough to run them
for the minimum and maximum versions. We already run all k8s tests
in main for all executors, so they "generally" work, and it
is very, very unlikely that if it works for Python 3.7, and 3.10,
it will not work for 3.8 or 3.9 (similarly for K8S version).
This will speed up the build at least 2 times.
(cherry picked from commit 907e720b9e312e9da1e645973a747ddbd3ba69f5)
---
.github/workflows/ci.yml | 23 +++++++++++++---------
.../src/airflow_breeze/utils/selective_checks.py | 16 +++++++++++++++
2 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cd056e4238..c459b7001d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -134,8 +134,13 @@ jobs:
all-python-versions-list-as-string: >-
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }}
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }}
+ min-max-python-versions-as-string: >-
+ ${{ steps.selective-checks.outputs.min-max-python-versions-as-string }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }}
+ default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }}
+ min-max-kubernetes-versions-as-string: >-
+ ${{ steps.selective-checks.outputs.min-max-kubernetes-versions-as-string }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
@@ -1524,7 +1529,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
tests-helm-executor-upgrade:
timeout-minutes: 150
- name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
needs: [build-info, wait-for-prod-images]
env:
@@ -1536,10 +1541,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
EXECUTOR: "KubernetesExecutor"
KIND_VERSION: "${{ needs.build-info.outputs.default-kind-version }}"
HELM_VERSION: "${{ needs.build-info.outputs.default-helm-version }}"
- CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.python-versions-list-as-string}}
- CURRENT_KUBERNETES_VERSIONS_AS_STRING: >
- ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
+ CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: >-
+ ${{needs.build-info.outputs.min-max-python-versions-as-string}}
+ CURRENT_KUBERNETES_VERSIONS_AS_STRING: >-
+ ${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
if: >
needs.build-info.outputs.run-kubernetes-tests == 'true' &&
needs.build-info.outputs.default-branch == 'main'
@@ -1569,12 +1574,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
with:
path: ".build/.kubernetes_venv"
key: "kubernetes-${{ needs.build-info.outputs.default-python-version }}\
- -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
- -${{needs.build-info.outputs.python-versions-list-as-string}}
+ -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}}
+ -${{needs.build-info.outputs.min-max-python-versions-as-string}}
-${{ hashFiles('setup.py','setup.cfg') }}"
restore-keys: "kubernetes-${{ needs.build-info.outputs.default-python-version }}-\
- -${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
- -${{needs.build-info.outputs.python-versions-list-as-string}}"
+ -${{needs.build-info.outputs.min-max-kubernetes-versions-as-string}} \
+ -${{needs.build-info.outputs.min-max-python-versions-as-string}}"
- name: "Cache bin folder with tools for kubernetes testing"
uses: actions/cache@v3
with:
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 642bb75043..af5b7a154d 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -330,6 +330,14 @@ class SelectiveChecks:
def python_versions_list_as_string(self) -> str:
return " ".join(self.python_versions)
+ @cached_property
+ def min_max_python_versions_as_string(self) -> str:
+ return " ".join(
+ [CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[0], CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[-1]]
+ if self._full_tests_needed
+ else [DEFAULT_PYTHON_MAJOR_MINOR_VERSION]
+ )
+
@cached_property
def all_python_versions(self) -> list[str]:
return (
@@ -386,6 +394,14 @@ class SelectiveChecks:
def kubernetes_versions(self) -> list[str]:
return CURRENT_KUBERNETES_VERSIONS if self._full_tests_needed else [DEFAULT_KUBERNETES_VERSION]
+ @cached_property
+ def min_max_kubernetes_versions_as_string(self) -> str:
+ return " ".join(
+ [CURRENT_KUBERNETES_VERSIONS[0], CURRENT_KUBERNETES_VERSIONS[-1]]
+ if self._full_tests_needed
+ else [DEFAULT_KUBERNETES_VERSION]
+ )
+
@cached_property
def kubernetes_versions_list_as_string(self) -> str:
return " ".join(self.kubernetes_versions)