[airflow] 16/22: Speed up Kubernetes tests ~30% on main. (#25143)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 4fe1dabd30593e3c10750f3ba66b21e244ada4c6
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Jul 19 13:37:00 2022 +0200

    Speed up Kubernetes tests ~30% on main. (#25143)
    
    We should have enough resources - we were limiting them to 2 parallel
    runs (we assumed 4 CPUs are needed for full K8S tests). However
    monitoring shows that ~20/30% memory and up to 4 CPUs are used
    when 2 tests are running in parallel (on a big instance) but the tests
    have also some peaks that might cause more than 3 parallel tests to fail.
    
    We currently run 5 tests so increasing the limit to 3 for self-hosted
    runners should actually speed it up quite a bit (generally speaking we
    had to run 2 tests in parallel twice and 5th test was run as standalone.
    By increasing the limit to 3 we switch to 3x, 2x mode - which not only
    will safe 30% of the time, but also has a free "slot" for one more
    K8S version.
    
    (cherry picked from commit b97d9b068368c76c52abdfbf972af5b7c93ea314)
---
 .github/workflows/ci.yml          |  4 ++--
 scripts/ci/libraries/_parallel.sh | 14 ++++----------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 71514b8245..86666cf17a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1454,7 +1454,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
 
   tests-kubernetes:
     timeout-minutes: 240
-    name: Helm Chart; ${{matrix.executor}}
+    name: Helm Chart; ${{matrix.executor}} - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
     runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
     needs: [build-info, wait-for-prod-images]
     strategy:
@@ -1524,7 +1524,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
 
   tests-helm-executor-upgrade:
     timeout-minutes: 150
-    name: Helm Chart Executor Upgrade
+    name: Helm Chart Executor Upgrade - ${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
     runs-on: ${{ fromJson(needs.build-info.outputs.runs-on) }}
     needs: [build-info, wait-for-prod-images]
     env:
diff --git a/scripts/ci/libraries/_parallel.sh b/scripts/ci/libraries/_parallel.sh
index c09ca656c7..e3bf5acb0e 100644
--- a/scripts/ci/libraries/_parallel.sh
+++ b/scripts/ci/libraries/_parallel.sh
@@ -252,16 +252,10 @@ function parallel::get_maximum_parallel_k8s_jobs() {
         echo
         export MAX_PARALLEL_K8S_JOBS="1"
     else
-        if [[ ${MAX_PARALLEL_K8S_JOBS=} != "" ]]; then
-            echo
-            echo "${COLOR_YELLOW}Maximum parallel k8s jobs forced vi MAX_PARALLEL_K8S_JOBS = ${MAX_PARALLEL_K8S_JOBS}${COLOR_RESET}"
-            echo
-        else
-            MAX_PARALLEL_K8S_JOBS=$((CPUS_AVAILABLE_FOR_DOCKER / 4))
-            echo
-            echo "${COLOR_YELLOW}Maximum parallel k8s jobs set to number of CPUs available for Docker = ${MAX_PARALLEL_K8S_JOBS}${COLOR_RESET}"
-            echo
-        fi
+        echo
+        echo "${COLOR_YELLOW}This is a Self-Hosted runner - forcing max parallel jobs to 5${COLOR_RESET}"
+        echo
+        export MAX_PARALLEL_K8S_JOBS="3"
     fi
     export MAX_PARALLEL_K8S_JOBS
 }