[airflow] branch main updated: Speed-up Python Helm Unit and Docker Compose tests by ~30% (#25583)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 1f3a97baa6 Speed-up Python Helm Unit and Docker Compose tests by ~30% (#25583)
1f3a97baa6 is described below

commit 1f3a97baa6332d690144dc0cf9868f12cbae67f1
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Mon Aug 8 07:53:43 2022 +0200

    Speed-up Python Helm Unit and Docker Compose tests by ~30% (#25583)
    
    The `pytest -n auto` switch does not give the accurate maximum number
    of parallel tests we can run. It uses the number of CPUs not the number
    of cores.
    
    The Helm tests utilise ~70% of available CPUs when the CPUS have dual cores,
    so we shoudl be able to gain 5 minutes (from 22 to 17) when Helm
    tests are run on self-hosted runners, and even more ~12 minutes
    (32 m instead of 45m) when run on Public Runner.
    
    The gains on docker-compose tests have not been measured yet but they are
    likely similar.
---
 Dockerfile.ci                                    | 4 +++-
 dev/breeze/src/airflow_breeze/utils/run_tests.py | 4 ++--
 scripts/docker/entrypoint_ci.sh                  | 4 +++-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/Dockerfile.ci b/Dockerfile.ci
index b57b726af6..ad6c0e7abf 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -775,9 +775,11 @@ EXTRA_PYTEST_ARGS=(
 )
 
 if [[ "${TEST_TYPE}" == "Helm" ]]; then
+    _cpus="$(grep -c 'cpu[0-9]' /proc/stat)"
+    echo "Running tests with ${_cpus} CPUs in parallel"
     # Enable parallelism
     EXTRA_PYTEST_ARGS+=(
-        "-n" "auto"
+        "-n" "${_cpus}"
     )
 else
     EXTRA_PYTEST_ARGS+=(
diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py
index 6cda9efa1e..315dc921da 100644
--- a/dev/breeze/src/airflow_breeze/utils/run_tests.py
+++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py
@@ -36,7 +36,7 @@ def verify_an_image(
             f"[error]Error when inspecting {image_type} image: {command_result.returncode}[/]"
         )
         return command_result.returncode, f"Testing {image_type} python {image_name}"
-    pytest_args = ("-n", "auto", "--color=yes")
+    pytest_args = ("-n", str(os.cpu_count()), "--color=yes")
     if image_type == 'PROD':
         test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" / "test_prod_image.py"
     else:
@@ -64,7 +64,7 @@ def run_docker_compose_tests(
     if command_result.returncode != 0:
         get_console().print(f"[error]Error when inspecting PROD image: {command_result.returncode}[/]")
         return command_result.returncode, f"Testing docker-compose python with {image_name}"
-    pytest_args = ("-n", "auto", "--color=yes")
+    pytest_args = ("-n", str(os.cpu_count()), "--color=yes")
     test_path = AIRFLOW_SOURCES_ROOT / "docker_tests" / "test_docker_compose_quick_start.py"
     env = os.environ.copy()
     env['DOCKER_IMAGE'] = image_name
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index 175c3ad8fd..5bc6d480da 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -272,9 +272,11 @@ EXTRA_PYTEST_ARGS=(
 )
 
 if [[ "${TEST_TYPE}" == "Helm" ]]; then
+    _cpus="$(grep -c 'cpu[0-9]' /proc/stat)"
+    echo "Running tests with ${_cpus} CPUs in parallel"
     # Enable parallelism
     EXTRA_PYTEST_ARGS+=(
-        "-n" "auto"
+        "-n" "${_cpus}"
     )
 else
     EXTRA_PYTEST_ARGS+=(