[airflow] 04/16: Add timeout to test jobs to prevent hanging docker containers (#15078)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 7a5c26cdde2dcd58378ff133ac7411e5a84aef55
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Mar 31 01:01:47 2021 +0200

    Add timeout to test jobs to prevent hanging docker containers (#15078)
    
    Some of the test jobs are hanging - either becasue of some
    weird race conditions in docker or because the test hangs (happens
    for quarantined tests). This change add maximum timeout we let
    the test suite execute to 25 minutes.
    
    (cherry picked from commit a4aee3f1d0c27f7c6010e784611ee943009c7498)
---
 scripts/ci/testing/ci_run_airflow_testing.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/ci/testing/ci_run_airflow_testing.sh b/scripts/ci/testing/ci_run_airflow_testing.sh
index 8286874..0867e3c 100755
--- a/scripts/ci/testing/ci_run_airflow_testing.sh
+++ b/scripts/ci/testing/ci_run_airflow_testing.sh
@@ -160,9 +160,10 @@ function run_test_types_in_parallel() {
         mkdir -p "${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE}"
         export JOB_LOG="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE}/stdout"
         export PARALLEL_JOB_STATUS="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${TEST_TYPE}/status"
+        # Each test job will get SIGTERM followed by SIGTERM 200ms later and SIGKILL 200ms later after 25 mins
         # shellcheck disable=SC2086
         parallel --ungroup --bg --semaphore --semaphorename "${SEMAPHORE_NAME}" \
-            --jobs "${MAX_PARALLEL_TEST_JOBS}" \
+            --jobs "${MAX_PARALLEL_TEST_JOBS}" --timeout 1500 \
             "$( dirname "${BASH_SOURCE[0]}" )/ci_run_single_airflow_test_in_docker.sh" "${@}" >${JOB_LOG} 2>&1
     done
     parallel --semaphore --semaphorename "${SEMAPHORE_NAME}" --wait