[flink] branch master updated: [FLINK-16768][tests] Let the watchdog also monitor mvn logs

[rm...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rmetzger pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 79e74f5  [FLINK-16768][tests] Let the watchdog also monitor mvn logs
79e74f5 is described below

commit 79e74f5ea1c8fe33bee1a5a142ce113d49ebae8e
Author: Robert Metzger <rm...@apache.org>
AuthorDate: Wed Aug 26 18:40:54 2020 +0200

    [FLINK-16768][tests] Let the watchdog also monitor mvn logs
    
    This experimental change aims to make the watchdog more accurate by extending the monitoring from the output to log files produced by the tests.
    In particular the S3 tests seem to produce no output while they are producing output to the log files.
    
    The risk of this change is that a hanging test is still producing log output (e.g. the test is stuck in a retry-loop).
---
 tools/ci/test_controller.sh |  6 +++++-
 tools/ci/watchdog.sh        | 20 +++++++++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/tools/ci/test_controller.sh b/tools/ci/test_controller.sh
index 8025e75..d723f4e 100755
--- a/tools/ci/test_controller.sh
+++ b/tools/ci/test_controller.sh
@@ -30,7 +30,7 @@ fi
 source "${HERE}/stage.sh"
 source "${HERE}/maven-utils.sh"
 source "${HERE}/controller_utils.sh"
-source "${HERE}/watchdog.sh"
+
 STAGE=$1
 
 # =============================================================================
@@ -69,6 +69,10 @@ export JAVA_TOOL_OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
 # some tests provide additional logs if they find this variable
 export IS_CI=true
 
+export WATCHDOG_ADDITIONAL_MONITORING_FILES="$DEBUG_FILES_OUTPUT_DIR/mvn-*.log"
+
+source "${HERE}/watchdog.sh"
+
 # =============================================================================
 # Step 1: Rebuild jars and install Flink to local maven repository
 # =============================================================================
diff --git a/tools/ci/watchdog.sh b/tools/ci/watchdog.sh
index 1e03430..1c069e3 100755
--- a/tools/ci/watchdog.sh
+++ b/tools/ci/watchdog.sh
@@ -38,8 +38,26 @@ CMD_EXIT="/tmp/watchdog.exit"
 # Utility functions
 # ============================================= 
 
+max_of() {
+  local max number
+
+  max="$1"
+
+  for number in "${@:2}"; do
+    if ((number > max)); then
+      max="$number"
+    fi
+  done
+
+  printf '%d\n' "$max"
+}
+
+# Returns the highest modification time out of $CMD_OUT (which is the command output file)
+# and any file(s) named "mvn-*.log" (which are logging files created by Flink's tests)
 mod_time () {
-	echo `stat -c "%Y" $CMD_OUT`
+	CMD_OUT_MOD_TIME=`stat -c "%Y" $CMD_OUT`
+	ADDITIONAL_FILES_MOD_TIMES=`stat -c "%Y" $WATCHDOG_ADDITIONAL_MONITORING_FILES 2> /dev/null`
+	echo `max_of $CMD_OUT_MOD_TIME $ADDITIONAL_FILES_MOD_TIMES`
 }
 
 the_time() {