[apisix] branch master updated: ci: rerun flaky tests (#5980)

[sp...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git


The following commit(s) were added to refs/heads/master by this push:
     new e7b0906  ci: rerun flaky tests (#5980)
e7b0906 is described below

commit e7b09064effd5f4b63e8eb6f16651d3504c47031
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Sun Jan 2 17:49:42 2022 +0800

    ci: rerun flaky tests (#5980)
---
 ci/centos7-ci.sh                    |  3 ++-
 ci/common.sh                        | 18 ++++++++++++++++++
 ci/linux_openresty_common_runner.sh |  3 ++-
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/ci/centos7-ci.sh b/ci/centos7-ci.sh
index c620417..3167fce 100755
--- a/ci/centos7-ci.sh
+++ b/ci/centos7-ci.sh
@@ -85,7 +85,8 @@ run_case() {
     make init
     ./utils/set-dns.sh
     # run test cases
-    FLUSH_ETCD=1 prove -I./test-nginx/lib -I./ -r t/
+    FLUSH_ETCD=1 prove -Itest-nginx/lib -I./ -r t | tee /tmp/test.result
+    rerun_flaky_tests /tmp/test.result
 }
 
 case_opt=$1
diff --git a/ci/common.sh b/ci/common.sh
index 612236e..51bee69 100644
--- a/ci/common.sh
+++ b/ci/common.sh
@@ -31,6 +31,24 @@ create_lua_deps() {
     # luarocks install luacov-coveralls --tree=deps --local > build.log 2>&1 || (cat build.log && exit 1)
 }
 
+rerun_flaky_tests() {
+    if tail -1 "$1" | grep "Result: PASS"; then
+        exit 0
+    fi
+
+    local tests
+    local n_test
+    tests="$(awk '/^t\/.*.t\s+\(.+ Failed: .+\)/{ print $1 }' "$1")"
+    n_test="$(echo "$tests" | wc -l)"
+    if [ "$n_test" -eq 0 ] || [ "$n_test" -gt 3 ]; then
+        # CI failure failed test or too many tests failed
+        exit 1
+    fi
+
+    echo "Rerun $(echo "$tests" | xargs)"
+    FLUSH_ETCD=1 prove -I./test-nginx/lib -I./ $(echo "$tests" | xargs)
+}
+
 install_grpcurl () {
     # For more versions, visit https://github.com/fullstorydev/grpcurl/releases
     GRPCURL_VERSION="1.8.5"
diff --git a/ci/linux_openresty_common_runner.sh b/ci/linux_openresty_common_runner.sh
index 98a9be2..9cfee16 100755
--- a/ci/linux_openresty_common_runner.sh
+++ b/ci/linux_openresty_common_runner.sh
@@ -83,7 +83,8 @@ script() {
     done
 
     # APISIX_ENABLE_LUACOV=1 PERL5LIB=.:$PERL5LIB prove -Itest-nginx/lib -r t
-    FLUSH_ETCD=1 PERL5LIB=.:$PERL5LIB prove -Itest-nginx/lib -r t
+    FLUSH_ETCD=1 prove -Itest-nginx/lib -I./ -r t | tee /tmp/test.result
+    rerun_flaky_tests /tmp/test.result
 }
 
 after_success() {