You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2023/04/25 01:03:44 UTC
[doris] branch master updated: [improvement](regression-test) add max_failure_num to skip tests when too much failure #19003
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 72632b1e32 [improvement](regression-test) add max_failure_num to skip tests when too much failure #19003
72632b1e32 is described below
commit 72632b1e3293394b1ff672d60252565c63e0841d
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Tue Apr 25 09:03:36 2023 +0800
[improvement](regression-test) add max_failure_num to skip tests when too much failure #19003
---
regression-test/conf/regression-conf.groovy | 5 +++++
.../apache/doris/regression/RegressionTest.groovy | 24 ++++++++++++++++++----
.../apache/doris/regression/util/Recorder.groovy | 24 ++++++++++++++++++++++
.../pipeline/p0/conf/regression-conf.groovy | 2 ++
.../pipeline/p1/conf/regression-conf.groovy | 2 ++
5 files changed, 53 insertions(+), 4 deletions(-)
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index ed1f393a33..30188b3c93 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -126,3 +126,8 @@ extEsPassword = "***********"
s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"
+
+// If the failure suite num exceeds this config
+// all following suite will be skipped to fast quit the run.
+// <=0 means no limit.
+max_failure_num=0
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
index 7c45cac9a6..e33577528c 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
@@ -136,6 +136,13 @@ class RegressionTest {
canRun(config, suiteName, groupName)
}
def file = source.getFile()
+ int failureLimit = Integer.valueOf(config.otherConfigs.getOrDefault("max_failure_num", "-1").toString());
+ if (Recorder.isFailureExceedLimit(failureLimit)) {
+ // too much failure, skip all following suits
+ log.warn("too much failure ${Recorder.getFailureOrFatalNum()}, limit ${failureLimit}, skip following suits: ${file}")
+ recorder.onSkip(file.absolutePath);
+ return;
+ }
def eventListeners = getEventListeners(config, recorder)
new ScriptContext(file, suiteExecutors, actionExecutors,
config, eventListeners, suiteFilter).start { scriptContext ->
@@ -255,10 +262,10 @@ class RegressionTest {
}
static boolean printResult(Config config, Recorder recorder) {
- int allSuiteNum = recorder.successList.size() + recorder.failureList.size()
+ int allSuiteNum = recorder.successList.size() + recorder.failureList.size() + recorder.skippedList.size()
int failedSuiteNum = recorder.failureList.size()
int fatalScriptNum = recorder.fatalScriptList.size()
- log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites, fatal ${fatalScriptNum} scripts".toString())
+ int skippedNum = recorder.skippedList.size()
// print success list
if (!recorder.successList.isEmpty()) {
@@ -268,6 +275,13 @@ class RegressionTest {
log.info("Success suites:\n${successList}".toString())
}
+ // print skipped list
+ if (!recorder.skippedList.isEmpty()) {
+ String skippedList = recorder.skippedList.collect { info -> "${info}" }.join('\n')
+ log.info("Skipped suites:\n${skippedList}".toString())
+ }
+
+ boolean pass = false;
// print failure list
if (!recorder.failureList.isEmpty() || !recorder.fatalScriptList.isEmpty()) {
if (!recorder.failureList.isEmpty()) {
@@ -283,11 +297,13 @@ class RegressionTest {
log.info("Fatal scripts:\n${failureList}".toString())
}
printFailed()
- return false
} else {
printPassed()
- return true
+ pass = true;
}
+
+ log.info("Test ${allSuiteNum} suites, failed ${failedSuiteNum} suites, fatal ${fatalScriptNum} scripts, skipped ${skippedNum} scripts".toString())
+ return pass;
}
static void loadPlugins(Config config) {
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy
index 2e976e3021..25bbd524c6 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/Recorder.groovy
@@ -18,24 +18,48 @@
package org.apache.doris.regression.util
import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
import org.apache.doris.regression.suite.ScriptInfo
import org.apache.doris.regression.suite.SuiteInfo
+import java.util.concurrent.atomic.AtomicLong;
+
+@Slf4j
@CompileStatic
class Recorder {
public final List<SuiteInfo> successList = new Vector<>()
public final List<SuiteInfo> failureList = new Vector<>()
+ public final List<String> skippedList = new Vector<>()
public final List<ScriptInfo> fatalScriptList = new Vector<>()
+ public static AtomicLong failureCounter = new AtomicLong(0);
+
+ public static boolean isFailureExceedLimit(int limit) {
+ if (limit <=0) {
+ return false;
+ }
+ return failureCounter.get() > limit;
+ }
+
+ public static int getFailureOrFatalNum() {
+ return failureCounter.get()
+ }
+
void onSuccess(SuiteInfo suiteInfo) {
successList.add(suiteInfo)
}
void onFailure(SuiteInfo suiteInfo) {
failureList.add(suiteInfo)
+ failureCounter.incrementAndGet();
+ }
+
+ void onSkip(String skippedFile) {
+ skippedList.add(skippedFile)
}
void onFatal(ScriptInfo scriptInfo) {
fatalScriptList.add(scriptInfo)
+ failureCounter.incrementAndGet();
}
}
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy
index 2beff2390b..33e2bc8ff5 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -82,3 +82,5 @@ cacheDataPath = "/data/regression/"
s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"
+
+max_failure_num=50
diff --git a/regression-test/pipeline/p1/conf/regression-conf.groovy b/regression-test/pipeline/p1/conf/regression-conf.groovy
index acf8f7788d..253e5eee5e 100644
--- a/regression-test/pipeline/p1/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p1/conf/regression-conf.groovy
@@ -46,3 +46,5 @@ cacheDataPath="/data/regression/"
s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
s3Region = "ap-hongkong"
+
+max_failure_num=0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org