You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/05 01:50:12 UTC
[incubator-doris] branch master updated: [improvement](regression-test) support exclude suite/group/directory (#9096)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4ca3eb7fe2 [improvement](regression-test) support exclude suite/group/directory (#9096)
4ca3eb7fe2 is described below
commit 4ca3eb7fe284fe49327065b298fc50c5affc31ba
Author: 924060929 <92...@qq.com>
AuthorDate: Thu May 5 09:50:07 2022 +0800
[improvement](regression-test) support exclude suite/group/directory (#9096)
regression testing framework support skip some suite/group/directory
---
docs/zh-CN/developer-guide/regression-testing.md | 16 ++++++
regression-test/conf/regression-conf.groovy | 7 +++
.../org/apache/doris/regression/Config.groovy | 63 +++++++++++++++++++---
.../apache/doris/regression/ConfigOptions.groovy | 33 ++++++++++++
.../apache/doris/regression/RegressionTest.groovy | 43 +++++++++++----
.../doris/regression/suite/ScriptSource.groovy | 23 ++++----
run-regression-test.sh | 24 ++-------
7 files changed, 161 insertions(+), 48 deletions(-)
diff --git a/docs/zh-CN/developer-guide/regression-testing.md b/docs/zh-CN/developer-guide/regression-testing.md
index 55fb5e8fea..7d2db8d59c 100644
--- a/docs/zh-CN/developer-guide/regression-testing.md
+++ b/docs/zh-CN/developer-guide/regression-testing.md
@@ -109,6 +109,13 @@ testSuites = ""
// 默认会加载的用例目录, 可以通过run-regression-test.sh --run -d来动态指定和覆盖
testDirectories = ""
+// 排除这些组的用例，可通过run-regression-test.sh --run -xg来动态指定和覆盖
+excludeGroups = ""
+// 排除这些suite，可通过run-regression-test.sh --run -xs来动态指定和覆盖
+excludeSuites = ""
+// 排除这些目录，可通过run-regression-test.sh --run -xd来动态指定和覆盖
+excludeDirectories = ""
+
// 其他自定义配置
customConf1 = "test_custom_conf_value"
```
@@ -517,6 +524,15 @@ thread, lazyCheck, events, connect, selectUnionAll
# 测试demo目录下的sql_action
./run-regression-test.sh --run -d demo -s sql_action
+# 测试demo目录下用例，排除sql_action用例
+./run-regression-test.sh --run -d demo -xs sql_action
+
+# 排除demo目录的用例
+./run-regression-test.sh --run -xd demo
+
+# 排除demo group的用例
+./run-regression-test.sh --run -xg demo
+
# 自定义配置
./run-regression-test.sh --run -conf a=b
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 8839331d3b..ec5b8a21bf 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -41,6 +41,13 @@ testSuites = ""
// empty directories will test all directories
testDirectories = ""
+// this groups will not be executed
+excludeGroups = ""
+// this suites will not be executed
+excludeSuites = ""
+// this directories will not be executed
+excludeDirectories = ""
+
customConf1 = "test_custom_conf_value"
// for test csv with header
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 69d8dfcc08..cf2bc0a0b8 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -47,8 +47,11 @@ class Config {
public String dataPath
public String testGroups
+ public String excludeGroups
public String testSuites
+ public String excludeSuites
public String testDirectories
+ public String excludeDirectories
public boolean generateOutputFile
public boolean forceGenerateOutputFile
public boolean randomOrder
@@ -58,6 +61,11 @@ class Config {
public Set<String> suiteWildcard = new HashSet<>()
public Set<String> groups = new HashSet<>()
public Set<String> directories = new HashSet<>()
+
+ public Set<String> excludeSuiteWildcard = new HashSet<>()
+ public Set<String> excludeGroupSet = new HashSet<>()
+ public Set<String> excludeDirectorySet = new HashSet<>()
+
public InetSocketAddress feHttpInetSocketAddress
public Integer parallel
public Integer suiteParallel
@@ -69,7 +77,8 @@ class Config {
Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
String feHttpAddress, String feHttpUser, String feHttpPassword,
- String suitePath, String dataPath, String testGroups, String testSuites, String testDirectories) {
+ String suitePath, String dataPath, String testGroups, String excludeGroups,
+ String testSuites, String excludeSuites, String testDirectories, String excludeDirectories) {
this.defaultDb = defaultDb
this.jdbcUrl = jdbcUrl
this.jdbcUser = jdbcUser
@@ -80,8 +89,11 @@ class Config {
this.suitePath = suitePath
this.dataPath = dataPath
this.testGroups = testGroups
+ this.excludeGroups = excludeGroups
this.testSuites = testSuites
+ this.excludeSuites = excludeSuites
this.testDirectories = testDirectories
+ this.excludeDirectories = excludeDirectories
}
static Config fromCommandLine(CommandLine cmd) {
@@ -103,8 +115,8 @@ class Config {
config.dataPath = FileUtils.getCanonicalPath(cmd.getOptionValue(dataOpt, config.dataPath))
config.suiteWildcard = cmd.getOptionValue(suiteOpt, config.testSuites)
.split(",")
- .collect({g -> g.trim()})
- .findAll({g -> g != null && g.length() > 0})
+ .collect({s -> s.trim()})
+ .findAll({s -> s != null && s.length() > 0})
.toSet()
config.groups = cmd.getOptionValue(groupsOpt, config.testGroups)
.split(",")
@@ -116,6 +128,21 @@ class Config {
.collect({d -> d.trim()})
.findAll({d -> d != null && d.length() > 0})
.toSet()
+ config.excludeSuiteWildcard = cmd.getOptionValue(excludeSuiteOpt, config.excludeSuites)
+ .split(",")
+ .collect({s -> s.trim()})
+ .findAll({s -> s != null && s.length() > 0})
+ .toSet()
+ config.excludeGroupSet = cmd.getOptionValue(excludeGroupsOpt, config.excludeGroups)
+ .split(",")
+ .collect({g -> g.trim()})
+ .findAll({g -> g != null && g.length() > 0})
+ .toSet()
+ config.excludeDirectorySet = cmd.getOptionValue(excludeDirectoriesOpt, config.excludeDirectories)
+ .split(",")
+ .collect({d -> d.trim()})
+ .findAll({d -> d != null && d.length() > 0})
+ .toSet()
config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress)
try {
@@ -162,8 +189,11 @@ class Config {
configToString(obj.suitePath),
configToString(obj.dataPath),
configToString(obj.testGroups),
+ configToString(obj.excludeGroups),
configToString(obj.testSuites),
- configToString(obj.testDirectories)
+ configToString(obj.excludeSuites),
+ configToString(obj.testDirectories),
+ configToString(obj.excludeDirectories)
)
def declareFileNames = config.getClass()
@@ -230,16 +260,31 @@ class Config {
log.info("Set testGroups to '${config.testGroups}' because not specify.".toString())
}
+ if (config.excludeGroups == null) {
+ config.excludeGroups = ""
+ log.info("Set excludeGroups to empty because not specify.".toString())
+ }
+
if (config.testDirectories == null) {
config.testDirectories = ""
log.info("Set testDirectories to empty because not specify.".toString())
}
+ if (config.excludeDirectories == null) {
+ config.excludeDirectories = ""
+ log.info("Set excludeDirectories to empty because not specify.".toString())
+ }
+
if (config.testSuites == null) {
config.testSuites = ""
log.info("Set testSuites to empty because not specify.".toString())
}
+ if (config.excludeSuites == null) {
+ config.excludeSuites = ""
+ log.info("Set excludeSuites to empty because not specify.".toString())
+ }
+
if (config.parallel == null) {
config.parallel = 1
log.info("Set parallel to 1 because not specify.".toString())
@@ -289,7 +334,7 @@ class Config {
Predicate<String> getDirectoryFilter() {
return (Predicate<String>) { String directoryName ->
- if (directories.isEmpty()) {
+ if (directories.isEmpty() && excludeDirectorySet.isEmpty()) {
return true
}
@@ -302,7 +347,13 @@ class Config {
parentPath = currentPath + File.separator
}
- return allLevelPaths.any {directories.contains(it) }
+ if (!directories.isEmpty() && !allLevelPaths.any({directories.contains(it) })) {
+ return false
+ }
+ if (!excludeDirectorySet.isEmpty() && allLevelPaths.any({ excludeDirectorySet.contains(it) })) {
+ return false
+ }
+ return true
}
}
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
index c0700dd859..e860f3abe7 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
@@ -38,8 +38,11 @@ class ConfigOptions {
static Option pathOpt
static Option dataOpt
static Option suiteOpt
+ static Option excludeSuiteOpt
static Option groupsOpt
+ static Option excludeGroupsOpt
static Option directoriesOpt
+ static Option excludeDirectoriesOpt
static Option confOpt
static Option genOutOpt
static Option forceGenOutOpt
@@ -123,6 +126,15 @@ class ConfigOptions {
.longOpt("suite")
.desc("the suite name wildcard to be test")
.build()
+ excludeSuiteOpt = Option.builder("xs")
+ .argName("excludeSuiteName")
+ .required(false)
+ .hasArg(true)
+ .optionalArg(true)
+ .type(String.class)
+ .longOpt("excludeSuite")
+ .desc("the suite name wildcard will not be tested")
+ .build()
groupsOpt = Option.builder("g")
.argName("groups")
.required(false)
@@ -132,6 +144,15 @@ class ConfigOptions {
.longOpt("groups")
.desc("the suite group to be test")
.build()
+ excludeGroupsOpt = Option.builder("xg")
+ .argName("excludeGroupNames")
+ .required(false)
+ .hasArg(true)
+ .optionalArg(true)
+ .type(String.class)
+ .longOpt("excludeGroups")
+ .desc("the suite group will not be tested")
+ .build()
directoriesOpt = Option.builder("d")
.argName("directories")
.required(false)
@@ -141,6 +162,15 @@ class ConfigOptions {
.longOpt("directories")
.desc("only the use cases in these directories can be executed")
.build()
+ excludeDirectoriesOpt = Option.builder("xd")
+ .argName("excludeDirectoryNames")
+ .required(false)
+ .hasArg(true)
+ .optionalArg(true)
+ .type(String.class)
+ .longOpt("excludeDirectories")
+ .desc("the use cases in these directories will not be tested")
+ .build()
feHttpAddressOpt = Option.builder("ha")
.argName("address")
.required(false)
@@ -239,8 +269,11 @@ class ConfigOptions {
.addOption(dataOpt)
.addOption(confOpt)
.addOption(suiteOpt)
+ .addOption(excludeSuiteOpt)
.addOption(groupsOpt)
+ .addOption(excludeGroupsOpt)
.addOption(directoriesOpt)
+ .addOption(excludeDirectoriesOpt)
.addOption(feHttpAddressOpt)
.addOption(feHttpUserOpt)
.addOption(feHttpPasswordOpt)
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
index f7674ef4b0..2456cdb3c0 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
@@ -173,18 +173,39 @@ class RegressionTest {
return recorder
}
- static boolean canRun(Config config, String suiteName, String group) {
+ static boolean filterSuites(Config config, String suiteName) {
+ if (config.suiteWildcard.isEmpty() && config.excludeSuiteWildcard.isEmpty()) {
+ return true
+ }
+ if (!config.suiteWildcard.isEmpty() && !config.suiteWildcard.any {
+ suiteWildcard -> Wildcard.match(suiteName, suiteWildcard)
+ }) {
+ return false
+ }
+ if (!config.excludeSuiteWildcard.isEmpty() && config.excludeSuiteWildcard.any {
+ excludeSuiteWildcard -> Wildcard.match(suiteName, excludeSuiteWildcard)
+ }) {
+ return false
+ }
+ return true
+ }
+
+ static boolean filterGroups(Config config, String group) {
+ if (config.groups.isEmpty() && config.excludeGroupSet.isEmpty()) {
+ return true
+ }
Set<String> suiteGroups = group.split(',').collect { g -> g.trim() }.toSet()
- if (config.suiteWildcard.size() == 0 ||
- (suiteName != null && (config.suiteWildcard.any {
- suiteWildcard -> Wildcard.match(suiteName, suiteWildcard)
- }))) {
- if (config.groups == null || config.groups.isEmpty()
- || !config.groups.intersect(suiteGroups).isEmpty()) {
- return true
- }
+ if (!config.groups.isEmpty() && config.groups.intersect(suiteGroups).isEmpty()) {
+ return false
}
- return false
+ if (!config.excludeGroupSet.isEmpty() && !config.excludeGroupSet.intersect(suiteGroups).isEmpty()) {
+ return false
+ }
+ return true
+ }
+
+ static boolean canRun(Config config, String suiteName, String group) {
+ return filterGroups(config, group) && filterSuites(config, suiteName)
}
static List<EventListener> getEventListeners(Config config, Recorder recorder) {
@@ -226,7 +247,7 @@ class RegressionTest {
String successList = recorder.successList.collect { info ->
"${info.file.absolutePath}: group=${info.group}, name=${info.suiteName}"
}.join('\n')
- log.info("SuccessList suites:\n${successList}".toString())
+ log.info("Success suites:\n${successList}".toString())
}
// print failure list
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
index 2b736a6090..7c47b22a55 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/ScriptSource.groovy
@@ -59,26 +59,27 @@ class SqlFileSource implements ScriptSource {
return SuiteScript.getDefaultGroups(suiteRoot, file)
}
- @Override
- SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
- String suiteName = file.name.substring(0, file.name.lastIndexOf("."))
- String groupName = getGroup()
- boolean order = suiteName.endsWith("_order")
- String tag = suiteName
- String sql = file.text
-
- List<String> sqls
+ List<String> getSqls(String sql) {
try {
- sqls = SqlUtils.splitAndGetNonEmptySql(sql)
+ return SqlUtils.splitAndGetNonEmptySql(sql)
} catch (Throwable t) {
- sqls = [sql]
log.warn("Try to execute whole file text as one sql, because can not split sql:\n${sql}", t)
+ return [sql]
}
+ }
+
+ @Override
+ SuiteScript toScript(ScriptContext scriptContext, GroovyShell shell) {
+ String suiteName = file.name.substring(0, file.name.lastIndexOf("."))
+ String groupName = getGroup()
SuiteScript script = new SuiteScript() {
@Override
Object run() {
suite(suiteName, groupName) {
+ String tag = suiteName
+ boolean order = suiteName.endsWith("_order")
+ List<String> sqls = getSqls(file.text)
for (int i = 0; i < sqls.size(); ++i) {
String singleSql = sqls.get(i)
String tagName = (i == 0) ? tag : "${tag}_${i + 1}"
diff --git a/run-regression-test.sh b/run-regression-test.sh
index 37364afc63..50ae8c5414 100755
--- a/run-regression-test.sh
+++ b/run-regression-test.sh
@@ -16,26 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-#####################################################################
-# This script is used to run regression test of Doris Backend
-# Usage: $0 <shell_options> <framework_options>
-# Optional shell_options:
-# --clean clean output of regression test
-# --teamcity print teamcity service messages
-# --run run regression test. build framework if necessary
-#
-# Optional framework_options
-# -h print all other_options
-# -s xxx suite name
-# -g xxx group name
-# -c xxx jdbc url
-# -u xxx jdbc user
-# -genOut generate .out file
-# -forceGenOut delete and generate .out file
-#
-# log to ${DORIS_HOME}/output/regression/log
-#####################################################################
-
set -eo pipefail
#set -x
@@ -56,7 +36,11 @@ Usage: $0 <shell_options> <framework_options>
Optional framework_options:
-s run a specified suite
-g run a specified group
+ -d run a specified directory
-h **print all framework options usage**
+ -xs exclude the specified suite
+ -xg exclude the specified group
+ -xd exclude the specified directory
-genOut generate .out file if not exist
-forceGenOut delete and generate .out file if not exist
-parallel run tests using specified threads
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org