You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/11 01:40:27 UTC
[incubator-doris] branch master updated: [regression] add regression test for compaction (#9437)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a738d385db [regression] add regression test for compaction (#9437)
a738d385db is described below
commit a738d385db23b0f7dc3749d443069a1f86a37cb3
Author: Gabriel <ga...@gmail.com>
AuthorDate: Wed May 11 09:40:21 2022 +0800
[regression] add regression test for compaction (#9437)
Trigger compaction via REST API in this case.
---
regression-test/conf/regression-conf.groovy | 2 +
.../data/compaction/test_compaction.out | 13 ++
.../org/apache/doris/regression/Config.groovy | 12 +-
.../apache/doris/regression/ConfigOptions.groovy | 10 ++
.../suites/compaction/test_compaction.groovy | 187 +++++++++++++++++++++
5 files changed, 223 insertions(+), 1 deletion(-)
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 934aab251a..dcc3fdc019 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -28,6 +28,8 @@ feHttpAddress = "127.0.0.1:8030"
feHttpUser = "root"
feHttpPassword = ""
+beHttpAddress = "127.0.0.1:8040"
+
// set DORIS_HOME by system properties
// e.g. java -DDORIS_HOME=./
suitePath = "${DORIS_HOME}/regression-test/suites"
diff --git a/regression-test/data/compaction/test_compaction.out b/regression-test/data/compaction/test_compaction.out
new file mode 100644
index 0000000000..41365e84bf
--- /dev/null
+++ b/regression-test/data/compaction/test_compaction.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_default --
+1 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 2 31 19 \N \N
+2 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 2 32 20 \N \N
+3 2017-10-01 Beijing 10 1 \N 2020-01-04T00:00 2020-01-05T00:00 3 34 20 \N \N
+4 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 34 20 \N \N
+
+-- !select_default2 --
+1 2017-10-01 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2020-01-02T00:00 2 31 19 \N \N
+2 2017-10-01 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2020-01-03T00:00 2 32 20 \N \N
+3 2017-10-01 Beijing 10 1 \N 2020-01-04T00:00 2020-01-05T00:00 3 34 20 \N \N
+4 2017-10-01 Beijing 10 1 \N \N 2020-01-05T00:00 1 34 20 \N \N
+
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 675ca676fb..f8cdfd5960 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -43,6 +43,8 @@ class Config {
public String feHttpUser
public String feHttpPassword
+ public String beHttpAddress
+
public String suitePath
public String dataPath
public String pluginPath
@@ -77,7 +79,7 @@ class Config {
Config() {}
Config(String defaultDb, String jdbcUrl, String jdbcUser, String jdbcPassword,
- String feHttpAddress, String feHttpUser, String feHttpPassword,
+ String feHttpAddress, String feHttpUser, String feHttpPassword, String beHttpAddress,
String suitePath, String dataPath, String testGroups, String excludeGroups,
String testSuites, String excludeSuites, String testDirectories, String excludeDirectories,
String pluginPath) {
@@ -88,6 +90,7 @@ class Config {
this.feHttpAddress = feHttpAddress
this.feHttpUser = feHttpUser
this.feHttpPassword = feHttpPassword
+ this.beHttpAddress = beHttpAddress
this.suitePath = suitePath
this.dataPath = dataPath
this.testGroups = testGroups
@@ -149,6 +152,7 @@ class Config {
.toSet()
config.feHttpAddress = cmd.getOptionValue(feHttpAddressOpt, config.feHttpAddress)
+ config.beHttpAddress = cmd.getOptionValue(beHttpAddressOpt, config.beHttpAddress)
try {
Inet4Address host = Inet4Address.getByName(config.feHttpAddress.split(":")[0]) as Inet4Address
int port = Integer.valueOf(config.feHttpAddress.split(":")[1])
@@ -190,6 +194,7 @@ class Config {
configToString(obj.feHttpAddress),
configToString(obj.feHttpUser),
configToString(obj.feHttpPassword),
+ configToString(obj.beHttpAddress),
configToString(obj.suitePath),
configToString(obj.dataPath),
configToString(obj.testGroups),
@@ -240,6 +245,11 @@ class Config {
log.info("Set feHttpAddress to '${config.feHttpAddress}' because not specify.".toString())
}
+ if (config.beHttpAddress == null) {
+ config.beHttpAddress = "127.0.0.1:8040"
+ log.info("Set beHttpAddress to '${config.beHttpAddress}' because not specify.".toString())
+ }
+
if (config.feHttpUser == null) {
config.feHttpUser = "root"
log.info("Set feHttpUser to '${config.feHttpUser}' because not specify.".toString())
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
index 3a25cfafa9..10c1b6ddef 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/ConfigOptions.groovy
@@ -35,6 +35,7 @@ class ConfigOptions {
static Option feHttpAddressOpt
static Option feHttpUserOpt
static Option feHttpPasswordOpt
+ static Option beHttpAddressOpt
static Option pathOpt
static Option dataOpt
static Option pluginOpt
@@ -204,6 +205,14 @@ class ConfigOptions {
.longOpt("feHttpPassword")
.desc("the password of fe http server")
.build()
+ beHttpAddressOpt = Option.builder("ba")
+ .argName("beAddress")
+ .required(false)
+ .hasArg(true)
+ .type(String.class)
+ .longOpt("beHttpAddress")
+ .desc("the be http address, format is ip:port")
+ .build()
genOutOpt = Option.builder("genOut")
.required(false)
.hasArg(false)
@@ -287,6 +296,7 @@ class ConfigOptions {
.addOption(feHttpAddressOpt)
.addOption(feHttpUserOpt)
.addOption(feHttpPasswordOpt)
+ .addOption(beHttpAddressOpt)
.addOption(genOutOpt)
.addOption(confFileOpt)
.addOption(forceGenOutOpt)
diff --git a/regression-test/suites/compaction/test_compaction.groovy b/regression-test/suites/compaction/test_compaction.groovy
new file mode 100644
index 0000000000..4722fcf5f0
--- /dev/null
+++ b/regression-test/suites/compaction/test_compaction.groovy
@@ -0,0 +1,187 @@
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_compaction") {
+ def tableName = "compaction_regression_test"
+
+ try {
+ StringBuilder showConfigCommand = new StringBuilder();
+ showConfigCommand.append("curl -X GET http://")
+ showConfigCommand.append(context.config.beHttpAddress)
+ showConfigCommand.append("/api/show_config")
+ def process = showConfigCommand.toString().execute()
+ int code = process.waitFor()
+ String err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ String out = process.getText()
+ logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def configList = parseJson(out.trim())
+ assert configList instanceof List
+
+ int cumulativeCompactionSkipWindowSeconds = -1
+ boolean disableAutoCompaction = true
+ for (Object ele in (List) configList) {
+ assert ele instanceof List<String>
+ if (((List<String>) ele)[0] == "cumulative_compaction_skip_window_seconds") {
+ cumulativeCompactionSkipWindowSeconds = Integer.parseInt(((List<String>) ele)[2])
+ } else if (((List<String>) ele)[0] == "disable_auto_compaction") {
+ disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
+ }
+ }
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName} (
+ `user_id` LARGEINT NOT NULL COMMENT "用户id",
+ `date` DATE NOT NULL COMMENT "数据灌入日期时间",
+ `city` VARCHAR(20) COMMENT "用户所在城市",
+ `age` SMALLINT COMMENT "用户年龄",
+ `sex` TINYINT COMMENT "用户性别",
+ `last_visit_date` DATETIME REPLACE DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间",
+ `last_update_date` DATETIME REPLACE_IF_NOT_NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次更新时间",
+ `last_visit_date_not_null` DATETIME REPLACE NOT NULL DEFAULT "1970-01-01 00:00:00" COMMENT "用户最后一次访问时间",
+ `cost` BIGINT SUM DEFAULT "0" COMMENT "用户总消费",
+ `max_dwell_time` INT MAX DEFAULT "0" COMMENT "用户最大停留时间",
+ `min_dwell_time` INT MIN DEFAULT "99999" COMMENT "用户最小停留时间",
+ `hll_col` HLL HLL_UNION NOT NULL COMMENT "HLL列",
+ `bitmap_col` Bitmap BITMAP_UNION NOT NULL COMMENT "bitmap列" )
+ AGGREGATE KEY(`user_id`, `date`, `city`, `age`, `sex`) DISTRIBUTED BY HASH(`user_id`)
+ PROPERTIES ( "replication_num" = "1" );
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-01', '2020-01-01', '2020-01-01', 1, 30, 20, hll_hash(1), to_bitmap(1))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 19, hll_hash(2), to_bitmap(2))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-02', '2020-01-02', '2020-01-02', 1, 31, 21, hll_hash(2), to_bitmap(2))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (2, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 20, hll_hash(3), to_bitmap(3))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-03', '2020-01-03', '2020-01-03', 1, 32, 22, hll_hash(3), to_bitmap(3))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (3, '2017-10-01', 'Beijing', 10, 1, '2020-01-04', '2020-01-04', '2020-01-04', 1, 33, 21, hll_hash(4), to_bitmap(4))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (3, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (4, '2017-10-01', 'Beijing', 10, 1, NULL, NULL, '2020-01-05', 1, 34, 20, hll_hash(5), to_bitmap(5))
+ """
+
+ qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+ String[][] tablets = sql """ show tablets from ${tableName}; """
+
+ if (cumulativeCompactionSkipWindowSeconds > 0) {
+ logger.info("Config `cumulative_compaction_skip_window_seconds` is set to " + cumulativeCompactionSkipWindowSeconds + " seconds so sleep for a while.")
+ Thread.sleep(cumulativeCompactionSkipWindowSeconds * 1000)
+ }
+
+ // trigger compactions for all tablets in ${tableName}
+ for (String[] tablet in tablets) {
+ String tablet_id = tablet[0]
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X POST http://")
+ sb.append(context.config.beHttpAddress)
+ sb.append("/api/compaction/run?tablet_id=")
+ sb.append(tablet_id)
+ sb.append("&compact_type=cumulative")
+
+ String command = sb.toString()
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def compactJson = parseJson(out.trim())
+ if (compactJson.status.toLowerCase() == "fail") {
+ assertEquals(disableAutoCompaction, false)
+ logger.info("Compaction was done automatically!")
+ }
+ if (disableAutoCompaction) {
+ assertEquals("success", compactJson.status.toLowerCase())
+ }
+ }
+
+ // wait for all compactions done
+ for (String[] tablet in tablets) {
+ boolean running = true
+ do {
+ Thread.sleep(1000)
+ String tablet_id = tablet[0]
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X GET http://")
+ sb.append(context.config.beHttpAddress)
+ sb.append("/api/compaction/run_status?tablet_id=")
+ sb.append(tablet_id)
+
+ String command = sb.toString()
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def compactionStatus = parseJson(out.trim())
+ assertEquals("success", compactionStatus.status.toLowerCase())
+ running = compactionStatus.run_status
+ } while (running)
+ }
+
+ int rowCount = 0
+ for (String[] tablet in tablets) {
+ String tablet_id = tablet[0]
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X GET http://")
+ sb.append(context.config.beHttpAddress)
+ sb.append("/api/compaction/show?tablet_id=")
+ sb.append(tablet_id)
+ String command = sb.toString()
+ // wait for cleaning stale_rowsets
+ process = command.execute()
+ code = process.waitFor()
+ err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ out = process.getText()
+ logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+ assertEquals(code, 0)
+ def tabletJson = parseJson(out.trim())
+ assert tabletJson.rowsets instanceof List
+ for (String rowset in (List<String>) tabletJson.rowsets) {
+ rowCount += Integer.parseInt(rowset.split(" ")[1])
+ }
+ }
+ assert (rowCount < 8)
+ qt_select_default2 """ SELECT * FROM ${tableName} t ORDER BY user_id; """
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org