You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2026/04/13 09:09:08 UTC
(doris) branch master updated: [chore](compaction) change time series threshold from 2000 to 1000 (#61979)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 52da1d6c33e [chore](compaction) change time series threshold from 2000 to 1000 (#61979)
52da1d6c33e is described below
commit 52da1d6c33e45b24d3ba182bfcd8ba337162ba88
Author: Chenyang Sun <su...@selectdb.com>
AuthorDate: Mon Apr 13 17:09:00 2026 +0800
[chore](compaction) change time series threshold from 2000 to 1000 (#61979)
---
be/src/storage/tablet/tablet_meta.h | 2 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 2 +-
gensrc/proto/olap_file.proto | 4 +--
gensrc/thrift/AgentService.thrift | 2 +-
.../data/query_p0/system/test_table_properties.out | 12 +++----
.../test_time_series_compaction_policy.groovy | 42 ++++++++++++++++++++++
6 files changed, 53 insertions(+), 11 deletions(-)
diff --git a/be/src/storage/tablet/tablet_meta.h b/be/src/storage/tablet/tablet_meta.h
index 32dc7e25607..0e52bd8429b 100644
--- a/be/src/storage/tablet/tablet_meta.h
+++ b/be/src/storage/tablet/tablet_meta.h
@@ -110,7 +110,7 @@ public:
std::optional<TBinlogConfig> binlog_config = {},
std::string compaction_policy = "size_based",
int64_t time_series_compaction_goal_size_mbytes = 1024,
- int64_t time_series_compaction_file_count_threshold = 2000,
+ int64_t time_series_compaction_file_count_threshold = 1000,
int64_t time_series_compaction_time_threshold_seconds = 3600,
int64_t time_series_compaction_empty_rowsets_threshold = 5,
int64_t time_series_compaction_level_threshold = 1,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 71d778ba758..7ba7ddbdb05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -270,7 +270,7 @@ public class PropertyAnalyzer {
public static final String SIZE_BASED_COMPACTION_POLICY = "size_based";
public static final String TIME_SERIES_COMPACTION_POLICY = "time_series";
public static final long TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES_DEFAULT_VALUE = 1024;
- public static final long TIME_SERIES_COMPACTION_FILE_COUNT_THRESHOLD_DEFAULT_VALUE = 2000;
+ public static final long TIME_SERIES_COMPACTION_FILE_COUNT_THRESHOLD_DEFAULT_VALUE = 1000;
public static final long TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE = 3600;
public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5;
public static final long TIME_SERIES_COMPACTION_LEVEL_THRESHOLD_DEFAULT_VALUE = 1;
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index c6760811345..11097d47e10 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -630,7 +630,7 @@ message TabletMetaPB {
optional BinlogConfigPB binlog_config = 27;
optional string compaction_policy = 28 [default = "size_based"];
optional int64 time_series_compaction_goal_size_mbytes = 29 [default = 1024];
- optional int64 time_series_compaction_file_count_threshold = 30 [default = 2000];
+ optional int64 time_series_compaction_file_count_threshold = 30 [default = 1000];
optional int64 time_series_compaction_time_threshold_seconds = 31 [default = 3600];
optional int64 time_series_compaction_empty_rowsets_threshold = 32 [default = 5];
optional int64 time_series_compaction_level_threshold = 33 [default = 1];
@@ -686,7 +686,7 @@ message TabletMetaCloudPB {
optional BinlogConfigPB binlog_config = 30;
optional string compaction_policy = 31 [default = "size_based"];
optional int64 time_series_compaction_goal_size_mbytes = 32 [default = 1024];
- optional int64 time_series_compaction_file_count_threshold = 33 [default = 2000];
+ optional int64 time_series_compaction_file_count_threshold = 33 [default = 1000];
optional int64 time_series_compaction_time_threshold_seconds = 34 [default = 3600];
reserved 35; // deprecated group_commit_interval_ms
reserved 36; // deprecated group_commit_data_bytes
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index 61ee5a0d144..c24753f3ec9 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -222,7 +222,7 @@ struct TCreateTabletReq {
21: optional TBinlogConfig binlog_config
22: optional string compaction_policy = "size_based"
23: optional i64 time_series_compaction_goal_size_mbytes = 1024
- 24: optional i64 time_series_compaction_file_count_threshold = 2000
+ 24: optional i64 time_series_compaction_file_count_threshold = 1000
25: optional i64 time_series_compaction_time_threshold_seconds = 3600
26: optional i64 time_series_compaction_empty_rowsets_threshold = 5
27: optional i64 time_series_compaction_level_threshold = 1
diff --git a/regression-test/data/query_p0/system/test_table_properties.out b/regression-test/data/query_p0/system/test_table_properties.out
index 5f34cd069f2..9f30b6cfde8 100644
--- a/regression-test/data/query_p0/system/test_table_properties.out
+++ b/regression-test/data/query_p0/system/test_table_properties.out
@@ -36,7 +36,7 @@ internal test_table_properties_db duplicate_table storage_page_size 65536
internal test_table_properties_db duplicate_table store_row_column false
internal test_table_properties_db duplicate_table tde_algorithm PLAINTEXT
internal test_table_properties_db duplicate_table time_series_compaction_empty_rowsets_threshold 5
-internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 2000
+internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 1000
internal test_table_properties_db duplicate_table time_series_compaction_goal_size_mbytes 1024
internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1
internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600
@@ -74,7 +74,7 @@ internal test_table_properties_db listtable storage_page_size 65536
internal test_table_properties_db listtable store_row_column false
internal test_table_properties_db listtable tde_algorithm PLAINTEXT
internal test_table_properties_db listtable time_series_compaction_empty_rowsets_threshold 5
-internal test_table_properties_db listtable time_series_compaction_file_count_threshold 2000
+internal test_table_properties_db listtable time_series_compaction_file_count_threshold 1000
internal test_table_properties_db listtable time_series_compaction_goal_size_mbytes 1024
internal test_table_properties_db listtable time_series_compaction_level_threshold 1
internal test_table_properties_db listtable time_series_compaction_time_threshold_seconds 3600
@@ -112,7 +112,7 @@ internal test_table_properties_db unique_table storage_page_size 65536
internal test_table_properties_db unique_table store_row_column false
internal test_table_properties_db unique_table tde_algorithm PLAINTEXT
internal test_table_properties_db unique_table time_series_compaction_empty_rowsets_threshold 5
-internal test_table_properties_db unique_table time_series_compaction_file_count_threshold 2000
+internal test_table_properties_db unique_table time_series_compaction_file_count_threshold 1000
internal test_table_properties_db unique_table time_series_compaction_goal_size_mbytes 1024
internal test_table_properties_db unique_table time_series_compaction_level_threshold 1
internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600
@@ -152,7 +152,7 @@ internal test_table_properties_db duplicate_table storage_page_size 65536
internal test_table_properties_db duplicate_table store_row_column false
internal test_table_properties_db duplicate_table tde_algorithm PLAINTEXT
internal test_table_properties_db duplicate_table time_series_compaction_empty_rowsets_threshold 5
-internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 2000
+internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 1000
internal test_table_properties_db duplicate_table time_series_compaction_goal_size_mbytes 1024
internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1
internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600
@@ -190,7 +190,7 @@ internal test_table_properties_db unique_table storage_page_size 65536
internal test_table_properties_db unique_table store_row_column false
internal test_table_properties_db unique_table tde_algorithm PLAINTEXT
internal test_table_properties_db unique_table time_series_compaction_empty_rowsets_threshold 5
-internal test_table_properties_db unique_table time_series_compaction_file_count_threshold 2000
+internal test_table_properties_db unique_table time_series_compaction_file_count_threshold 1000
internal test_table_properties_db unique_table time_series_compaction_goal_size_mbytes 1024
internal test_table_properties_db unique_table time_series_compaction_level_threshold 1
internal test_table_properties_db unique_table time_series_compaction_time_threshold_seconds 3600
@@ -232,7 +232,7 @@ internal test_table_properties_db duplicate_table storage_page_size 65536
internal test_table_properties_db duplicate_table store_row_column false
internal test_table_properties_db duplicate_table tde_algorithm PLAINTEXT
internal test_table_properties_db duplicate_table time_series_compaction_empty_rowsets_threshold 5
-internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 2000
+internal test_table_properties_db duplicate_table time_series_compaction_file_count_threshold 1000
internal test_table_properties_db duplicate_table time_series_compaction_goal_size_mbytes 1024
internal test_table_properties_db duplicate_table time_series_compaction_level_threshold 1
internal test_table_properties_db duplicate_table time_series_compaction_time_threshold_seconds 3600
diff --git a/regression-test/suites/compaction/test_time_series_compaction_policy.groovy b/regression-test/suites/compaction/test_time_series_compaction_policy.groovy
index 69c8fb97fa7..7f3aa2abf81 100644
--- a/regression-test/suites/compaction/test_time_series_compaction_policy.groovy
+++ b/regression-test/suites/compaction/test_time_series_compaction_policy.groovy
@@ -36,6 +36,48 @@ suite("test_time_series_compaction_polciy", "p0") {
return rowsetCount
}
+ sql """ DROP TABLE IF EXISTS ${tableName}; """
+ sql """
+ CREATE TABLE ${tableName} (
+ `id` int(11) NULL,
+ `name` varchar(255) NULL,
+ `hobbies` text NULL,
+ `score` int(11) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1",
+ "disable_auto_compaction" = "true",
+ "compaction_policy" = "time_series"
+ );
+ """
+
+ for (int i = 0; i < 1005; i++) {
+ sql """ INSERT INTO ${tableName} VALUES (${i}, "andy", "andy love apple", 100); """
+ }
+
+ def triggered_tablets = sql_return_maparray """show tablets from ${tableName}"""
+ for (tablet in triggered_tablets) {
+ def be_host = backendId_to_backendIP["${tablet.BackendId}"]
+ def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"]
+ curl("POST", "http://${be_host}:${be_port}/api/compaction/run?tablet_id=${tablet.TabletId}&compact_type=cumulative")
+ }
+
+ Thread.sleep(10000)
+ for (tablet in triggered_tablets) {
+ def be_host = backendId_to_backendIP["${tablet.BackendId}"]
+ def be_port = backendId_to_backendHttpPort["${tablet.BackendId}"]
+
+ def (exit_code, stdout, stderr) = be_get_compaction_status(be_host, be_port, tablet.TabletId)
+ assert exit_code == 0: "get compaction status failed, exit code: ${exit_code}, stdout: ${stdout}, stderr: ${stderr}"
+ def compactionStatus = parseJson(stdout.trim())
+ logger.info("compaction status: ${compactionStatus}")
+ assert compactionStatus.status.toLowerCase() == "success": "compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${compactionStatus.status}"
+ }
+ tableName = "test_time_series_compaction_polciy_2"
+
sql """ DROP TABLE IF EXISTS ${tableName}; """
sql """
CREATE TABLE ${tableName} (
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org