You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/06/17 12:32:40 UTC
[doris] branch master updated: [fix](compaction) fix time series compaction policy to adjust vertical compaction max segment size (#20889)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0585a1f004 [fix](compaction) fix time series compaction policy to adjust vertical compaction max segment size (#20889)
0585a1f004 is described below
commit 0585a1f0045983bf9057d44b8dfccdb13d9cfd56
Author: Chenyang Sun <cs...@gmail.com>
AuthorDate: Sat Jun 17 20:32:34 2023 +0800
[fix](compaction) fix time series compaction policy to adjust vertical compaction max segment size (#20889)
---
be/src/olap/compaction.cpp | 5 +++++
be/src/olap/cumulative_compaction_policy.cpp | 2 +-
be/src/olap/cumulative_compaction_policy.h | 2 +-
be/src/olap/cumulative_compaction_time_series_policy.h | 2 +-
be/src/olap/tablet.cpp | 3 ++-
5 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index b870d9b824..005a948c55 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -36,6 +36,7 @@
#include "io/fs/file_system.h"
#include "io/fs/remote_file_system.h"
#include "olap/cumulative_compaction_policy.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/data_dir.h"
#include "olap/olap_define.h"
#include "olap/rowset/beta_rowset.h"
@@ -126,6 +127,10 @@ int64_t Compaction::get_avg_segment_rows() {
// input_rowsets_size is total disk_size of input_rowset, this size is the
// final size after codec and compress, so expect dest segment file size
// in disk is config::vertical_compaction_max_segment_size
+ if (config::compaction_policy == CUMULATIVE_TIME_SERIES_POLICY) {
+ return (config::time_series_compaction_goal_size_mbytes * 1024 * 1024 * 2) /
+ (_input_rowsets_size / (_input_row_num + 1) + 1);
+ }
return config::vertical_compaction_max_segment_size /
(_input_rowsets_size / (_input_row_num + 1) + 1);
}
diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp
index 549d01c111..e2a7af9da8 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -346,7 +346,7 @@ int64_t SizeBasedCumulativeCompactionPolicy::_level_size(const int64_t size) {
std::shared_ptr<CumulativeCompactionPolicy>
CumulativeCompactionPolicyFactory::create_cumulative_compaction_policy() {
- if (config::compaction_policy == "time_series") {
+ if (config::compaction_policy == CUMULATIVE_TIME_SERIES_POLICY) {
return std::make_shared<TimeSeriesCumulativeCompactionPolicy>();
}
return std::make_shared<SizeBasedCumulativeCompactionPolicy>();
diff --git a/be/src/olap/cumulative_compaction_policy.h b/be/src/olap/cumulative_compaction_policy.h
index 89014045ae..65f970e4af 100644
--- a/be/src/olap/cumulative_compaction_policy.h
+++ b/be/src/olap/cumulative_compaction_policy.h
@@ -33,7 +33,7 @@ namespace doris {
class Tablet;
struct Version;
-inline std::string_view CUMULATIVE_SIZE_BASED_POLICY = "SIZE_BASED";
+inline std::string_view CUMULATIVE_SIZE_BASED_POLICY = "size_based";
/// This class CumulativeCompactionPolicy is the base class of cumulative compaction policy.
/// It defines the policy to do cumulative compaction. It has different derived classes, which implements
diff --git a/be/src/olap/cumulative_compaction_time_series_policy.h b/be/src/olap/cumulative_compaction_time_series_policy.h
index 5273ddad18..3fc3362fa8 100644
--- a/be/src/olap/cumulative_compaction_time_series_policy.h
+++ b/be/src/olap/cumulative_compaction_time_series_policy.h
@@ -21,7 +21,7 @@
namespace doris {
-inline std::string_view CUMULATIVE_TIME_SERIES_POLICY = "TIME_SERIES";
+inline std::string_view CUMULATIVE_TIME_SERIES_POLICY = "time_series";
/// TimeSeries cumulative compaction policy implementation.
/// The following three conditions will be considered when calculating compaction scores and selecting input rowsets in this policy:
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index dea85713f0..fb9d0bdadd 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -77,6 +77,7 @@
#include "olap/binlog.h"
#include "olap/cumulative_compaction.h"
#include "olap/cumulative_compaction_policy.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_bitmap_calculator.h"
#include "olap/memtable.h"
#include "olap/olap_common.h"
@@ -1078,7 +1079,7 @@ uint32_t Tablet::_calc_base_compaction_score() const {
// In the time series compaction policy, we want the base compaction to be triggered
// when there are delete versions present.
- if (config::compaction_policy == "time_series") {
+ if (config::compaction_policy == CUMULATIVE_TIME_SERIES_POLICY) {
return (base_rowset_exist && has_delete) ? score : 0;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org