You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/19 06:11:46 UTC
[doris] branch branch-1.2-lts updated: [cherry-pick](compaction) fix promotion size bug #14836 (#15099)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new ef706bf0e7 [cherry-pick](compaction) fix promotion size bug #14836 (#15099)
ef706bf0e7 is described below
commit ef706bf0e75db4a752c692bcfb2d1b96ae09d18e
Author: yixiutt <10...@users.noreply.github.com>
AuthorDate: Mon Dec 19 14:11:40 2022 +0800
[cherry-pick](compaction) fix promotion size bug #14836 (#15099)
---
be/src/common/config.h | 2 +-
be/src/olap/cumulative_compaction_policy.cpp | 20 +++++++++++---------
be/src/olap/cumulative_compaction_policy.h | 11 +++++------
be/src/olap/tablet.cpp | 2 +-
be/src/olap/tablet.h | 11 +++++++++++
be/test/olap/cumulative_compaction_policy_test.cpp | 11 ++---------
6 files changed, 31 insertions(+), 26 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a77a63ecb8..707222447b 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -279,7 +279,7 @@ CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
// cumulative compaction policy: min and max delta file's number
CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5");
-CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000");
+CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "100");
// if compaction of a tablet failed, this tablet should not be chosen to
// compaction until this interval passes.
diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp
index 599fad92be..163e29a5ca 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -76,7 +76,7 @@ void SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point(
CHECK((*base_rowset_meta)->start_version() == 0);
int64_t promotion_size = 0;
- _calc_promotion_size(*base_rowset_meta, &promotion_size);
+ _calc_promotion_size(tablet, *base_rowset_meta, &promotion_size);
int64_t prev_version = -1;
for (const RowsetMetaSharedPtr& rs : existing_rss) {
@@ -119,7 +119,8 @@ void SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point(
}
}
-void SizeBasedCumulativeCompactionPolicy::_calc_promotion_size(RowsetMetaSharedPtr base_rowset_meta,
+void SizeBasedCumulativeCompactionPolicy::_calc_promotion_size(Tablet* tablet,
+ RowsetMetaSharedPtr base_rowset_meta,
int64_t* promotion_size) {
int64_t base_size = base_rowset_meta->total_disk_size();
*promotion_size = base_size * _size_based_promotion_ratio;
@@ -130,12 +131,12 @@ void SizeBasedCumulativeCompactionPolicy::_calc_promotion_size(RowsetMetaSharedP
} else if (*promotion_size <= _size_based_promotion_min_size) {
*promotion_size = _size_based_promotion_min_size;
}
- _refresh_tablet_size_based_promotion_size(*promotion_size);
+ _refresh_tablet_size_based_promotion_size(tablet, *promotion_size);
}
void SizeBasedCumulativeCompactionPolicy::_refresh_tablet_size_based_promotion_size(
- int64_t promotion_size) {
- _tablet_size_based_promotion_size = promotion_size;
+ Tablet* tablet, int64_t promotion_size) {
+ tablet->set_cumulative_promotion_size(promotion_size);
}
void SizeBasedCumulativeCompactionPolicy::update_cumulative_point(
@@ -152,14 +153,14 @@ void SizeBasedCumulativeCompactionPolicy::update_cumulative_point(
// if rowsets have no delete version, check output_rowset total disk size
// satisfies promotion size.
size_t total_size = output_rowset->rowset_meta()->total_disk_size();
- if (total_size >= _tablet_size_based_promotion_size) {
+ if (total_size >= tablet->cumulative_promotion_size()) {
tablet->set_cumulative_layer_point(output_rowset->end_version() + 1);
}
}
}
void SizeBasedCumulativeCompactionPolicy::calc_cumulative_compaction_score(
- TabletState state, const std::vector<RowsetMetaSharedPtr>& all_metas,
+ Tablet* tablet, TabletState state, const std::vector<RowsetMetaSharedPtr>& all_metas,
int64_t current_cumulative_point, uint32_t* score) {
bool base_rowset_exist = false;
const int64_t point = current_cumulative_point;
@@ -200,7 +201,7 @@ void SizeBasedCumulativeCompactionPolicy::calc_cumulative_compaction_score(
// Use "first"(not base) version to calc promotion size
// because some tablet do not have base version(under alter operation)
- _calc_promotion_size(first_meta, &promotion_size);
+ _calc_promotion_size(tablet, first_meta, &promotion_size);
// If base version does not exist, but its state is RUNNING.
// It is abnormal, do not select it and set *score = 0
@@ -240,7 +241,8 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
const int64_t max_compaction_score, const int64_t min_compaction_score,
std::vector<RowsetSharedPtr>* input_rowsets, Version* last_delete_version,
size_t* compaction_score) {
- size_t promotion_size = _tablet_size_based_promotion_size;
+ size_t promotion_size = tablet->cumulative_promotion_size();
+ ;
int transient_size = 0;
*compaction_score = 0;
int64_t total_size = 0;
diff --git a/be/src/olap/cumulative_compaction_policy.h b/be/src/olap/cumulative_compaction_policy.h
index 16e9b3b19b..4cda31e41a 100644
--- a/be/src/olap/cumulative_compaction_policy.h
+++ b/be/src/olap/cumulative_compaction_policy.h
@@ -55,7 +55,7 @@ public:
/// param current_cumulative_point, current cumulative point value.
/// return score, the result score after calculate.
virtual void calc_cumulative_compaction_score(
- TabletState state, const std::vector<RowsetMetaSharedPtr>& all_rowsets,
+ Tablet* tablet, TabletState state, const std::vector<RowsetMetaSharedPtr>& all_rowsets,
int64_t current_cumulative_point, uint32_t* score) = 0;
/// This function implements the policy which represents how to pick the candidate rowsets for compaction.
@@ -154,7 +154,7 @@ public:
/// Num based cumulative compaction policy implements calc cumulative compaction score function.
/// Its main policy is calculating the accumulative compaction score after current cumulative_point in tablet.
- void calc_cumulative_compaction_score(TabletState state,
+ void calc_cumulative_compaction_score(Tablet* tablet, TabletState state,
const std::vector<RowsetMetaSharedPtr>& all_rowsets,
int64_t current_cumulative_point,
uint32_t* score) override;
@@ -163,7 +163,8 @@ public:
private:
/// calculate promotion size using current base rowset meta size and promotion configs
- void _calc_promotion_size(RowsetMetaSharedPtr base_rowset_meta, int64_t* promotion_size);
+ void _calc_promotion_size(Tablet* tablet, RowsetMetaSharedPtr base_rowset_meta,
+ int64_t* promotion_size);
/// calculate the disk size belong to which level, the level is divide by power of 2
/// between cumulative_size_based_promotion_min_size_mbytes
@@ -171,7 +172,7 @@ private:
int _level_size(const int64_t size);
/// when policy calculate cumulative_compaction_score, update promotion size at the same time
- void _refresh_tablet_size_based_promotion_size(int64_t promotion_size);
+ void _refresh_tablet_size_based_promotion_size(Tablet* tablet, int64_t promotion_size);
private:
/// cumulative compaction promotion size, unit is byte.
@@ -182,8 +183,6 @@ private:
int64_t _size_based_promotion_min_size;
/// lower bound size to do compaction compaction.
int64_t _size_based_compaction_lower_bound_size;
- /// record tablet promotion size, it is updated each time when calculate cumulative_compaction_score
- int64_t _tablet_size_based_promotion_size;
/// levels division of disk size, same level rowsets can do compaction
std::vector<int64_t> _levels;
};
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index e15bd61610..d0dd89c0bd 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -826,7 +826,7 @@ const uint32_t Tablet::_calc_cumulative_compaction_score(
#endif
uint32_t score = 0;
_cumulative_compaction_policy->calc_cumulative_compaction_score(
- tablet_state(), _tablet_meta->all_rs_metas(), cumulative_layer_point(), &score);
+ this, tablet_state(), _tablet_meta->all_rs_metas(), cumulative_layer_point(), &score);
return score;
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 31e31d2958..72f8e6a71f 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -85,6 +85,8 @@ public:
const int64_t cumulative_layer_point() const;
void set_cumulative_layer_point(int64_t new_point);
+ inline const int64_t cumulative_promotion_size() const;
+ inline void set_cumulative_promotion_size(int64_t new_size);
// Disk space occupied by tablet, contain local and remote.
size_t tablet_footprint();
@@ -442,6 +444,7 @@ private:
std::atomic<int64_t> _last_base_compaction_success_millis;
std::atomic<int64_t> _last_quick_compaction_success_time_millis;
std::atomic<int64_t> _cumulative_point;
+ std::atomic<int64_t> _cumulative_promotion_size;
std::atomic<int32_t> _newly_created_rowset_num;
std::atomic<int64_t> _last_checkpoint_time;
@@ -511,6 +514,14 @@ inline void Tablet::set_cumulative_layer_point(int64_t new_point) {
_cumulative_point = new_point;
}
+inline const int64_t Tablet::cumulative_promotion_size() const {
+ return _cumulative_promotion_size;
+}
+
+inline void Tablet::set_cumulative_promotion_size(int64_t new_size) {
+ _cumulative_promotion_size = new_size;
+}
+
inline bool Tablet::enable_unique_key_merge_on_write() const {
#ifdef BE_TEST
if (_tablet_meta == nullptr) {
diff --git a/be/test/olap/cumulative_compaction_policy_test.cpp b/be/test/olap/cumulative_compaction_policy_test.cpp
index e0425c36b8..54272110b6 100644
--- a/be/test/olap/cumulative_compaction_policy_test.cpp
+++ b/be/test/olap/cumulative_compaction_policy_test.cpp
@@ -649,11 +649,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, _calc_promotion_size_big) {
_tablet->init();
_tablet->calculate_cumulative_point();
- SizeBasedCumulativeCompactionPolicy* policy =
- dynamic_cast<SizeBasedCumulativeCompactionPolicy*>(
- _tablet->_cumulative_compaction_policy.get());
-
- EXPECT_EQ(1073741824, policy->_tablet_size_based_promotion_size);
+ EXPECT_EQ(1073741824, _tablet->cumulative_promotion_size());
}
TEST_F(TestSizeBasedCumulativeCompactionPolicy, _calc_promotion_size_small) {
@@ -668,10 +664,7 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy, _calc_promotion_size_small) {
_tablet->init();
_tablet->calculate_cumulative_point();
- SizeBasedCumulativeCompactionPolicy* policy =
- dynamic_cast<SizeBasedCumulativeCompactionPolicy*>(
- _tablet->_cumulative_compaction_policy.get());
- EXPECT_EQ(67108864, policy->_tablet_size_based_promotion_size);
+ EXPECT_EQ(67108864, _tablet->cumulative_promotion_size());
}
TEST_F(TestSizeBasedCumulativeCompactionPolicy, _level_size) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org