You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/10/21 02:12:49 UTC
[incubator-doris] branch master updated: [Bug][Compaction] Fix bug
that compaction may be blocked (#4750)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 77835dd [Bug][Compaction] Fix bug that compaction may be blocked (#4750)
77835dd is described below
commit 77835dd9c4db8fa778836b62ff9bfab442423252
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Wed Oct 21 10:12:37 2020 +0800
[Bug][Compaction] Fix bug that compaction may be blocked (#4750)
the logic of compaction producer thread may failed to produce compaction task due to
invalid order of modifying task map.
---
be/src/common/config.h | 4 ++--
be/src/olap/cumulative_compaction.cpp | 2 +-
be/src/olap/cumulative_compaction_policy.cpp | 2 +-
be/src/olap/olap_server.cpp | 9 ++++++---
be/src/olap/tablet_manager.cpp | 1 +
5 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 42edb26..83dc172 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -292,10 +292,10 @@ namespace config {
// rowset will be not given to base compaction. The unit is m byte.
CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64");
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
- // this size, size_based policy also does cumulative compaction. The unit is m byte.
+ // this size, size_based policy may not do to cumulative compaction. The unit is m byte.
CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
- // cumulative compaction policy: max delta file's size unit:B
+ // cumulative compaction policy: min and max delta file's number
CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5");
CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000");
CONF_mInt64(cumulative_compaction_budgeted_bytes, "104857600");
diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp
index 25398a0..e89e6ed 100755
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -44,7 +44,7 @@ OLAPStatus CumulativeCompaction::compact() {
// 1.calculate cumulative point
_tablet->calculate_cumulative_point();
TRACE("calculated cumulative point");
- LOG(INFO) << "after calculate, current cumulative point is " << _tablet->cumulative_layer_point()
+ VLOG(1) << "after calculate, current cumulative point is " << _tablet->cumulative_layer_point()
<< ", tablet=" << _tablet->full_name() ;
// 2. pick rowsets to compact
diff --git a/be/src/olap/cumulative_compaction_policy.cpp b/be/src/olap/cumulative_compaction_policy.cpp
index ee39fb3..39901f3 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -270,7 +270,7 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
rs_iter = input_rowsets->erase(rs_iter);
}
- LOG(INFO) << "cumulative compaction size_based policy, compaction_score = " << *compaction_score
+ VLOG(1) << "cumulative compaction size_based policy, compaction_score = " << *compaction_score
<< ", total_size = " << total_size
<< ", calc promotion size value = " << promotion_size
<< ", tablet = " << tablet->full_name() << ", input_rowset size "
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 0aa8867..fa1aa7b 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -351,6 +351,12 @@ void StorageEngine::_compaction_tasks_producer_callback() {
for (const auto& tablet : tablets_compaction) {
int64_t permits = tablet->calc_compaction_score(compaction_type);
if (_permit_limiter.request(permits)) {
+ {
+ // Push to _tablet_submitted_compaction before submitting task
+ std::unique_lock<std::mutex> lock(_tablet_submitted_compaction_mutex);
+ _tablet_submitted_compaction[tablet->data_dir()].emplace_back(
+ tablet->tablet_id());
+ }
if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
_compaction_thread_pool->submit_func([=]() {
CgroupsMgr::apply_system_cgroup();
@@ -386,9 +392,6 @@ void StorageEngine::_compaction_tasks_producer_callback() {
}
});
}
- std::unique_lock<std::mutex> lock(_tablet_submitted_compaction_mutex);
- _tablet_submitted_compaction[tablet->data_dir()].emplace_back(
- tablet->tablet_id());
}
}
} else {
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 4146c77..ce2d136 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -766,6 +766,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
VLOG(1) << "Found the best tablet for compaction. "
<< "compaction_type=" << compaction_type_str
<< ", tablet_id=" << best_tablet->tablet_id()
+ << ", path=" << data_dir->path()
<< ", highest_score=" << highest_score;
// TODO(lingbin): Remove 'max' from metric name, it would be misunderstood as the
// biggest in history(like peak), but it is really just the value at current moment.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org