You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2021/04/08 01:10:51 UTC
[incubator-doris] branch master updated: [Metrics][LOG] Update
metrics of 'max_compaction_score' and log for compaction (#5592)
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 79544d3 [Metrics][LOG] Update metrics of 'max_compaction_score' and log for compaction (#5592)
79544d3 is described below
commit 79544d39cb5f9614bf8b68d079a3ef170977e87e
Author: weizuo93 <68...@users.noreply.github.com>
AuthorDate: Thu Apr 8 09:10:40 2021 +0800
[Metrics][LOG] Update metrics of 'max_compaction_score' and log for compaction (#5592)
* optimize compaction metrics and log
Co-authored-by: weizuo <we...@xiaomi.com>
---
be/src/olap/compaction.cpp | 5 +++--
be/src/olap/olap_server.cpp | 12 +++++++++++-
be/src/olap/tablet_manager.cpp | 11 ++---------
be/src/olap/tablet_manager.h | 3 ++-
4 files changed, 18 insertions(+), 13 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 35dcbb8..a9255dd 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -133,8 +133,9 @@ OLAPStatus Compaction::do_compaction_impl(int64_t permits) {
LOG(INFO) << "succeed to do " << compaction_name() << ". tablet=" << _tablet->full_name()
<< ", output_version=" << _output_version.first << "-" << _output_version.second
- << ", segments=" << segments_num << ". elapsed time=" << watch.get_elapse_second()
- << "s.";
+ << ", current_max_version=" << _tablet->rowset_with_max_version()->end_version()
+ << ", disk=" << _tablet->data_dir()->path() << ", segments=" << segments_num
+ << ". elapsed time=" << watch.get_elapse_second() << "s.";
return OLAP_SUCCESS;
}
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 19d2865..97bbf84 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -386,6 +386,7 @@ void StorageEngine::_compaction_tasks_producer_callback() {
std::vector<TabletSharedPtr> StorageEngine::_compaction_tasks_generator(
CompactionType compaction_type, std::vector<DataDir*> data_dirs) {
std::vector<TabletSharedPtr> tablets_compaction;
+ uint32_t max_compaction_score = 0;
std::random_shuffle(data_dirs.begin(), data_dirs.end());
for (auto data_dir : data_dirs) {
std::unique_lock<std::mutex> lock(_tablet_submitted_compaction_mutex);
@@ -393,13 +394,22 @@ std::vector<TabletSharedPtr> StorageEngine::_compaction_tasks_generator(
continue;
}
if (!data_dir->reach_capacity_limit(0)) {
+ uint32_t disk_max_score = 0;
TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction(
- compaction_type, data_dir, _tablet_submitted_compaction[data_dir]);
+ compaction_type, data_dir, _tablet_submitted_compaction[data_dir], &disk_max_score);
if (tablet != nullptr) {
tablets_compaction.emplace_back(tablet);
+ max_compaction_score = std::max(max_compaction_score, disk_max_score);
}
}
}
+ if (!tablets_compaction.empty()) {
+ if (compaction_type == CompactionType::BASE_COMPACTION) {
+ DorisMetrics::instance()->tablet_base_max_compaction_score->set_value(max_compaction_score);
+ } else {
+ DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value(max_compaction_score);
+ }
+ }
return tablets_compaction;
}
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index df81c8b..92a4724 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -681,7 +681,7 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) {
TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
CompactionType compaction_type, DataDir* data_dir,
- std::vector<TTabletId>& tablet_submitted_compaction) {
+ std::vector<TTabletId>& tablet_submitted_compaction, uint32_t* score) {
int64_t now_ms = UnixMillis();
const string& compaction_type_str =
compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative";
@@ -776,14 +776,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
<< ", compaction_score=" << compaction_score
<< ", tablet_scan_frequency=" << tablet_scan_frequency
<< ", highest_score=" << highest_score;
- // TODO(lingbin): Remove 'max' from metric name, it would be misunderstood as the
- // biggest in history(like peak), but it is really just the value at current moment.
- if (compaction_type == CompactionType::BASE_COMPACTION) {
- DorisMetrics::instance()->tablet_base_max_compaction_score->set_value(compaction_score);
- } else {
- DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value(
- compaction_score);
- }
+ *score = compaction_score;
}
return best_tablet;
}
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index bbb417d..75eebc5 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -71,7 +71,8 @@ public:
TabletSharedPtr find_best_tablet_to_compaction(CompactionType compaction_type,
DataDir* data_dir,
- vector<TTabletId>& tablet_submitted_compaction);
+ vector<TTabletId>& tablet_submitted_compaction,
+ uint32_t* score);
TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash,
bool include_deleted = false, std::string* err = nullptr);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org